Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Stricter bounding of typo correction threshold.
Kishore Nallanb6f1885aec50
diff --git a/src/index.cpp b/src/index.cppindex ac0f953f..f483cfdf 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2052,10 +2052,18 @@ void Index::search_field(const uint8_t & field_id,int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {int bounded_cost = max_cost;- if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {- bounded_cost = token_len - 1;++ if(token_len < 4) {+ // typo correction is disabled for small tokens+ return 0;+ }++ if(token_len < 7) {+ // 2-typos are enabled only at token length of 7 chars+ return std::min<int>(max_cost, 1);}- return bounded_cost;++ return std::min<int>(max_cost, 2);}void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 8e088361..245aa490 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -463,9 +463,9 @@ TEST_F(CollectionSortingTest, SingleFieldTextMatchScoreDefault) {}std::vector<std::vector<std::string>> records = {- {"Ayxha Beta"},- {"Alpha Beta"},{"Alppha Beta"},+ {"Alpha Beta"},+ {"Alphas Beta"},};for(size_t i=0; i<records.size(); i++) {diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 1811c45a..42e1e9d5 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -363,13 +363,14 @@ TEST_F(CollectionTest, QueryWithTypo) {}results.clear();- results = collection->search("fer thx", query_fields, "", facets, sort_fields, {1}, 3,+ results = collection->search("lauxnch rcket", query_fields, "", facets, sort_fields, {1}, 3,1, FREQUENCY,{false}, 10,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10).get();- ids = {"1", "10", "13"};++ ids = {"8", "1", "17"};ASSERT_EQ(3, results["hits"].size());@@ -442,15 +443,15 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {}TEST_F(CollectionTest, TextContainingAnActualTypo) {- // A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens+ // A line contains "ISSX" but not "what" - need to ensure that correction to "ISSS what" happensstd::vector<std::string> facets;- nlohmann::json results = collection->search("ISX what", query_fields, "", facets, sort_fields, {1}, 4, 1, FREQUENCY, {false},+ nlohmann::json results = collection->search("ISSX what", query_fields, "", facets, sort_fields, {1}, 4, 1, FREQUENCY, {false},10, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(),10, "", 30, 5, "", 10).get();ASSERT_EQ(4, results["hits"].size());- ASSERT_EQ(13, results["found"].get<uint32_t>());+ ASSERT_EQ(11, results["found"].get<uint32_t>());- std::vector<std::string> ids = {"8", "19", "6", "21"};+ std::vector<std::string> ids = {"19", "6", "21", "22"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -460,14 +461,15 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}// Record containing exact token match should appear first- results = collection->search("ISX", query_fields, "", facets, sort_fields, {1}, 10, 1, FREQUENCY, {false}, 10,+ results = collection->search("ISSX", query_fields, "", facets, sort_fields, {1}, 10, 1, FREQUENCY, {false}, 10,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10).get();- ASSERT_EQ(8, results["hits"].size());- ASSERT_EQ(8, results["found"].get<uint32_t>());- ids = {"20", "19", "6", "4", "3", "10", "8", "21"};+ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(5, results["found"].get<uint32_t>());++ ids = {"20", "19", "6", "3", "21"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -3164,8 +3166,8 @@ TEST_F(CollectionTest, MultiFieldRelevance4) {}std::vector<std::vector<std::string>> records = {- {"Madras Dreams", "Chennai King"},- {"Madurai Express", "Madura Maddy"},+ {"Maddras Dreams", "Chennai King"},+ {"Maddurai Express", "Maddura Maddy"},};for(size_t i=0; i<records.size(); i++) {@@ -3179,7 +3181,7 @@ TEST_F(CollectionTest, MultiFieldRelevance4) {ASSERT_TRUE(coll1->add(doc.dump()).ok());}- auto results = coll1->search("madras",+ auto results = coll1->search("maddras",{"title", "artist"}, "", {}, {}, {2}, 10, 1, FREQUENCY,{true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,diff --git a/test/documents.jsonl b/test/documents.jsonlindex ae97bbd6..8caaa289 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -1,9 +1,9 @@{"points":15,"title":"How are cryogenic rocket propellants delivered for the launch pad?"}{"points":14,"title":"Are there any (free) online data archives for data from instruments on Soviet / Russian missions?"}-{"points":13,"title":"Where should I look in ISS to find mouldy food?"}+{"points":13,"title":"Where should I look in ISSS to find mouldy food?"}{"points":13,"title":"Is solar system active cryovolcanism a potential viable power source for future colonies?"}{"id": "foo", "points":13,"title":"The heaviest martian spacecraft"}-{"points":13,"title":"To what extent are the US modules of ISS based on the Spacelab design?"}+{"points":13,"title":"To what extent are the US modules of ISSS based on the Spacelab design?"}{"points":12,"title":"Could future astronauts eat during EVAs?"}{"points":12,"title":"What is the power, requirement of a rocket launch these days?"}{"points":12,"title":"How does plant growing medium not scatter around?"}@@ -16,9 +16,9 @@{"points":10,"title":"How late do the launch propellants ionize in a chemical rocket mission?"}{"points":8,"title":"How much does it cost to launch (right from start) a rocket today?"}{"points":16,"title":"Difference between Space Dynamics & Astrodynamics in engineering perspective?"}-{"points":18,"title":"What kind of biological research does ISS do then?"}-{"points":10,"title":"Which kinds of radiation hit ISX ?"}-{"points":7,"title":"What kinds of things have been tossed out of ISS in space?"}+{"points":18,"title":"What kind of biological research does ISSS do then?"}+{"points":10,"title":"Which kinds of radiation hit ISSX ?"}+{"points":7,"title":"What kinds of things have been tossed out of ISSS in space?"}{"points":17,"title":"What does triple redundant closed loop digital avionics system mean?"}{"points":11,"title":"How are rockets guided to follow specific loop trajectory?"}{"points":8,"title":"What do remotely controlled bolts look like?"}\ No newline at end of file