Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix an edge case in match score calculation.
kishorence254a1d5190b
diff --git a/include/match_score.h b/include/match_score.hindex d9f039b4..cc6667bd 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -100,7 +100,7 @@ struct Match {Create vector with first offset from each token.Sort vector descending.Calculate distance, use only tokens within max window size from lowest offset.- Reassign best distance and window if found.+ Reassign best window and distance if found.Pop end of vector (smallest offset).Push to vector next offset of token just popped.Until queue size is 1.@@ -157,7 +157,8 @@ struct Match {this_window[window.back().token_id] = window.back();}- if (this_num_match > best_num_match && this_displacement < best_displacement) {+ if ( (this_num_match > best_num_match) ||+ (this_num_match == best_num_match && this_displacement < best_displacement)) {best_displacement = this_displacement;best_num_match = this_num_match;if(populate_window) {@@ -165,7 +166,7 @@ struct Match {}}- if (best_displacement == (window.size() - 1)) {+ if (best_num_match == tokens_size && best_displacement == (window.size() - 1)) {// this is the best we can get, so quit early!break;}diff --git a/src/collection.cpp b/src/collection.cppindex f6019188..b4c5e3ee 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -227,6 +227,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {nlohmann::json index_res;index_res["error"] = "Max memory ratio exceeded.";index_res["success"] = false;+ index_res["document"] = json_line;json_lines[i] = index_res.dump();continue;@@ -285,7 +286,11 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batchjson_out[index_record.position] = R"({"success": true})";num_indexed++;} else {- json_out[index_record.position] = R"({"success": false, "error": ")" + index_record.indexed.error() + "\"}";+ nlohmann::json res;+ res["success"] = false;+ res["error"] = index_record.indexed.error();+ res["document"] = index_record.document.dump();+ json_out[index_record.position] = res.dump();}}}@@ -1266,7 +1271,7 @@ void Collection::highlight_result(const field &search_field,std::vector<size_t> token_indices;spp::sparse_hash_set<std::string> token_hits;- for(size_t i = 0; i < match.words_present; i++) {+ for(size_t i = 0; i < match.offsets.size(); i++) {if(match.offsets[i].offset != MAX_DISPLACEMENT) {size_t token_index = (size_t)(match.offsets[i].offset);token_indices.push_back(token_index);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 461da75c..f851c178 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1254,7 +1254,7 @@ TEST_F(CollectionTest, FilterOnFloatFields) {collectionManager.drop_collection("coll_array_fields");}-std::vector<nlohmann::json> import_res_to_json(std::vector<std::string> imported_results) {+std::vector<nlohmann::json> import_res_to_json(const std::vector<std::string>& imported_results) {std::vector<nlohmann::json> out;for(const auto& imported_result: imported_results) {@@ -1343,13 +1343,15 @@ TEST_F(CollectionTest, ImportDocuments) {ASSERT_STREQ("Field `title` must be a string.", import_results[1]["error"].get<std::string>().c_str());ASSERT_STREQ("Field `starring` has been declared in the schema, but is not found in the document.",import_results[3]["error"].get<std::string>().c_str());+ ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"19\",\"points\":23,\"starring\":\"Jazz Gosh\",\"title\":123}",+ import_results[1]["document"].get<std::string>().c_str());// record with duplicate IDsmore_records = {"{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "- "\"cast\": [\"Tom Skerritt\"] }",- "{\"id\": \"id2\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "- "\"cast\": [\"Tom Skerritt\"] }"};+ "\"cast\": [\"Tom Skerritt\"] }",+ "{\"id\": \"id2\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "+ "\"cast\": [\"Tom Skerritt\"] }"};import_response = coll_mul_fields->add_many(more_records);@@ -1362,6 +1364,8 @@ TEST_F(CollectionTest, ImportDocuments) {ASSERT_TRUE(import_results[1]["success"].get<bool>());ASSERT_STREQ("A document with id id1 already exists.", import_results[0]["error"].get<std::string>().c_str());+ ASSERT_STREQ("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, \"cast\": [\"Tom Skerritt\"] }",+ import_results[0]["document"].get<std::string>().c_str());// handle bad import json@@ -1373,7 +1377,10 @@ TEST_F(CollectionTest, ImportDocuments) {import_results = import_res_to_json(more_records);ASSERT_EQ(1, import_results.size());- ASSERT_STREQ("Bad JSON.", import_results[0]["error"].get<std::string>().c_str());++ ASSERT_EQ(false, import_results[0]["success"].get<bool>());+ ASSERT_STREQ("Bad JSON: not a properly formed document.", import_results[0]["error"].get<std::string>().c_str());+ ASSERT_STREQ("[]", import_results[0]["document"].get<std::string>().c_str());collectionManager.drop_collection("coll_mul_fields");}@@ -1747,7 +1754,9 @@ TEST_F(CollectionTest, IndexingWithBadData) {doc_str = "{\"name\": \"foo\", \"age\": asdadasd, \"tags\": [], \"average\": 34 }";const Option<nlohmann::json> & bad_default_sorting_field_op4 = sample_collection->add(doc_str);ASSERT_FALSE(bad_default_sorting_field_op4.ok());- ASSERT_STREQ("Bad JSON.", bad_default_sorting_field_op4.error().c_str());+ ASSERT_STREQ("Bad JSON: [json.exception.parse_error.101] parse error at line 1, column 24: syntax error "+ "while parsing value - invalid literal; last read: '\"age\": a'",+ bad_default_sorting_field_op4.error().c_str());// should return an error when a document with pre-existing id is being addedstd::string doc = "{\"id\": \"100\", \"name\": \"foo\", \"age\": 29, \"tags\": [], \"average\": 78}";@@ -2035,6 +2044,17 @@ TEST_F(CollectionTest, SearchLargeTextField) {ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ // only single matched token in match window++ res_op = coll_large_text->search("molestie maecenas accumsan", {"text"}, "", {}, sort_fields, 0, 10);+ ASSERT_TRUE(res_op.ok());+ results = res_op.get();++ ASSERT_EQ(1, results["hits"].size());++ ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo.",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+collectionManager.drop_collection("coll_large_text");}diff --git a/test/large_text_field.jsonl b/test/large_text_field.jsonlindex 979d98f0..125a5a64 100644--- a/test/large_text_field.jsonl+++ b/test/large_text_field.jsonl@@ -1,2 +1,2 @@-{"age": 44, "text": "Fusce tellus orci, lacinia non egestas sit amet, vulputate in felis. Donec eu metus ac ligula sodales semper vitae ornare ante. Aenean viverra sollicitudin rhoncus. Fusce laoreet tincidunt dui, lacinia semper odio pellentesque ut. Sed arcu velit, lobortis nec nunc non, rutrum mollis diam. Morbi non felis tincidunt, ornare mauris facilisis, hendrerit erat. Proin feugiat commodo tincidunt. Quisque nec quam non odio elementum gravida. Cras porttitor fermentum sapien, non pulvinar tortor faucibus et. Praesent tristique nunc ac sem sagittis congue eu ut felis. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Cras aliquet nunc vel leo scelerisque, eu finibus urna mattis. Phasellus eget sem eu urna dignissim rhoncus. Morbi sed tincidunt lectus. Fusce non euismod metus. Vivamus id faucibus eros. Aenean vestibulum quis mauris nec auctor. Nulla scelerisque, odio quis hendrerit blandit, enim elit scelerisque velit, eget rutrum arcu metus sed felis. Pellentesque augue ante, rhoncus vel purus et, fermentum facilisis erat. Maecenas tincidunt facilisis urna, vitae convallis erat sollicitudin eu. Maecenas nec euismod nisi, eget faucibus mauris. Nulla facilisi. Phasellus vitae sapien vehicula, fermentum nulla a, sollicitudin libero. Praesent volutpat, est non porttitor sagittis, felis nulla mollis nunc, vitae viverra magna arcu et augue. Cras volutpat, leo id imperdiet laoreet, lorem nisl efficitur nunc, at viverra ligula leo in felis. Nulla non tincidunt erat. Aliquam id ultricies augue. Nullam convallis interdum ex. Proin quis ipsum nec dui varius ullamcorper. Quisque id interdum nulla, in sagittis metus. Nullam tristique elementum enim non porttitor. Aliquam pretium, purus a ullamcorper sodales, dui nunc rhoncus enim, eget vestibulum urna lacus non lacus. Phasellus nec nisl hendrerit, finibus magna eu, varius augue. Praesent vehicula nisi nec neque tincidunt, pulvinar ornare nisl dignissim. Mauris lectus nisi, vulputate a ultrices et, gravida vel dui. Nulla non quam feugiat, eguilazer finibus metus eu."}+{"age": 44, "text": "Fusce tellus orci, lacinia non egestas sit amet, vulputate in felis. Donec eu metus ac ligula sodales semper vitae ornare ante. Aenean viverra sollicitudin rhoncus. Fusce laoreet tincidunt dui, lacinia semper odio pellentesque ut. Sed arcu velit, lobortis nec nunc non, rutrum mollis diam. Morbi non felis tincidunt, ornare mauris facilisis, hendrerit erat. Proin feugiat commodo tincidunt. Quisque nec quam non odio elementum gravida. Cras porttitor fermentum sapien, non pulvinar tortor faucibus et. Praesent tristique nunc ac sem sagittis congue eu ut felis. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Cras aliquet nunc vel leo scelerisque, eu finibus urna mattis. Phasellus eget sem eu urna dignissim rhoncus. Morbi sed tincidunt lectus. Fusce non euismod metus. Vivamus id faucibus eros. Aenean vestibulum quis mauris nec auctor. Nulla scelerisque, odio quis hendrerit blandit, enim elit scelerisque velit, eget rutrum arcu metus sed felis. Pellentesque augue ante, rhoncus vel purus et, fermentum facilisis erat. Maecenax tincidunt facilisis urna, vitae convallis erat sollicitudin eu. Maecenax nec euismod nisi, eget faucibus mauris. Nulla facilisi. Phasellus vitae sapien vehicula, fermentum nulla a, sollicitudin libero. Praesent volutpat, est non porttitor sagittis, felis nulla mollis nunc, vitae viverra magna arcu et augue. Cras volutpat, leo id imperdiet laoreet, lorem nisl efficitur nunc, at viverra ligula leo in felis. Nulla non tincidunt erat. Aliquam id ultricies augue. Nullam convallis interdum ex. Proin quis ipsum nec dui varius ullamcorper. Quisque id interdum nulla, in sagittis metus. Nullam tristique elementum enim non porttitor. Aliquam pretium, purus a ullamcorper sodales, dui nunc rhoncus enim, eget vestibulum urna lacus non lacus. Phasellus nec nisl hendrerit, finibus magna eu, varius augue. Praesent vehicula nisi nec neque tincidunt, pulvinar ornare nisl dignissim. Mauris lectus nisi, vulputate a ultrices et, gravida vel dui. Nulla non quam feugiat, eguilazer finibus metus eu."}{"age": 22, "text": "Phasellus non tristique elit. Praesent non arcu id lectus accumsan venenatis at at justo. Sed convallis auctor neque, ut ullamcorper augue ullamcorper quis. Vestibulum pellentesque varius lectus nec semper. Vivamus non feugiat felis, id malesuada augue. Phasellus posuere est ut lacinia scelerisque. Donec feugiat varius turpis sit amet rhoncus. Donec massa turpis, viverra ut libero a, pretium ultricies nunc. Praesent faucibus ullamcorper ligula, vel tempus ipsum. Nunc magna risus, porta eget bibendum vitae, venenatis non nibh. Aliquam a nisi malesuada, venenatis elit venenatis, tempus libero. Sed dolor nulla, fermentum vel rhoncus at, sollicitudin id dolor. Sed dignissim quam vel porttitor ornare. Mauris at congue nulla, nec eleifend eros. Suspendisse ac felis vel lectus venenatis tincidunt nec ac enim. Quisque et lectus feugiat libero varius fermentum. Curabitur quis dapibus lectus. Praesent tincidunt id quam eu elementum. Proin quis magna eu lectus vestibulum fringilla. Quisque ut felis lectus. Nam vulputate nibh non odio convallis hendrerit. Mauris mollis tempor suscipit. Morbi luctus dui ac augue interdum tempor. Aenean sodales, sapien sed molestie blandit, nunc lorem pulvinar massa, vitae tincidunt quam quam id libero. Vivamus diam sem, volutpat facilisis rutrum quis, fermentum quis risus. Morbi sed pharetra quam. Morbi leo velit, fringilla id odio sit amet, bibendum egestas nunc. Nam elementum dolor pulvinar gravida pulvinar. Sed imperdiet libero eu tellus interdum consectetur. Maecenas in pulvinar odio. Integer malesuada blandit enim eu fermentum. Morbi tristique erat et lacus feugiat, a iaculis ligula pellentesque. Nullam ut luctus enim, nec molestie risus. Nullam venenatis ut velit ac lobortis. Aenean eget odio eleifend, laoreet felis vel, sagittis tellus. Nullam ex risus, dictum non dolor et, blandit accumsan ante. Etiam neque erat, efficitur vitae efficitur ut, ornare et leo. Aliquam erat volutpat. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec at laoreet orci. Maecenas."}\ No newline at end of filediff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 3ca48140..ae6a1047 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -16,15 +16,62 @@ TEST(MatchTest, TokenOffsetsExceedWindowSize) {TEST(MatchTest, MatchScoreV2) {std::vector<std::vector<uint16_t>> token_offsets;+ token_offsets.push_back({25});+ token_offsets.push_back({26});+ token_offsets.push_back({11, 18, 24, 60});+ token_offsets.push_back({14, 27, 63});++ auto match = Match(100, token_offsets, true);+ ASSERT_EQ(4, match.words_present);+ ASSERT_EQ(97, match.distance);++ std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};+ for(size_t i=0; i<token_offsets.size(); i++) {+ ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);+ }++ // without populate window+ match = Match(100, token_offsets, false);+ ASSERT_EQ(4, match.words_present);+ ASSERT_EQ(97, match.distance);+ ASSERT_EQ(0, match.offsets.size());++ token_offsets.clear();token_offsets.push_back({38, 50, 170, 187, 195, 222});token_offsets.push_back({39, 140, 171, 189, 223});token_offsets.push_back({169, 180});-// token_offsets.push_back({38, 50, 187, 195, 201});-// token_offsets.push_back({120, 167, 171, 223}); // 39,-// token_offsets.push_back({240, 250});+ match = Match(100, token_offsets, true);+ ASSERT_EQ(3, match.words_present);+ ASSERT_EQ(98, match.distance);++ expected_offsets = {170, 171, 169};+ for(size_t i=0; i<token_offsets.size(); i++) {+ ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);+ }++ token_offsets.clear();+ token_offsets.push_back({38, 50, 187, 195, 201});+ token_offsets.push_back({120, 167, 171, 223});+ token_offsets.push_back({240, 250});- size_t total_distance = 0, words_present = 0, offset_sum = 0;+ match = Match(100, token_offsets, true);+ ASSERT_EQ(1, match.words_present);+ ASSERT_EQ(100, match.distance);++ expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};+ for(size_t i=0; i<token_offsets.size(); i++) {+ ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);+ }++ // without populate window+ match = Match(100, token_offsets, false);+ ASSERT_EQ(1, match.words_present);+ ASSERT_EQ(100, match.distance);+ ASSERT_EQ(0, match.offsets.size());+++ /*size_t total_distance = 0, words_present = 0, offset_sum = 0;auto begin = std::chrono::high_resolution_clock::now();for(size_t i = 0; i < 1; i++) {@@ -37,7 +84,5 @@ TEST(MatchTest, MatchScoreV2) {uint64_t timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();LOG(INFO) << "Time taken: " << timeNanos;-- LOG(INFO) << total_distance << ", " << words_present << ", " << offset_sum;-+ LOG(INFO) << total_distance << ", " << words_present << ", " << offset_sum;*/}\ No newline at end of file