Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Exact matches should be ranked above other matches.
kishorenc1387f1c21b0c
diff --git a/include/match_score.h b/include/match_score.hindex 8cdcec07..5224c9dd 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -50,7 +50,7 @@ struct Match {uint64_t match_score = ((int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |- (int64_t(distance))+ (int64_t(100 - distance)));return match_score;@@ -61,7 +61,7 @@ struct Match {uint64_t match_score = ((int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |- (int64_t(distance))+ (int64_t(100 - distance)));return match_score;@@ -206,10 +206,8 @@ struct Match {best_displacement = 0;}- uint8_t best_distance = uint8_t(100 - best_displacement);-words_present = best_num_match;- distance = best_distance;+ distance = uint8_t(best_displacement);if(populate_window) {offsets = best_window;}diff --git a/src/collection.cpp b/src/collection.cppindex c5be2187..a98e0e1a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -579,10 +579,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// process weights for search fieldsstd::vector<search_field_t> weighted_search_fields;- if(query_by_weights.empty() && !search_fields.empty()) {- for(int i=search_fields.size(); i > 0; i--) {+ if(query_by_weights.empty()) {+ for(size_t i=1; i <= search_fields.size(); i++) {query_by_weights.push_back(i);}+ } else {+ auto max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());+ for(size_t i=0; i < query_by_weights.size(); i++) {+ query_by_weights[i] = (max_weight - query_by_weights[i]) + 1;+ }}for(size_t i=0; i < search_fields.size(); i++) {diff --git a/src/index.cpp b/src/index.cppindex b2cf8566..c272eeec 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1636,6 +1636,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,all_result_ids = filter_ids;filter_ids = nullptr;} else {+ // In multi-field searches, a record can be matched across different fields, so we use this for aggregationspp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;std::vector<token_t> q_include_pos_tokens;@@ -1700,48 +1701,42 @@ void Index::search(const std::vector<std::string>& q_include_tokens,}}- for(const auto& key_kvs: topster_ids) {- // first calculate existing aggregate scores across best matching fields- spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;-- const auto& kvs = key_kvs.second;- const uint64_t seq_id = key_kvs.first;+ for(const auto& seq_id_kvs: topster_ids) {+ const uint64_t seq_id = seq_id_kvs.first;+ const auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field// LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index];- /*if(seq_id == 12 || seq_id == 15) {- LOG(INFO) << "here";- }*/-+ // to calculate existing aggregate scores across best matching fields+ spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;for(const auto kv: kvs) {existing_field_kvs.emplace(kv->field_id, kv);}- uint64_t aggregated_score = 0;uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set+ uint64_t total_typos = 0, total_distances = 0;//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);- for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {+ for(size_t i = 0; i < num_search_fields; i++) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);size_t weight = search_fields[i].weight;- //LOG(INFO) << "--- field index: " << i;-- if(field_id == kvs[0]->field_id) {- token_bits |= kvs[0]->token_bits;- //LOG(INFO) << "kvs[0]->field_id pop count: " << __builtin_popcount(token_bits);- aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);- continue;- }+ //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;if(existing_field_kvs.count(field_id) != 0) {// for existing field, we will simply sum field-wise weighted scorestoken_bits |= existing_field_kvs[field_id]->token_bits;//LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);- aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]- * weight);+ int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];+ total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;+ total_typos += ((255 - ((match_score >> 8) & 0xFF)) + 1) * weight;++ /*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))+ << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * weight+ << ", total dist: " << (((match_score & 0xFF)))+ << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * weight;*/continue;}@@ -1780,14 +1775,25 @@ void Index::search(const std::vector<std::string>& q_include_tokens,}if(words_present != 0) {- uint64_t match_score = Match::get_match_score(words_present, 0, 100);- aggregated_score += (match_score * weight);+ uint64_t match_score = Match::get_match_score(words_present, 0, 0);+ total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;+ total_typos += ((255 - ((match_score >> 8) & 0xFF)) + 1) * weight;+ //LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);}}- aggregated_score |= (uint64_t(__builtin_popcount(token_bits)) << 48);+ int64_t tokens_present = int64_t(__builtin_popcount(token_bits)) - 1;+ total_typos = std::min<uint64_t>(255, total_typos);+ total_distances = std::min<uint64_t>(100, total_distances);++ uint64_t aggregated_score = (+ (tokens_present << 16) |+ ((255 - total_typos) << 8) |+ (100 - total_distances)+ );- /*LOG(INFO) << "seq id: " << seq_id << ", pop count: " << __builtin_popcount(token_bits)+ /*LOG(INFO) << "seq id: " << seq_id << ", tokens_present: " << tokens_present+ << ", total_distances: " << total_distances << ", total_typos: " << total_typos<< ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;@@ -2154,6 +2160,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16groups_processed.emplace(distinct_id);}+ //LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);topster->add(&kv);}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 448a7850..49428829 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -859,8 +859,6 @@ TEST_F(CollectionTest, MultipleFields) {query_fields = {"title", "starring"};std::vector<std::string> facets;- auto x = coll_mul_fields->search("Will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false);-nlohmann::json results = coll_mul_fields->search("Will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(4, results["hits"].size());@@ -2901,6 +2899,61 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ results = coll1->search("swift",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 1}).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionTest, MultiFieldRelevance4) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Madras Dreams", "Chennai King"},+ {"Madurai Express", "Madura Maddy"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("madras",+ {"title", "artist"}, "", {}, {}, 2, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 1}).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());+collectionManager.drop_collection("coll1");}diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex ae6a1047..0be9931a 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -23,7 +23,7 @@ TEST(MatchTest, MatchScoreV2) {auto match = Match(100, token_offsets, true);ASSERT_EQ(4, match.words_present);- ASSERT_EQ(97, match.distance);+ ASSERT_EQ(3, match.distance);std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};for(size_t i=0; i<token_offsets.size(); i++) {@@ -33,7 +33,7 @@ TEST(MatchTest, MatchScoreV2) {// without populate windowmatch = Match(100, token_offsets, false);ASSERT_EQ(4, match.words_present);- ASSERT_EQ(97, match.distance);+ ASSERT_EQ(3, match.distance);ASSERT_EQ(0, match.offsets.size());token_offsets.clear();@@ -43,7 +43,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true);ASSERT_EQ(3, match.words_present);- ASSERT_EQ(98, match.distance);+ ASSERT_EQ(2, match.distance);expected_offsets = {170, 171, 169};for(size_t i=0; i<token_offsets.size(); i++) {@@ -57,7 +57,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true);ASSERT_EQ(1, match.words_present);- ASSERT_EQ(100, match.distance);+ ASSERT_EQ(0, match.distance);expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};for(size_t i=0; i<token_offsets.size(); i++) {@@ -67,7 +67,7 @@ TEST(MatchTest, MatchScoreV2) {// without populate windowmatch = Match(100, token_offsets, false);ASSERT_EQ(1, match.words_present);- ASSERT_EQ(100, match.distance);+ ASSERT_EQ(0, match.distance);ASSERT_EQ(0, match.offsets.size());