Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed a bug where the distance between matched query tokens could exceed previously assumed limit.
Kishore Nallan22e3d36cc0e8
diff --git a/include/index.h b/include/index.hindex 05555a75..87fcc26b 100644--- a/include/index.h+++ b/include/index.h@@ -147,7 +147,6 @@ public:Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);- static const int MAX_SEARCH_TOKENS = 10;static const int SEARCH_LIMIT_NUM = 100; // for limiting number of results on multiple candidates / query rewrites// strings under this length will be fully highlighted, instead of showing a snippet of relevant portiondiff --git a/include/match_score.h b/include/match_score.hindex 10265e44..49edf5c6 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -17,7 +17,7 @@const size_t WINDOW_SIZE = 10;const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();-+const size_t MAX_TOKENS_DISTANCE = 100;struct TokenOffset {uint8_t token_id; // token identifier@@ -180,6 +180,6 @@ struct Match {}pack_token_offsets(min_token_offset, token_offsets.size(), token_start_offset, packed_offset_diffs);- return Match(max_match, min_displacement, token_start_offset, packed_offset_diffs);+ return Match(max_match, std::min(MAX_TOKENS_DISTANCE, (size_t)min_displacement), token_start_offset, packed_offset_diffs);}};diff --git a/src/collection.cpp b/src/collection.cppindex a9d1df12..b8eb9050 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -537,6 +537,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}wrapper_doc["document"] = document;+ //wrapper_doc["match_score"] = field_order_kv.second.match_score;+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.second.key;// highlight query words in the resultconst std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];diff --git a/src/index.cpp b/src/index.cppindex 1ceeb5ba..e005de1f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -694,7 +694,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_const bool prefix_search = prefix && ((token_index == tokens.size()-1) ? true : false);const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;- // If this is a prefix search, we should for more candidates and do a union of those document IDs+ // If this is a prefix search, look for more candidates and do a union of those document IDsconst int max_candidates = prefix_search ? 10 : 3;art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,costs[token_index], costs[token_index], max_candidates, token_order, prefix_search, leaves);@@ -846,7 +846,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &Match single_token_match = Match(1, 0, 0, empty_offset_diffs);const uint64_t single_token_match_score = ((int64_t)(single_token_match.words_present) << 24) |((int64_t)(255 - total_cost) << 16) |- ((int64_t)(MAX_SEARCH_TOKENS - single_token_match.distance));+ ((int64_t)abs(int(MAX_TOKENS_DISTANCE - (size_t)single_token_match.distance)));for(size_t i=0; i<result_size; i++) {const uint32_t seq_id = result_ids[i];@@ -863,7 +863,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &// Construct a single match score from individual components (for multi-field sort)match_score = ((int64_t)(match.words_present) << 24) |((int64_t)(255 - total_cost) << 16) |- ((int64_t)(MAX_SEARCH_TOKENS - match.distance));+ ((int64_t)(MAX_TOKENS_DISTANCE - match.distance));}const int64_t default_score = 0;@@ -884,12 +884,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);- /*std::ostringstream os;+ /*+ std::ostringstream os;os << name << ", total_cost: " << (255 - total_cost)- << ", words_present: " << match.words_present << ", match_score: " << match- << ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance)- << ", seq_id: " << seq_id << std::endl;- LOG(INFO) << os.str();*/+ << ", words_present: " << match.words_present << ", match_score: " << match_score+ << ", match.distance: " << match.distance+ << ", distance: "+ << (MAX_TOKENS_DISTANCE - match.distance)+ << ", seq_id: " << seq_id << std::endl;+ LOG(INFO) << os.str();+ */}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();