Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed a bug where the distance between matched query tokens could exceed previously assumed limit.

Kishore Nallan22e3d36cc0e8
diff --git a/include/index.h b/include/index.h
index 05555a75..87fcc26b 100644
--- a/include/index.h
+++ b/include/index.h
@@ -147,7 +147,6 @@ public:
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);
- static const int MAX_SEARCH_TOKENS = 10;
static const int SEARCH_LIMIT_NUM = 100; // for limiting number of results on multiple candidates / query rewrites
// strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
diff --git a/include/match_score.h b/include/match_score.h
index 10265e44..49edf5c6 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -17,7 +17,7 @@
const size_t WINDOW_SIZE = 10;
const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();
-
+const size_t MAX_TOKENS_DISTANCE = 100;
struct TokenOffset {
uint8_t token_id; // token identifier
@@ -180,6 +180,6 @@ struct Match {
}
pack_token_offsets(min_token_offset, token_offsets.size(), token_start_offset, packed_offset_diffs);
- return Match(max_match, min_displacement, token_start_offset, packed_offset_diffs);
+ return Match(max_match, std::min(MAX_TOKENS_DISTANCE, (size_t)min_displacement), token_start_offset, packed_offset_diffs);
}
};
diff --git a/src/collection.cpp b/src/collection.cpp
index a9d1df12..b8eb9050 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -537,6 +537,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
wrapper_doc["document"] = document;
+ //wrapper_doc["match_score"] = field_order_kv.second.match_score;
+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.second.key;
// highlight query words in the result
const std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];
diff --git a/src/index.cpp b/src/index.cpp
index 1ceeb5ba..e005de1f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -694,7 +694,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
const bool prefix_search = prefix && ((token_index == tokens.size()-1) ? true : false);
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
- // If this is a prefix search, we should for more candidates and do a union of those document IDs
+ // If this is a prefix search, look for more candidates and do a union of those document IDs
const int max_candidates = prefix_search ? 10 : 3;
art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], max_candidates, token_order, prefix_search, leaves);
@@ -846,7 +846,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
Match single_token_match = Match(1, 0, 0, empty_offset_diffs);
const uint64_t single_token_match_score = ((int64_t)(single_token_match.words_present) << 24) |
((int64_t)(255 - total_cost) << 16) |
- ((int64_t)(MAX_SEARCH_TOKENS - single_token_match.distance));
+ ((int64_t)abs(int(MAX_TOKENS_DISTANCE - (size_t)single_token_match.distance)));
for(size_t i=0; i<result_size; i++) {
const uint32_t seq_id = result_ids[i];
@@ -863,7 +863,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
// Construct a single match score from individual components (for multi-field sort)
match_score = ((int64_t)(match.words_present) << 24) |
((int64_t)(255 - total_cost) << 16) |
- ((int64_t)(MAX_SEARCH_TOKENS - match.distance));
+ ((int64_t)(MAX_TOKENS_DISTANCE - match.distance));
}
const int64_t default_score = 0;
@@ -884,12 +884,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;
topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);
- /*std::ostringstream os;
+ /*
+ std::ostringstream os;
os << name << ", total_cost: " << (255 - total_cost)
- << ", words_present: " << match.words_present << ", match_score: " << match
- << ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance)
- << ", seq_id: " << seq_id << std::endl;
- LOG(INFO) << os.str();*/
+ << ", words_present: " << match.words_present << ", match_score: " << match_score
+ << ", match.distance: " << match.distance
+ << ", distance: "
+ << (MAX_TOKENS_DISTANCE - match.distance)
+ << ", seq_id: " << seq_id << std::endl;
+ LOG(INFO) << os.str();
+ */
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();