Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed bug when only vector search produced results in hybrid.

Kishore Nallanc0686a59365a
diff --git a/src/index.cpp b/src/index.cpp
index 65f8f7fc..285b2af8 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3186,17 +3186,26 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
result->scores[result->match_score_index] = float_to_int64_t((1.0 / (i + 1)) * TEXT_MATCH_WEIGHT);
}
- for(int i = 0; i < vec_results.size(); i++) {
- auto& vec_result = vec_results[i];
- auto doc_id = vec_result.first;
+ std::vector<uint32_t> vec_search_ids; // list of IDs found only in vector search
+ for(size_t res_index = 0; res_index < vec_results.size(); res_index++) {
+ auto& vec_result = vec_results[res_index];
+ auto doc_id = vec_result.first;
auto result_it = topster->kv_map.find(doc_id);
- if(result_it != topster->kv_map.end()&& result_it->second->match_score_index >= 0 && result_it->second->match_score_index <= 2) {
+ if(result_it != topster->kv_map.end()) {
+ if(result_it->second->match_score_index < 0 || result_it->second->match_score_index > 2) {
+ continue;
+ }
+
+ // result overlaps with keyword search: we have to combine the scores
+
auto result = result_it->second;
// old_score + (1 / rank_of_document) * WEIGHT)
result->vector_distance = vec_result.second;
- result->scores[result->match_score_index] = float_to_int64_t((int64_t_to_float(result->scores[result->match_score_index])) + ((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT));
+ result->scores[result->match_score_index] = float_to_int64_t(
+ (int64_t_to_float(result->scores[result->match_score_index])) +
+ ((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));
for(size_t i = 0;i < 3; i++) {
if(field_values[i] == &vector_distance_sentinel_value) {
@@ -3209,17 +3218,26 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
} else {
- int64_t scores[3] = {0};
+ // Result has been found only in vector search: we have to add it to both KV and result_ids
// (1 / rank_of_document) * WEIGHT)
- int64_t match_score = float_to_int64_t((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT);
+ int64_t scores[3] = {0};
+ int64_t match_score = float_to_int64_t((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT);
int64_t match_score_index = -1;
compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);
KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);
kv.vector_distance = vec_result.second;
topster->add(&kv);
- ++all_result_ids_len;
+ vec_search_ids.push_back(doc_id);
}
}
+
+ if(!vec_search_ids.empty()) {
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, &vec_search_ids[0],
+ vec_search_ids.size(), &new_all_result_ids);
+ delete[] all_result_ids;
+ all_result_ids = new_all_result_ids;
+ }
}
}
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 995bddd7..b6b598dc 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -711,10 +711,40 @@ TEST_F(CollectionVectorTest, HybridSearchWithExplicitVector) {
4, {off}, 32767, 32767, 2,
false, true, "vec:(" + dummy_vec_string +")");
ASSERT_EQ(true, results_op.ok());
+ ASSERT_EQ(1, results_op.get()["found"].get<size_t>());
+ ASSERT_EQ(1, results_op.get()["hits"].size());
+}
+TEST_F(CollectionVectorTest, HybridSearchOnlyVectorMatches) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string", "facet": true},
+ {"name": "vec", "type": "float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["name"] = "john doe";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results_op = coll1->search("zzz", {"name", "vec"}, "", {"name"}, {}, {0}, 20, 1, FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2);
+ ASSERT_EQ(true, results_op.ok());
ASSERT_EQ(1, results_op.get()["found"].get<size_t>());
ASSERT_EQ(1, results_op.get()["hits"].size());
+ ASSERT_EQ(1, results_op.get()["facet_counts"].size());
+ ASSERT_EQ(4, results_op.get()["facet_counts"][0].size());
+ ASSERT_EQ("name", results_op.get()["facet_counts"][0]["field_name"]);
}
TEST_F(CollectionVectorTest, DistanceThresholdTest) {