Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle references with vector, hybrid and infix search.

Harpreet Sangar6a2b9549d5e4
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index b705b122..f0d26c82 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -37,8 +37,21 @@ struct single_filter_result_t {
single_filter_result_t() = default;
- single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t> reference_filter_results) :
+ single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results) :
seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)) {}
+
+ single_filter_result_t(const single_filter_result_t& obj) {
+ if (&obj == this)
+ return;
+
+ seq_id = obj.seq_id;
+
+ // Copy every collection's reference.
+ for (const auto &item: obj.reference_filter_results) {
+ auto& ref_coll_name = item.first;
+ reference_filter_results[ref_coll_name] = item.second;
+ }
+ }
};
struct filter_result_t {
@@ -232,6 +245,8 @@ public:
/// \return size of the results array
uint32_t and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results);
+ void and_scalar(const uint32_t* A, const uint32_t& lenA, filter_result_t& result);
+
static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,
uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);
};
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index dcc2dbdf..08933c70 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -1411,6 +1411,72 @@ uint32_t filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t&
return filter_ids.size();
}
+void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenA, filter_result_t& result) {
+ if (!is_valid) {
+ return;
+ }
+
+ if (filter_result.reference_filter_results.empty()) {
+ if (is_filter_result_initialized) {
+ result.count = ArrayUtils::and_scalar(A, lenA, filter_result.docs, filter_result.count, &result.docs);
+ return;
+ }
+
+ std::vector<uint32_t> filter_ids;
+ for (uint32_t i = 0; i < lenA; i++) {
+ auto _result = valid(A[i]);
+
+ if (_result == -1) {
+ break;
+ }
+
+ if (_result == 1) {
+ filter_ids.push_back(A[i]);
+ }
+ }
+
+ if (filter_ids.empty()) {
+ return;
+ }
+
+ result.count = filter_ids.size();
+ result.docs = new uint32_t[filter_ids.size()];
+ std::copy(filter_ids.begin(), filter_ids.end(), result.docs);
+ return;
+ }
+
+ if (!is_filter_result_initialized) {
+ compute_result();
+ }
+
+ std::vector<uint32_t> match_indexes;
+ for (uint32_t i = 0; i < lenA; i++) {
+ auto _result = valid(A[i]);
+
+ if (_result == -1) {
+ break;
+ }
+
+ if (_result == 1) {
+ match_indexes.push_back(result_index);
+ }
+ }
+
+ result.count = match_indexes.size();
+ result.docs = new uint32_t[match_indexes.size()];
+ for (auto const& item: filter_result.reference_filter_results) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];
+ }
+
+ for (uint32_t i = 0; i < match_indexes.size(); i++) {
+ auto const& match_index = match_indexes[i];
+ result.docs[i] = filter_result.docs[match_index];
+ for (auto const& item: filter_result.reference_filter_results) {
+ result.reference_filter_results[item.first][i] = item.second[match_index];
+ }
+ }
+}
+
filter_result_iterator_t::filter_result_iterator_t(const std::string collection_name, const Index *const index,
const filter_node_t *const filter_node) :
collection_name(collection_name),
diff --git a/src/index.cpp b/src/index.cpp
index 58061789..f215138d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2392,7 +2392,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
(filter_id_count >= vector_query.flat_search_cutoff && filter_result_iterator->is_valid)) {
dist_results.clear();
- // TODO: Need to pass reference info
VectorFilterFunctor filterFunctor(filter_result_iterator);
std::vector<std::pair<float, size_t>> pairs;
@@ -2403,18 +2402,29 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
} else {
pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);
}
+ filter_result_iterator->reset();
- for (const auto &pair: pairs) {
- auto filter_result = single_filter_result_t(static_cast<uint32_t>(pair.second), {});
- dist_results.emplace_back(pair.first, filter_result);
+ if (filter_result_iterator->is_valid && !filter_result_iterator->reference.empty()) {
+ // We'll have to get the references of each document.
+ for (auto pair: pairs) {
+ // The doc_id must be valid otherwise it would've been filtered out upstream.
+ filter_result_iterator->skip_to(pair.second);
+ auto filter_result = single_filter_result_t(pair.second,
+ std::move(filter_result_iterator->reference));
+ dist_results.emplace_back(pair.first, filter_result);
+ filter_result_iterator->reset();
+ }
+ } else {
+ for (const auto &pair: pairs) {
+ auto filter_result = single_filter_result_t(pair.second, {});
+ dist_results.emplace_back(pair.first, filter_result);
+ }
}
}
- filter_result_iterator->reset();
-
std::vector<uint32_t> nearest_ids;
- for (const auto& dist_result : dist_results) {
+ for (auto& dist_result : dist_results) {
auto& seq_id = dist_result.second.seq_id;
if(vector_query.query_doc_given && vector_query.seq_id == seq_id) {
@@ -2442,9 +2452,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices,
seq_id, 0, 0, scores, match_score_index, vec_dist_score);
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores,
- dist_result.second.reference_filter_results);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
kv.vector_distance = vec_dist_score;
+ kv.reference_filter_results = std::move(dist_result.second.reference_filter_results);
int ret = topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -2671,7 +2681,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
VectorFilterFunctor filterFunctor(filter_result_iterator);
auto& field_vector_index = vector_index.at(vector_query.field_name);
- // TODO: Need to pass reference info
std::vector<std::pair<float, size_t>> dist_labels;
// use k as 100 by default for ensuring results stability in pagination
size_t default_k = 100;
@@ -2756,6 +2765,15 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);
KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);
kv.vector_distance = vec_result.second;
+
+ if (filter_result_iterator->is_valid &&
+ !filter_result_iterator->reference.empty()) {
+ // The doc_id must be valid otherwise it would've been filtered out upstream.
+ filter_result_iterator->skip_to(doc_id);
+ kv.reference_filter_results = std::move(filter_result_iterator->reference);
+ filter_result_iterator->reset();
+ }
+
topster->add(&kv);
vec_search_ids.push_back(doc_id);
}
@@ -4202,17 +4220,17 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {
std::vector<uint32_t> infix_ids;
+ filter_result_t filtered_infix_ids;
search_infix(query_tokens[0].value, field_name, infix_ids, max_extra_prefix, max_extra_suffix);
if(!infix_ids.empty()) {
gfx::timsort(infix_ids.begin(), infix_ids.end());
infix_ids.erase(std::unique( infix_ids.begin(), infix_ids.end() ), infix_ids.end());
- // TODO: Need to pass reference info
- uint32_t *raw_infix_ids = nullptr;
- size_t raw_infix_ids_length = 0;
+ auto& raw_infix_ids = filtered_infix_ids.docs;
+ auto& raw_infix_ids_length = filtered_infix_ids.count;
- if(curated_ids_sorted.size() != 0) {
+ if(!curated_ids_sorted.empty()) {
raw_infix_ids_length = ArrayUtils::exclude_scalar(&infix_ids[0], infix_ids.size(), &curated_ids_sorted[0],
curated_ids_sorted.size(), &raw_infix_ids);
infix_ids.clear();
@@ -4222,15 +4240,13 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
}
if(filter_result_iterator->is_valid) {
- uint32_t *filtered_raw_infix_ids = nullptr;
-
- raw_infix_ids_length = filter_result_iterator->and_scalar(raw_infix_ids, raw_infix_ids_length,
- filtered_raw_infix_ids);
+ filter_result_t result;
+ filter_result_iterator->and_scalar(raw_infix_ids, raw_infix_ids_length, result);
if(raw_infix_ids != &infix_ids[0]) {
delete [] raw_infix_ids;
}
- raw_infix_ids = filtered_raw_infix_ids;
+ filtered_infix_ids = std::move(result);
}
bool field_is_array = search_schema.at(the_fields[field_id].name).is_array();
@@ -4258,6 +4274,9 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
}
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
+ for (const auto& item: filtered_infix_ids.reference_filter_results) {
+ kv.reference_filter_results[item.first] = item.second[i];
+ }
int ret = actual_topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -4276,8 +4295,8 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
delete[] all_result_ids;
all_result_ids = new_all_result_ids;
- if(raw_infix_ids != &infix_ids[0]) {
- delete [] raw_infix_ids;
+ if (raw_infix_ids == &infix_ids[0]) {
+ raw_infix_ids = nullptr;
}
searched_queries.push_back({});
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index f334bb3f..1d4302ea 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -1025,7 +1025,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
"name": "Products",
"fields": [
{"name": "product_id", "type": "string"},
- {"name": "product_name", "type": "string"},
+ {"name": "product_name", "type": "string", "infix": true},
{"name": "product_description", "type": "string"},
{"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}
]
@@ -1260,6 +1260,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ // Exclude token search
req_params = {
{"collection", "Products"},
{"q", "-shampoo"},
@@ -1278,6 +1279,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ // Phrase search
req_params = {
{"collection", "Products"},
{"q", R"("soap")"},
@@ -1296,6 +1298,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ // Combining normal and reference filter
req_params = {
{"collection", "Products"},
{"q", "*"},
@@ -1316,6 +1319,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price"));
+ // Multiple references
req_params = {
{"collection", "Products"},
{"q", "soap"},
@@ -1340,6 +1344,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price").at(0));
ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price").at(1));
+ // Vector search
req_params = {
{"collection", "Products"},
{"q", "natural products"},
@@ -1357,5 +1362,71 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
- ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price").at(0));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+
+ // Hybrid search - Both text match and vector match
+ req_params = {
+ {"collection", "Products"},
+ {"q", "soap"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
+ {"include_fields", "product_name, $Customers(product_price)"},
+ {"exclude_fields", ""}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("text_match_score"));
+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("vector_distance"));
+
+ // Hybrid search - Only vector match
+ req_params = {
+ {"collection", "Products"},
+ {"q", "natural products"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
+ {"include_fields", "product_name, $Customers(product_price)"},
+ {"exclude_fields", ""}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ(0, res_obj["hits"][0]["hybrid_search_info"].at("text_match_score"));
+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("vector_distance"));
+
+ // Infix search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "ap"},
+ {"query_by", "product_name"},
+ {"infix", "always"},
+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
+ {"include_fields", "product_name, $Customers(product_price)"},
+ {"exclude_fields", ""}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
}