Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle references with vector, hybrid and infix search.
Harpreet Sangar6a2b9549d5e4
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex b705b122..f0d26c82 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -37,8 +37,21 @@ struct single_filter_result_t {single_filter_result_t() = default;- single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t> reference_filter_results) :+ single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results) :seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)) {}++ single_filter_result_t(const single_filter_result_t& obj) {+ if (&obj == this)+ return;++ seq_id = obj.seq_id;++ // Copy every collection's reference.+ for (const auto &item: obj.reference_filter_results) {+ auto& ref_coll_name = item.first;+ reference_filter_results[ref_coll_name] = item.second;+ }+ }};struct filter_result_t {@@ -232,6 +245,8 @@ public:/// \return size of the results arrayuint32_t and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results);+ void and_scalar(const uint32_t* A, const uint32_t& lenA, filter_result_t& result);+static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);};diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex dcc2dbdf..08933c70 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1411,6 +1411,72 @@ uint32_t filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t&return filter_ids.size();}+void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenA, filter_result_t& result) {+ if (!is_valid) {+ return;+ }++ if (filter_result.reference_filter_results.empty()) {+ if (is_filter_result_initialized) {+ result.count = ArrayUtils::and_scalar(A, lenA, filter_result.docs, filter_result.count, &result.docs);+ return;+ }++ std::vector<uint32_t> filter_ids;+ for (uint32_t i = 0; i < lenA; i++) {+ auto _result = valid(A[i]);++ if (_result == -1) {+ break;+ }++ if (_result == 1) {+ filter_ids.push_back(A[i]);+ }+ }++ if (filter_ids.empty()) {+ return;+ }++ result.count = filter_ids.size();+ result.docs = new uint32_t[filter_ids.size()];+ std::copy(filter_ids.begin(), filter_ids.end(), result.docs);+ return;+ }++ if (!is_filter_result_initialized) {+ compute_result();+ }++ std::vector<uint32_t> match_indexes;+ for (uint32_t i = 0; i < lenA; i++) {+ auto _result = valid(A[i]);++ if (_result == -1) {+ break;+ }++ if (_result == 1) {+ match_indexes.push_back(result_index);+ }+ }++ result.count = match_indexes.size();+ result.docs = new uint32_t[match_indexes.size()];+ for (auto const& item: filter_result.reference_filter_results) {+ result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];+ }++ for (uint32_t i = 0; i < match_indexes.size(); i++) {+ auto const& match_index = match_indexes[i];+ result.docs[i] = filter_result.docs[match_index];+ for (auto const& item: filter_result.reference_filter_results) {+ result.reference_filter_results[item.first][i] = item.second[match_index];+ }+ }+}+filter_result_iterator_t::filter_result_iterator_t(const std::string collection_name, const Index *const index,const filter_node_t *const filter_node) :collection_name(collection_name),diff --git a/src/index.cpp b/src/index.cppindex 58061789..f215138d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2392,7 +2392,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons(filter_id_count >= vector_query.flat_search_cutoff && filter_result_iterator->is_valid)) {dist_results.clear();- // TODO: Need to pass reference infoVectorFilterFunctor filterFunctor(filter_result_iterator);std::vector<std::pair<float, size_t>> pairs;@@ -2403,18 +2402,29 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons} else {pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);}+ filter_result_iterator->reset();- for (const auto &pair: pairs) {- auto filter_result = single_filter_result_t(static_cast<uint32_t>(pair.second), {});- dist_results.emplace_back(pair.first, filter_result);+ if (filter_result_iterator->is_valid && !filter_result_iterator->reference.empty()) {+ // We'll have to get the references of each document.+ for (auto pair: pairs) {+ // The doc_id must be valid otherwise it would've been filtered out upstream.+ filter_result_iterator->skip_to(pair.second);+ auto filter_result = single_filter_result_t(pair.second,+ std::move(filter_result_iterator->reference));+ dist_results.emplace_back(pair.first, filter_result);+ filter_result_iterator->reset();+ }+ } else {+ for (const auto &pair: pairs) {+ auto filter_result = single_filter_result_t(pair.second, {});+ dist_results.emplace_back(pair.first, filter_result);+ }}}- filter_result_iterator->reset();-std::vector<uint32_t> nearest_ids;- for (const auto& dist_result : dist_results) {+ for (auto& dist_result : dist_results) {auto& seq_id = dist_result.second.seq_id;if(vector_query.query_doc_given && vector_query.seq_id == seq_id) {@@ -2442,9 +2452,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscompute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices,seq_id, 0, 0, scores, match_score_index, vec_dist_score);- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores,- dist_result.second.reference_filter_results);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);kv.vector_distance = vec_dist_score;+ kv.reference_filter_results = std::move(dist_result.second.reference_filter_results);int ret = topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -2671,7 +2681,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consVectorFilterFunctor filterFunctor(filter_result_iterator);auto& field_vector_index = vector_index.at(vector_query.field_name);- // TODO: Need to pass reference infostd::vector<std::pair<float, size_t>> dist_labels;// use k as 100 by default for ensuring results stability in paginationsize_t default_k = 100;@@ -2756,6 +2765,15 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscompute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);kv.vector_distance = vec_result.second;++ if (filter_result_iterator->is_valid &&+ !filter_result_iterator->reference.empty()) {+ // The doc_id must be valid otherwise it would've been filtered out upstream.+ filter_result_iterator->skip_to(doc_id);+ kv.reference_filter_results = std::move(filter_result_iterator->reference);+ filter_result_iterator->reset();+ }+topster->add(&kv);vec_search_ids.push_back(doc_id);}@@ -4202,17 +4220,17 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seif(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {std::vector<uint32_t> infix_ids;+ filter_result_t filtered_infix_ids;search_infix(query_tokens[0].value, field_name, infix_ids, max_extra_prefix, max_extra_suffix);if(!infix_ids.empty()) {gfx::timsort(infix_ids.begin(), infix_ids.end());infix_ids.erase(std::unique( infix_ids.begin(), infix_ids.end() ), infix_ids.end());- // TODO: Need to pass reference info- uint32_t *raw_infix_ids = nullptr;- size_t raw_infix_ids_length = 0;+ auto& raw_infix_ids = filtered_infix_ids.docs;+ auto& raw_infix_ids_length = filtered_infix_ids.count;- if(curated_ids_sorted.size() != 0) {+ if(!curated_ids_sorted.empty()) {raw_infix_ids_length = ArrayUtils::exclude_scalar(&infix_ids[0], infix_ids.size(), &curated_ids_sorted[0],curated_ids_sorted.size(), &raw_infix_ids);infix_ids.clear();@@ -4222,15 +4240,13 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se}if(filter_result_iterator->is_valid) {- uint32_t *filtered_raw_infix_ids = nullptr;-- raw_infix_ids_length = filter_result_iterator->and_scalar(raw_infix_ids, raw_infix_ids_length,- filtered_raw_infix_ids);+ filter_result_t result;+ filter_result_iterator->and_scalar(raw_infix_ids, raw_infix_ids_length, result);if(raw_infix_ids != &infix_ids[0]) {delete [] raw_infix_ids;}- raw_infix_ids = filtered_raw_infix_ids;+ filtered_infix_ids = std::move(result);}bool field_is_array = search_schema.at(the_fields[field_id].name).is_array();@@ -4258,6 +4274,9 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se}KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);+ for (const auto& item: filtered_infix_ids.reference_filter_results) {+ kv.reference_filter_results[item.first] = item.second[i];+ }int ret = actual_topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -4276,8 +4295,8 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<sedelete[] all_result_ids;all_result_ids = new_all_result_ids;- if(raw_infix_ids != &infix_ids[0]) {- delete [] raw_infix_ids;+ if (raw_infix_ids == &infix_ids[0]) {+ raw_infix_ids = nullptr;}searched_queries.push_back({});diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex f334bb3f..1d4302ea 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -1025,7 +1025,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {"name": "Products","fields": [{"name": "product_id", "type": "string"},- {"name": "product_name", "type": "string"},+ {"name": "product_name", "type": "string", "infix": true},{"name": "product_description", "type": "string"},{"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}]@@ -1260,6 +1260,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ // Exclude token searchreq_params = {{"collection", "Products"},{"q", "-shampoo"},@@ -1278,6 +1279,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));+ // Phrase searchreq_params = {{"collection", "Products"},{"q", R"("soap")"},@@ -1296,6 +1298,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));+ // Combining normal and reference filterreq_params = {{"collection", "Products"},{"q", "*"},@@ -1316,6 +1319,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price"));+ // Multiple referencesreq_params = {{"collection", "Products"},{"q", "soap"},@@ -1340,6 +1344,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price").at(0));ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price").at(1));+ // Vector searchreq_params = {{"collection", "Products"},{"q", "natural products"},@@ -1357,5 +1362,71 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(2, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));- ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price").at(0));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));++ // Hybrid search - Both text match and vector match+ req_params = {+ {"collection", "Products"},+ {"q", "soap"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ {"include_fields", "product_name, $Customers(product_price)"},+ {"exclude_fields", ""}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("text_match_score"));+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("vector_distance"));++ // Hybrid search - Only vector match+ req_params = {+ {"collection", "Products"},+ {"q", "natural products"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ {"include_fields", "product_name, $Customers(product_price)"},+ {"exclude_fields", ""}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ(0, res_obj["hits"][0]["hybrid_search_info"].at("text_match_score"));+ ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("vector_distance"));++ // Infix search+ req_params = {+ {"collection", "Products"},+ {"q", "ap"},+ {"query_by", "product_name"},+ {"infix", "always"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ {"include_fields", "product_name, $Customers(product_price)"},+ {"exclude_fields", ""}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));}