Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Include reference fields without join.
Harpreet Sangar710ee61c873f
diff --git a/include/collection.h b/include/collection.hindex 692e4142..391a4bac 100644--- a/include/collection.h+++ b/include/collection.h@@ -276,10 +276,10 @@ private:const spp::sparse_hash_set<std::string>& exclude_fields,tsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;-+ Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;- Option<std::string> get_reference_field(const std::string & collection_name) const;+ Option<std::string> get_reference_field(const std::string& ref_collection_name) const;static void hide_credential(nlohmann::json& json, const std::string& credential_name);@@ -377,8 +377,10 @@ public:static void remove_flat_fields(nlohmann::json& document);static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,- const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0,- const std::map<std::string, reference_filter_result_t>& reference_filter_results = {});+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",+ size_t depth = 0,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},+ Collection *const collection = nullptr, const uint32_t& seq_id = 0);const Index* _get_index() const;diff --git a/include/index.h b/include/index.hindex e9d20cda..bc961ec2 100644--- a/include/index.h+++ b/include/index.h@@ -482,24 +482,8 @@ private:const size_t num_search_fields,std::vector<size_t>& popular_field_ids);- void numeric_not_equals_filter(num_tree_t* const num_tree,- const int64_t value,- const uint32_t& context_ids_length,- uint32_t* const& context_ids,- size_t& ids_len,- uint32_t*& ids) const;-bool field_is_indexed(const std::string& field_name) const;- void aproximate_numerical_match(num_tree_t* const num_tree,- const NUM_COMPARATOR& comparator,- const int64_t& value,- const int64_t& range_end_value,- uint32_t& filter_ids_length) const;-- void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,- const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;-static void tokenize_string(const std::string& text,const field& a_field,const std::vector<char>& symbols_to_index,@@ -941,6 +925,9 @@ public:Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,std::vector<uint32_t>& outside_seq_ids);+ Option<uint32_t> get_reference_doc_id_with_lock(const std::string& reference_helper_field_name,+ const uint32_t& seq_id) const;+friend class filter_result_iterator_t;};diff --git a/src/collection.cpp b/src/collection.cppindex 99bfacf0..a0f6e341 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1945,7 +1945,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,exclude_fields_full,"",0,- field_order_kv->reference_filter_results);+ field_order_kv->reference_filter_results,+ const_cast<Collection *>(this), get_seq_id_from_key(seq_id_key));if (!prune_op.ok()) {return Option<nlohmann::json>(prune_op.code(), prune_op.error());}@@ -2697,11 +2698,21 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_return index->do_filtering_with_lock(filter_tree_root, filter_result, name);}-Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {+Option<uint32_t> Collection::get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const {+ auto get_reference_field_op = get_reference_field(ref_collection_name);+ if (!get_reference_field_op.ok()) {+ return Option<uint32_t>(get_reference_field_op.code(), get_reference_field_op.error());+ }++ auto field_name = get_reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;+ return index->get_reference_doc_id_with_lock(field_name, seq_id);+}++Option<std::string> Collection::get_reference_field(const std::string& ref_collection_name) const {std::string reference_field_name;for (auto const& pair: reference_fields) {auto reference_pair = pair.second;- if (reference_pair.collection == collection_name) {+ if (reference_pair.collection == ref_collection_name) {reference_field_name = pair.first;break;}@@ -2709,7 +2720,7 @@ Option<std::string> Collection::get_reference_field(const std::string & collectiif (reference_field_name.empty()) {return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"- + collection_name + "`.");+ + ref_collection_name + "`.");}return Option(reference_field_name);@@ -4044,7 +4055,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,const std::string& parent_name, size_t depth,- const std::map<std::string, reference_filter_result_t>& reference_filter_results) {+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ Collection *const collection, const uint32_t& seq_id) {// doc can only be an objectauto it = doc.begin();while(it != doc.end()) {@@ -4120,12 +4132,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,it++;}- if (reference_filter_results.empty()) {- return Option<bool>(true);- }-- auto include_reference_it = include_names.equal_prefix_range("$");- for (auto reference = include_reference_it.first; reference != include_reference_it.second; reference++) {+ auto include_reference_it_pair = include_names.equal_prefix_range("$");+ for (auto reference = include_reference_it_pair.first; reference != include_reference_it_pair.second; reference++) {auto ref = reference.key();size_t parenthesis_index = ref.find('(');@@ -4162,9 +4170,36 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());}- if (reference_filter_results.count(ref_collection_name) == 0 ||- reference_filter_results.at(ref_collection_name).count == 0) {- // doc has no references.+ bool has_filter_reference = reference_filter_results.count(ref_collection_name) > 0;+ if (!has_filter_reference) {+ if (collection == nullptr) {+ continue;+ }++ // Reference include_by without join, check if doc itself contains the reference.+ auto get_reference_doc_id_op = collection->get_reference_doc_id(ref_collection_name, seq_id);+ if (!get_reference_doc_id_op.ok()) {+ continue;+ }++ auto ref_doc_seq_id = get_reference_doc_id_op.get();++ nlohmann::json ref_doc;+ auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);+ if (!get_doc_op.ok()) {+ return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());+ }++ auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);+ if (!prune_op.ok()) {+ return Option<bool>(prune_op.code(), error_prefix + prune_op.error());+ }++ doc.update(ref_doc);+ continue;+ }++ if (has_filter_reference && reference_filter_results.at(ref_collection_name).count == 0) {continue;}diff --git a/src/index.cpp b/src/index.cppindex de07a27a..c61a2ff1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1656,34 +1656,6 @@ bool Index::field_is_indexed(const std::string& field_name) const {geo_range_index.count(field_name) != 0;}-void Index::aproximate_numerical_match(num_tree_t* const num_tree,- const NUM_COMPARATOR& comparator,- const int64_t& value,- const int64_t& range_end_value,- uint32_t& filter_ids_length) const {- if (comparator == RANGE_INCLUSIVE) {- num_tree->approx_range_inclusive_search_count(value, range_end_value, filter_ids_length);- return;- }-- if (comparator == NOT_EQUALS) {- uint32_t to_exclude_ids_len = 0;- num_tree->approx_search_count(EQUALS, value, to_exclude_ids_len);-- if (to_exclude_ids_len == 0) {- filter_ids_length += seq_ids->num_ids();- } else if (to_exclude_ids_len >= seq_ids->num_ids()) {- filter_ids_length += 0;- } else {- filter_ids_length += (seq_ids->num_ids() - to_exclude_ids_len);- }-- return;- }-- num_tree->approx_search_count(comparator, value, filter_ids_length);-}-Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,const std::string& collection_name) const {@@ -6150,6 +6122,17 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,}}+Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_helper_field_name,+ const uint32_t& seq_id) const {+ std::shared_lock lock(mutex);+ if (sort_index.count(reference_helper_field_name) == 0 ||+ sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {+ return Option<uint32_t>(400, "Could not find a reference for doc " + std::to_string(seq_id));+ }++ return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));+}+/*// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex a93c0d08..13d8b2a6 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -1427,6 +1427,26 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));++ // Reference include_by without join+ req_params = {+ {"collection", "Customers"},+ {"q", "Joe"},+ {"query_by", "customer_name"},+ {"filter_by", "product_price:<100"},+ {"include_fields", "$Products(product_name), product_price"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));}TEST_F(CollectionJoinTest, CascadeDeletion) {