Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-filter

Kishore Nalland5048f689b97
diff --cc include/index.h
index 38c02238,3f93e182..8db2945f
--- a/include/index.h
+++ b/include/index.h
@@@ -929,16 -948,16 +933,16 @@@ public
size_t filter_index,
int64_t max_field_match_score,
int64_t* scores,
- int64_t& match_score_index) const;
+ int64_t& match_score_index, float vector_distance = 0) const;
- void
- process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
- const size_t group_limit, const bool filter_curated_hits, const uint32_t* filter_ids,
- uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,
- std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- std::vector<uint32_t>& included_ids_vec,
- std::unordered_set<uint32_t>& excluded_group_ids) const;
+ void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
+ const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
+ const size_t group_limit, const bool filter_curated_hits,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint32_t>& curated_ids,
+ std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ std::vector<uint32_t>& included_ids_vec,
+ std::unordered_set<uint32_t>& excluded_group_ids) const;
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
diff --cc src/index.cpp
index d2fbf3f5,5cfb2287..f7cd7235
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -2753,10 -3140,12 +2753,12 @@@ Option<bool> Index::search(std::vector<
constexpr float TEXT_MATCH_WEIGHT = 0.7;
constexpr float VECTOR_SEARCH_WEIGHT = 1.0 - TEXT_MATCH_WEIGHT;
- VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
+ VectorFilterFunctor filterFunctor(filter_result_iterator);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
- auto k = std::max<size_t>(vector_query.k, fetch_size);
+ // use k as 100 by default for ensuring results stability in pagination
+ size_t default_k = 100;
+ auto k = std::max<size_t>(vector_query.k, default_k);
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
@@@ -4679,15 -5118,13 +4720,14 @@@ void Index::populate_sort_mapping(int*
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
-
- filter_result_t result;
- recursive_filter(sort_fields_std[i].eval.filter_tree_root, result);
- sort_fields_std[i].eval.ids = result.docs;
- sort_fields_std[i].eval.size = result.count;
- result.docs = nullptr;
+ auto filter_result_iterator = filter_result_iterator_t("", this, sort_fields_std[i].eval.filter_tree_root);
+ auto filter_init_op = filter_result_iterator.init_status();
+ if (!filter_init_op.ok()) {
+ return;
+ }
-
+ sort_fields_std[i].eval.size = filter_result_iterator.to_filter_id_array(sort_fields_std[i].eval.ids);
-
+ } else if(sort_fields_std[i].name == sort_field_const::vector_distance) {
+ field_values[i] = &vector_distance_sentinel_value;
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);