Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallane6876fa14794
diff --cc include/collection.h
index 3b2bbdd2,eb53ae5e..044c2438
--- a/include/collection.h
+++ b/include/collection.h
@@@ -217,14 -209,11 +217,16 @@@ private
Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
std::vector<sort_by>& sort_fields_std,
- bool is_wildcard_query,const bool is_vector_query,
- bool is_group_by_query = false) const;
+ bool is_wildcard_query, const bool is_vector_query,
+ const std::string& query, bool is_group_by_query = false,
+ const size_t remote_embedding_timeout_ms = 30000,
+ const size_t remote_embedding_num_tries = 2) const;
+ Option<bool> validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,
+ std::vector<sort_by>& sort_fields_std,
+ bool is_wildcard_query,const bool is_vector_query,
+ bool is_group_by_query = false) const;
+
Option<bool> persist_collection_meta();
@@@ -593,36 -566,11 +595,34 @@@ public
Option<bool> alter(nlohmann::json& alter_payload);
- void
- process_search_field_weights(const std::vector<std::string>& raw_search_fields,
- std::vector<uint32_t>& query_by_weights,
- std::vector<search_field_t>& weighted_search_fields,
- std::vector<std::string>& reordered_search_fields) const;
+ void process_search_field_weights(const std::vector<search_field_t>& search_fields,
+ std::vector<uint32_t>& query_by_weights,
+ std::vector<search_field_t>& weighted_search_fields) const;
Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);
+
+ void reference_populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,
+ std::vector<sort_by>& sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;
+
+ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;
+
+ bool is_referenced_in(const std::string& collection_name) const;
+
+ void add_referenced_in(const reference_pair& pair);
+
+ void add_referenced_ins(const std::set<reference_pair>& pairs);
+
+ void add_referenced_in(const std::string& collection_name, const std::string& field_name);
+
+ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;
+
+ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const;
+
+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;
+
+ friend class filter_result_iterator_t;
};
template<class T>
diff --cc include/field.h
index a2612f53,b6f245eb..8cf4e773
--- a/include/field.h
+++ b/include/field.h
@@@ -8,9 -9,9 +8,10 @@@
#include "store.h"
#include <sparsepp.h>
#include <tsl/htrie_map.h>
+#include <filter.h>
#include "json.hpp"
#include "text_embedder_manager.h"
+ #include "vector_query_ops.h"
namespace field_types {
// first field value indexed will determine the type
@@@ -492,13 -662,16 +493,21 @@@ namespace sort_field_const
static const std::string missing_values = "missing_values";
static const std::string vector_distance = "_vector_distance";
+ static const std::string vector_query = "_vector_query";
}
+struct ref_include_fields {
+ std::string expression;
+ std::string alias;
+};
+
+ struct hnsw_index_t;
+
+ struct sort_vector_query_t {
+ vector_query_t query;
+ hnsw_index_t* vector_index;
+ };
+
struct sort_by {
enum missing_values_t {
first,
@@@ -528,47 -699,22 +537,46 @@@
missing_values_t missing_values;
eval_t eval;
+ std::string reference_collection_name;
+ sort_vector_query_t vector_query;
sort_by(const std::string & name, const std::string & order):
name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
missing_values(normal) {
-
}
+ sort_by(std::vector<std::string> eval_expressions, std::vector<int64_t> scores, std::string order):
+ eval_expressions(std::move(eval_expressions)), order(std::move(order)), text_match_buckets(0), geopoint(0), exclude_radius(0),
+ geo_precision(0), missing_values(normal) {
+ name = sort_field_const::eval;
+ eval.scores = std::move(scores);
+ }
+
sort_by(const std::string &name, const std::string &order, uint32_t text_match_buckets, int64_t geopoint,
uint32_t exclude_radius, uint32_t geo_precision) :
name(name), order(order), text_match_buckets(text_match_buckets),
geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision),
missing_values(normal) {
-
}
+ sort_by(const sort_by& other) {
+ if (&other == this)
+ return;
+ name = other.name;
+ eval_expressions = other.eval_expressions;
+ order = other.order;
+ text_match_buckets = other.text_match_buckets;
+ geopoint = other.geopoint;
+ exclude_radius = other.exclude_radius;
+ geo_precision = other.geo_precision;
+ missing_values = other.missing_values;
+ eval = other.eval;
+ reference_collection_name = other.reference_collection_name;
+ }
+
sort_by& operator=(const sort_by& other) {
name = other.name;
+ eval_expressions = other.eval_expressions;
order = other.order;
text_match_buckets = other.text_match_buckets;
geopoint = other.geopoint;
diff --cc src/collection.cpp
index 112e322c,23050d4d..34532af0
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -19,7 -17,7 +19,8 @@@
#include "thread_local_vars.h"
#include "vector_query_ops.h"
#include "text_embedder_manager.h"
+#include "stopwords_manager.h"
+ #include "field.h"
const std::string override_t::MATCH_EXACT = "exact";
const std::string override_t::MATCH_CONTAINS = "contains";
@@@ -843,11 -743,13 +844,14 @@@ void Collection::curate_results(string
Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
std::vector<sort_by>& sort_fields_std,
- const bool is_wildcard_query,
+ const bool is_wildcard_query,
const bool is_vector_query,
- const bool is_group_by_query) const {
+ const std::string& query, const bool is_group_by_query,
+ const size_t remote_embedding_timeout_ms,
+ const size_t remote_embedding_num_tries) const {
+ uint32_t eval_sort_count = 0;
+ size_t num_sort_expressions = 0;
for(size_t i = 0; i < sort_fields.size(); i++) {
const sort_by& _sort_field = sort_fields[i];
@@@ -947,6 -780,93 +951,93 @@@
sort_field_std.name = actual_field_name;
sort_field_std.text_match_buckets = std::stoll(match_parts[1]);
+ } else if(actual_field_name == sort_field_const::eval) {
+ const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,
+ sort_field_std.name.size() - paran_start -
+ 2);
+ if(filter_exp.empty()) {
+ return Option<bool>(400, "The eval expression in sort_by is empty.");
+ }
+
+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
- store, "", sort_field_std.eval.filter_tree_root);
++ store, "", sort_field_std.eval.filter_trees);
+ if(!parse_filter_op.ok()) {
+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
+ }
+
+ sort_field_std.name = actual_field_name;
+ num_sort_expressions++;
+ } else if(actual_field_name == sort_field_const::vector_query) {
+ const std::string& vector_query_str = sort_field_std.name.substr(paran_start + 1,
+ sort_field_std.name.size() - paran_start -
+ 2);
+ if(vector_query_str.empty()) {
+ return Option<bool>(400, "The vector query in sort_by is empty.");
+ }
+
+
+ auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query.query,
+ is_wildcard_query, this, true);
+ if(!parse_vector_op.ok()) {
+ return Option<bool>(400, parse_vector_op.error());
+ }
+
+ auto vector_field_it = search_schema.find(sort_field_std.vector_query.query.field_name);
+ if(vector_field_it == search_schema.end() || vector_field_it.value().num_dim == 0) {
+ return Option<bool>(400, "Could not find a field named `" + sort_field_std.vector_query.query.field_name + "` in vector index.");
+ }
+
+ if(sort_field_std.vector_query.query.values.empty() && embedding_fields.find(sort_field_std.vector_query.query.field_name) != embedding_fields.end()) {
+ // generate embeddings for the query
+
+ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
+ auto embedder_op = embedder_manager.get_text_embedder(vector_field_it.value().embed[fields::model_config]);
+ if(!embedder_op.ok()) {
+ return Option<bool>(embedder_op.code(), embedder_op.error());
+ }
+
+ auto embedder = embedder_op.get();
+
+ if(embedder->is_remote() && remote_embedding_num_tries == 0) {
+ std::string error = "`remote_embedding_num_tries` must be greater than 0.";
+ return Option<bool>(400, error);
+ }
+
+ std::string embed_query = embedder_manager.get_query_prefix(vector_field_it.value().embed[fields::model_config]) + query;
+ auto embedding_op = embedder->Embed(embed_query, remote_embedding_timeout_ms, remote_embedding_num_tries);
+
+ if(!embedding_op.success) {
+ if(!embedding_op.error["error"].get<std::string>().empty()) {
+ return Option<bool>(400, embedding_op.error["error"].get<std::string>());
+ } else {
+ return Option<bool>(400, embedding_op.error.dump());
+ }
+ }
+
+ sort_field_std.vector_query.query.values = embedding_op.embedding;
+ }
+
+ const auto& vector_index_map = index->_get_vector_index();
+ if(vector_index_map.find(sort_field_std.vector_query.query.field_name) == vector_index_map.end()) {
+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector index.");
+ }
+
+
+ if(vector_field_it.value().num_dim != sort_field_std.vector_query.query.values.size()) {
+ return Option<bool>(400, "Query field `" + sort_field_std.vector_query.query.field_name + "` must have " +
+ std::to_string(vector_field_it.value().num_dim) + " dimensions.");
+ }
+
+ sort_field_std.vector_query.vector_index = vector_index_map.at(sort_field_std.vector_query.query.field_name);
+
+ if(sort_field_std.vector_query.vector_index->distance_type == cosine) {
+ std::vector<float> normalized_values(sort_field_std.vector_query.query.values.size());
+ hnsw_index_t::normalize_vector(sort_field_std.vector_query.query.values, normalized_values);
+ sort_field_std.vector_query.query.values = normalized_values;
+ }
+
+ sort_field_std.name = actual_field_name;
+
} else {
if(field_it == search_schema.end()) {
std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";
@@@ -1071,8 -991,9 +1162,8 @@@
}
if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&
- sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {
-
+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance &&
+ sort_field_std.name != sort_field_const::vector_query) {
-
const auto field_it = search_schema.find(sort_field_std.name);
if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {
std::string error = "Could not find a field named `" + sort_field_std.name +
@@@ -1923,8 -1630,8 +2026,9 @@@ Option<nlohmann::json> Collection::sear
bool is_vector_query = !vector_query.field_name.empty();
if(curated_sort_by.empty()) {
- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,
- sort_fields_std, is_wildcard_query, is_vector_query, raw_query, is_group_by_query, remote_embedding_timeout_ms, remote_embedding_num_tries);
+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,
- sort_fields_std, is_wildcard_query, is_vector_query, is_group_by_query);
++ sort_fields_std, is_wildcard_query, is_vector_query,
++ raw_query, is_group_by_query, remote_embedding_timeout_ms, remote_embedding_num_tries);
if(!sort_validation_op.ok()) {
return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());
}
diff --cc src/index.cpp
index 04987fc7,aa55920b..f15b289a
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -50,7 -43,18 +50,8 @@@ spp::sparse_hash_map<uint32_t, int64_t
spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::vector_distance_sentinel_value;
+ spp::sparse_hash_map<uint32_t, int64_t> Index::vector_query_sentinel_value;
-struct token_posting_t {
- uint32_t token_id;
- const posting_list_t::iterator_t& posting;
-
- token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):
- token_id(token_id), posting(posting) {
-
- }
-};
-
Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,
SynonymIndex* synonym_index, ThreadPool* thread_pool,
const tsl::htrie_map<char, field> & search_schema,
@@@ -3149,23 -3398,18 +3150,23 @@@ Option<bool> Index::search(std::vector<
num_queued++;
thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,
- batch_result_ids, batch_res_len, &facet_infos,
- estimate_facets, facet_sample_percent, group_missing_values,
+ batch_result_ids, batch_res_len, &facet_infos, max_facet_values,
+ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,
+ group_missing_values,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- &num_processed, &m_process, &cv_process]() {
+ &num_processed, &m_process, &cv_process, facet_index_type]() {
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
- search_cutoff = parent_search_cutoff;
+ search_cutoff = false;
auto fq = facet_query;
+ std::vector<std::pair<std::string, uint32_t>> found_docs;
do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, group_missing_values,
- batch_result_ids, batch_res_len);
+ batch_result_ids, batch_res_len, max_facet_values,
+ is_wildcard_query, no_filters_provided,
+ facet_index_type);
+
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
parent_search_cutoff = parent_search_cutoff || search_cutoff;
@@@ -4331,34 -4398,36 +4330,46 @@@ Option<bool> Index::compute_sort_scores
}
}
} else if(field_values[0] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[0].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[0].eval.ids + filter_index,
- sort_fields[0].eval.ids + sort_fields[0].eval.size, seq_id) -
- sort_fields[0].eval.ids;
+ auto const& count = sort_fields[0].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[0].eval.size && sort_fields[0].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[0].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[0] = int64_t(found);
+ scores[0] = found ? eval.scores[index] : 0;
} else if(field_values[0] == &vector_distance_sentinel_value) {
scores[0] = float_to_int64_t(vector_distance);
+ } else if(field_values[0] == &vector_query_sentinel_value) {
+ scores[0] = float_to_int64_t(2.0f);
+ try {
+ const auto& values = sort_fields[0].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[0].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[0].vector_query.query.values.data(), values.data(), &sort_fields[0].vector_query.vector_index->num_dim);
+
+ scores[0] = float_to_int64_t(dist);
+ } catch(...) {
+ // probably not found
+ // do nothing
+ }
} else {
auto it = field_values[0]->find(seq_id);
scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
@@@ -4494,34 -4468,37 +4505,47 @@@
}
}
} else if(field_values[1] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[1].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[1].eval.ids + filter_index,
- sort_fields[1].eval.ids + sort_fields[1].eval.size, seq_id) -
- sort_fields[1].eval.ids;
+ auto const& count = sort_fields[1].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[1].eval.size && sort_fields[1].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[1].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[1] = int64_t(found);
+ scores[1] = found ? eval.scores[index] : 0;
} else if(field_values[1] == &vector_distance_sentinel_value) {
scores[1] = float_to_int64_t(vector_distance);
+ } else if(field_values[1] == &vector_query_sentinel_value) {
+ scores[1] = float_to_int64_t(2.0f);
+ try {
+ const auto& values = sort_fields[1].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[1].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[1].vector_query.query.values.data(), values.data(), &sort_fields[1].vector_query.vector_index->num_dim);
+
+ scores[1] = float_to_int64_t(dist);
+ } catch(...) {
+ // probably not found
+ // do nothing
+ }
+
} else {
auto it = field_values[1]->find(seq_id);
scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
@@@ -4653,34 -4535,36 +4677,46 @@@
}
}
} else if(field_values[2] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[2].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[2].eval.ids + filter_index,
- sort_fields[2].eval.ids + sort_fields[2].eval.size, seq_id) -
- sort_fields[2].eval.ids;
+ auto const& count = sort_fields[2].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[2].eval.size && sort_fields[2].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[2].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[2] = int64_t(found);
+ scores[2] = found ? eval.scores[index] : 0;
} else if(field_values[2] == &vector_distance_sentinel_value) {
scores[2] = float_to_int64_t(vector_distance);
+ } else if(field_values[2] == &vector_query_sentinel_value) {
+ scores[2] = float_to_int64_t(2.0f);
+ try {
+ const auto& values = sort_fields[2].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[2].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[2].vector_query.query.values.data(), values.data(), &sort_fields[2].vector_query.vector_index->num_dim);
+
+ scores[2] = float_to_int64_t(dist);
+ } catch(...) {
+ // probably not found
+ // do nothing
+ }
} else {
auto it = field_values[2]->find(seq_id);
scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
@@@ -5381,22 -5200,12 +5416,22 @@@ Option<bool> Index::search_wildcard(fil
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
- search_cutoff = parent_search_cutoff;
+ search_cutoff = false;
- size_t filter_index = 0;
+ std::vector<uint32_t> filter_indexes;
+
+ std::vector<group_by_field_it_t> group_by_field_it_vec;
+ if (group_limit != 0) {
+ group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);
+ }
+
+ for(size_t i = 0; i < batch_result->count; i++) {
+ const uint32_t seq_id = batch_result->docs[i];
+ std::map<basic_string<char>, reference_filter_result_t> references;
+ if (batch_result->coll_to_references != nullptr) {
+ references = std::move(batch_result->coll_to_references[i]);
+ }
- for(size_t i = 0; i < batch_res_len; i++) {
- const uint32_t seq_id = batch_result_ids[i];
int64_t match_score = 0;
score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,
@@@ -5521,22 -5288,15 +5556,24 @@@ void Index::populate_sort_mapping(int*
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
- filter_result_t result;
- recursive_filter(sort_fields_std[i].eval.filter_tree_root, result);
- sort_fields_std[i].eval.ids = result.docs;
- sort_fields_std[i].eval.size = result.count;
- result.docs = nullptr;
+ auto& eval_exp = sort_fields_std[i].eval;
+ auto count = sort_fields_std[i].eval_expressions.size();
+ for (uint32_t j = 0; j < count; j++) {
+ auto filter_result_iterator = filter_result_iterator_t("", this, &eval_exp.filter_trees[j]);
+ auto filter_init_op = filter_result_iterator.init_status();
+ if (!filter_init_op.ok()) {
+ return;
+ }
+ uint32_t* eval_ids = nullptr;
+ auto eval_ids_count = filter_result_iterator.to_filter_id_array(eval_ids);
+
+ eval_exp.eval_ids_vec.push_back(eval_ids);
+ eval_exp.eval_ids_count_vec.push_back(eval_ids_count);
+ }
} else if(sort_fields_std[i].name == sort_field_const::vector_distance) {
field_values[i] = &vector_distance_sentinel_value;
+ } else if(sort_fields_std[i].name == sort_field_const::vector_query) {
+ field_values[i] = &vector_query_sentinel_value;
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);
diff --cc test/collection_sorting_test.cpp
index 2465a481,8d51a079..366f2f93
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@@ -2469,4 -2387,84 +2469,83 @@@ TEST_F(CollectionSortingTest, InvalidVe
ASSERT_FALSE(results.ok());
ASSERT_EQ("sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.", results.error());
- }
+ }
+
-
-TEST_F(CollectionSortingTest, TestSortByVectorQuery) {
++TEST_F(CollectionSortingTest, DISABLED_TestSortByVectorQuery) {
+ std::string coll_schema = R"(
+ {
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string" },
+ {"name": "points", "type": "float[]", "num_dim": 2}
+ ]
+ }
+ )";
+
+ nlohmann::json schema = nlohmann::json::parse(coll_schema);
+ auto create_coll = collectionManager.create_collection(schema);
+ ASSERT_TRUE(create_coll.ok());
+
+ auto coll = create_coll.get();
+
+ std::vector<std::vector<float>> points = {
+ {7.0, 8.0},
+ {8.0, 15.0},
+ {5.0, 12.0},
+ };
+
+ for(size_t i = 0; i < points.size(); i++) {
+ nlohmann::json doc;
+ doc["name"] = "Title " + std::to_string(i);
+ doc["points"] = points[i];
+ ASSERT_TRUE(coll->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = {};
+
+ auto results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "").get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"]);
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);
+ ASSERT_EQ("0", results["hits"][2]["document"]["id"]);
+
+ sort_fields = {
+ sort_by("_vector_query(points:([5.0, 5.0]))", "asc"),
+ };
+
+ results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "").get();
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"]);
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);
+ ASSERT_EQ("2", results["hits"][2]["document"]["id"]);
+
+ sort_fields = {
+ sort_by("_vector_query(points:([5.0, 5.0]))", "desc"),
+ };
+
+ results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "").get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"]);
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);
+ ASSERT_EQ("0", results["hits"][2]["document"]["id"]);
-}
++}
diff --cc test/collection_vector_search_test.cpp
index bfdb2f78,04cd0096..70e0a15b
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -2666,3 -2649,84 +2666,84 @@@ TEST_F(CollectionVectorTest, TestHybrid
"`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
}
+
+ TEST_F(CollectionVectorTest, TestSearchNonIndexedEmbeddingField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "index": false,
+ "optional": true,
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_res = coll->search("soccer", {"name", "embedding"}, "", {}, {}, {0});
+ ASSERT_FALSE(search_res.ok());
+
+ ASSERT_EQ("Field `embedding` is marked as a non-indexed field in the schema.", search_res.error());
+ }
+
+ TEST_F(CollectionVectorTest, TestSearchNonIndexedVectorField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "vec",
+ "type": "float[]",
+ "index": false,
+ "optional": true,
+ "num_dim": 2
+ }
+ ]
+ })"_json;
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "vec": [0.1, 0.2]
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_result = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.96826, 0.94])");
+
+ ASSERT_FALSE(search_result.ok());
+ ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());
-}
++}