Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallane6876fa14794
diff --cc include/collection.hindex 3b2bbdd2,eb53ae5e..044c2438--- a/include/collection.h+++ b/include/collection.h@@@ -217,14 -209,11 +217,16 @@@ privateOption<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,std::vector<sort_by>& sort_fields_std,- bool is_wildcard_query,const bool is_vector_query,- bool is_group_by_query = false) const;+ bool is_wildcard_query, const bool is_vector_query,+ const std::string& query, bool is_group_by_query = false,+ const size_t remote_embedding_timeout_ms = 30000,+ const size_t remote_embedding_num_tries = 2) const;+ Option<bool> validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,+ std::vector<sort_by>& sort_fields_std,+ bool is_wildcard_query,const bool is_vector_query,+ bool is_group_by_query = false) const;+Option<bool> persist_collection_meta();@@@ -593,36 -566,11 +595,34 @@@ publicOption<bool> alter(nlohmann::json& alter_payload);- void- process_search_field_weights(const std::vector<std::string>& raw_search_fields,- std::vector<uint32_t>& query_by_weights,- std::vector<search_field_t>& weighted_search_fields,- std::vector<std::string>& reordered_search_fields) const;+ void process_search_field_weights(const std::vector<search_field_t>& search_fields,+ std::vector<uint32_t>& query_by_weights,+ std::vector<search_field_t>& weighted_search_fields) const;Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);++ void reference_populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,+ std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;++ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;++ bool is_referenced_in(const std::string& collection_name) const;++ void add_referenced_in(const reference_pair& pair);++ void add_referenced_ins(const std::set<reference_pair>& pairs);++ void add_referenced_in(const std::string& collection_name, const std::string& field_name);++ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;++ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;++ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;++ friend class filter_result_iterator_t;};template<class T>diff --cc include/field.hindex a2612f53,b6f245eb..8cf4e773--- a/include/field.h+++ b/include/field.h@@@ -8,9 -9,9 +8,10 @@@#include "store.h"#include <sparsepp.h>#include <tsl/htrie_map.h>+#include <filter.h>#include "json.hpp"#include "text_embedder_manager.h"+ #include "vector_query_ops.h"namespace field_types {// first field value indexed will determine the type@@@ -492,13 -662,16 +493,21 @@@ namespace sort_field_conststatic const std::string missing_values = "missing_values";static const std::string vector_distance = "_vector_distance";+ static const std::string vector_query = "_vector_query";}+struct ref_include_fields {+ std::string expression;+ std::string alias;+};++ struct hnsw_index_t;++ struct sort_vector_query_t {+ vector_query_t query;+ hnsw_index_t* vector_index;+ };+struct sort_by {enum missing_values_t {first,@@@ -528,47 -699,22 +537,46 @@@missing_values_t missing_values;eval_t eval;+ std::string reference_collection_name;+ sort_vector_query_t vector_query;sort_by(const std::string & name, const std::string & order):name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),missing_values(normal) {-}+ sort_by(std::vector<std::string> eval_expressions, std::vector<int64_t> scores, std::string order):+ eval_expressions(std::move(eval_expressions)), order(std::move(order)), text_match_buckets(0), geopoint(0), exclude_radius(0),+ geo_precision(0), missing_values(normal) {+ name = sort_field_const::eval;+ eval.scores = std::move(scores);+ }+sort_by(const std::string &name, const std::string &order, uint32_t text_match_buckets, int64_t geopoint,uint32_t exclude_radius, uint32_t geo_precision) :name(name), order(order), text_match_buckets(text_match_buckets),geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision),missing_values(normal) {-}+ sort_by(const sort_by& other) {+ if (&other == this)+ return;+ name = other.name;+ eval_expressions = other.eval_expressions;+ order = other.order;+ text_match_buckets = other.text_match_buckets;+ geopoint = other.geopoint;+ exclude_radius = other.exclude_radius;+ geo_precision = other.geo_precision;+ missing_values = other.missing_values;+ eval = other.eval;+ reference_collection_name = other.reference_collection_name;+ }+sort_by& operator=(const sort_by& other) {name = other.name;+ eval_expressions = other.eval_expressions;order = other.order;text_match_buckets = other.text_match_buckets;geopoint = other.geopoint;diff --cc src/collection.cppindex 112e322c,23050d4d..34532af0--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -19,7 -17,7 +19,8 @@@#include "thread_local_vars.h"#include "vector_query_ops.h"#include "text_embedder_manager.h"+#include "stopwords_manager.h"+ #include "field.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@@ -843,11 -743,13 +844,14 @@@ void Collection::curate_results(stringOption<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,std::vector<sort_by>& sort_fields_std,- const bool is_wildcard_query,+ const bool is_wildcard_query,const bool is_vector_query,- const bool is_group_by_query) const {+ const std::string& query, const bool is_group_by_query,+ const size_t remote_embedding_timeout_ms,+ const size_t remote_embedding_num_tries) const {+ uint32_t eval_sort_count = 0;+ size_t num_sort_expressions = 0;for(size_t i = 0; i < sort_fields.size(); i++) {const sort_by& _sort_field = sort_fields[i];@@@ -947,6 -780,93 +951,93 @@@sort_field_std.name = actual_field_name;sort_field_std.text_match_buckets = std::stoll(match_parts[1]);+ } else if(actual_field_name == sort_field_const::eval) {+ const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,+ sort_field_std.name.size() - paran_start -+ 2);+ if(filter_exp.empty()) {+ return Option<bool>(400, "The eval expression in sort_by is empty.");+ }++ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,- store, "", sort_field_std.eval.filter_tree_root);++ store, "", sort_field_std.eval.filter_trees);+ if(!parse_filter_op.ok()) {+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");+ }++ sort_field_std.name = actual_field_name;+ num_sort_expressions++;+ } else if(actual_field_name == sort_field_const::vector_query) {+ const std::string& vector_query_str = sort_field_std.name.substr(paran_start + 1,+ sort_field_std.name.size() - paran_start -+ 2);+ if(vector_query_str.empty()) {+ return Option<bool>(400, "The vector query in sort_by is empty.");+ }+++ auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query.query,+ is_wildcard_query, this, true);+ if(!parse_vector_op.ok()) {+ return Option<bool>(400, parse_vector_op.error());+ }++ auto vector_field_it = search_schema.find(sort_field_std.vector_query.query.field_name);+ if(vector_field_it == search_schema.end() || vector_field_it.value().num_dim == 0) {+ return Option<bool>(400, "Could not find a field named `" + sort_field_std.vector_query.query.field_name + "` in vector index.");+ }++ if(sort_field_std.vector_query.query.values.empty() && embedding_fields.find(sort_field_std.vector_query.query.field_name) != embedding_fields.end()) {+ // generate embeddings for the query++ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();+ auto embedder_op = embedder_manager.get_text_embedder(vector_field_it.value().embed[fields::model_config]);+ if(!embedder_op.ok()) {+ return Option<bool>(embedder_op.code(), embedder_op.error());+ }++ auto embedder = embedder_op.get();++ if(embedder->is_remote() && remote_embedding_num_tries == 0) {+ std::string error = "`remote_embedding_num_tries` must be greater than 0.";+ return Option<bool>(400, error);+ }++ std::string embed_query = embedder_manager.get_query_prefix(vector_field_it.value().embed[fields::model_config]) + query;+ auto embedding_op = embedder->Embed(embed_query, remote_embedding_timeout_ms, remote_embedding_num_tries);++ if(!embedding_op.success) {+ if(!embedding_op.error["error"].get<std::string>().empty()) {+ return Option<bool>(400, embedding_op.error["error"].get<std::string>());+ } else {+ return Option<bool>(400, embedding_op.error.dump());+ }+ }++ sort_field_std.vector_query.query.values = embedding_op.embedding;+ }++ const auto& vector_index_map = index->_get_vector_index();+ if(vector_index_map.find(sort_field_std.vector_query.query.field_name) == vector_index_map.end()) {+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector index.");+ }+++ if(vector_field_it.value().num_dim != sort_field_std.vector_query.query.values.size()) {+ return Option<bool>(400, "Query field `" + sort_field_std.vector_query.query.field_name + "` must have " ++ std::to_string(vector_field_it.value().num_dim) + " dimensions.");+ }++ sort_field_std.vector_query.vector_index = vector_index_map.at(sort_field_std.vector_query.query.field_name);++ if(sort_field_std.vector_query.vector_index->distance_type == cosine) {+ std::vector<float> normalized_values(sort_field_std.vector_query.query.values.size());+ hnsw_index_t::normalize_vector(sort_field_std.vector_query.query.values, normalized_values);+ sort_field_std.vector_query.query.values = normalized_values;+ }++ sort_field_std.name = actual_field_name;+} else {if(field_it == search_schema.end()) {std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";@@@ -1071,8 -991,9 +1162,8 @@@}if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&- sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {-+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance &&+ sort_field_std.name != sort_field_const::vector_query) {-const auto field_it = search_schema.find(sort_field_std.name);if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {std::string error = "Could not find a field named `" + sort_field_std.name +@@@ -1923,8 -1630,8 +2026,9 @@@ Option<nlohmann::json> Collection::searbool is_vector_query = !vector_query.field_name.empty();if(curated_sort_by.empty()) {- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,- sort_fields_std, is_wildcard_query, is_vector_query, raw_query, is_group_by_query, remote_embedding_timeout_ms, remote_embedding_num_tries);+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,- sort_fields_std, is_wildcard_query, is_vector_query, is_group_by_query);++ sort_fields_std, is_wildcard_query, is_vector_query,++ raw_query, is_group_by_query, remote_embedding_timeout_ms, remote_embedding_num_tries);if(!sort_validation_op.ok()) {return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());}diff --cc src/index.cppindex 04987fc7,aa55920b..f15b289a--- a/src/index.cpp+++ b/src/index.cpp@@@ -50,7 -43,18 +50,8 @@@ spp::sparse_hash_map<uint32_t, int64_tspp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::vector_distance_sentinel_value;+ spp::sparse_hash_map<uint32_t, int64_t> Index::vector_query_sentinel_value;-struct token_posting_t {- uint32_t token_id;- const posting_list_t::iterator_t& posting;-- token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):- token_id(token_id), posting(posting) {-- }-};-Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,SynonymIndex* synonym_index, ThreadPool* thread_pool,const tsl::htrie_map<char, field> & search_schema,@@@ -3149,23 -3398,18 +3150,23 @@@ Option<bool> Index::search(std::vector<num_queued++;thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,- batch_result_ids, batch_res_len, &facet_infos,- estimate_facets, facet_sample_percent, group_missing_values,+ batch_result_ids, batch_res_len, &facet_infos, max_facet_values,+ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,+ group_missing_values,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,- &num_processed, &m_process, &cv_process]() {+ &num_processed, &m_process, &cv_process, facet_index_type]() {search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;- search_cutoff = parent_search_cutoff;+ search_cutoff = false;auto fq = facet_query;+ std::vector<std::pair<std::string, uint32_t>> found_docs;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, group_missing_values,- batch_result_ids, batch_res_len);+ batch_result_ids, batch_res_len, max_facet_values,+ is_wildcard_query, no_filters_provided,+ facet_index_type);+std::unique_lock<std::mutex> lock(m_process);num_processed++;parent_search_cutoff = parent_search_cutoff || search_cutoff;@@@ -4331,34 -4398,36 +4330,46 @@@ Option<bool> Index::compute_sort_scores}}} else if(field_values[0] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[0].eval.size) {- size_t found_index = std::lower_bound(sort_fields[0].eval.ids + filter_index,- sort_fields[0].eval.ids + sort_fields[0].eval.size, seq_id) -- sort_fields[0].eval.ids;+ auto const& count = sort_fields[0].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[0].eval.size && sort_fields[0].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[0].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[0] = int64_t(found);+ scores[0] = found ? eval.scores[index] : 0;} else if(field_values[0] == &vector_distance_sentinel_value) {scores[0] = float_to_int64_t(vector_distance);+ } else if(field_values[0] == &vector_query_sentinel_value) {+ scores[0] = float_to_int64_t(2.0f);+ try {+ const auto& values = sort_fields[0].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[0].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[0].vector_query.query.values.data(), values.data(), &sort_fields[0].vector_query.vector_index->num_dim);++ scores[0] = float_to_int64_t(dist);+ } catch(...) {+ // probably not found+ // do nothing+ }} else {auto it = field_values[0]->find(seq_id);scores[0] = (it == field_values[0]->end()) ? default_score : it->second;@@@ -4494,34 -4468,37 +4505,47 @@@}}} else if(field_values[1] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[1].eval.size) {- size_t found_index = std::lower_bound(sort_fields[1].eval.ids + filter_index,- sort_fields[1].eval.ids + sort_fields[1].eval.size, seq_id) -- sort_fields[1].eval.ids;+ auto const& count = sort_fields[1].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[1].eval.size && sort_fields[1].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[1].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[1] = int64_t(found);+ scores[1] = found ? eval.scores[index] : 0;} else if(field_values[1] == &vector_distance_sentinel_value) {scores[1] = float_to_int64_t(vector_distance);+ } else if(field_values[1] == &vector_query_sentinel_value) {+ scores[1] = float_to_int64_t(2.0f);+ try {+ const auto& values = sort_fields[1].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[1].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[1].vector_query.query.values.data(), values.data(), &sort_fields[1].vector_query.vector_index->num_dim);++ scores[1] = float_to_int64_t(dist);+ } catch(...) {+ // probably not found+ // do nothing+ }+} else {auto it = field_values[1]->find(seq_id);scores[1] = (it == field_values[1]->end()) ? default_score : it->second;@@@ -4653,34 -4535,36 +4677,46 @@@}}} else if(field_values[2] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[2].eval.size) {- size_t found_index = std::lower_bound(sort_fields[2].eval.ids + filter_index,- sort_fields[2].eval.ids + sort_fields[2].eval.size, seq_id) -- sort_fields[2].eval.ids;+ auto const& count = sort_fields[2].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[2].eval.size && sort_fields[2].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[2].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[2] = int64_t(found);+ scores[2] = found ? eval.scores[index] : 0;} else if(field_values[2] == &vector_distance_sentinel_value) {scores[2] = float_to_int64_t(vector_distance);+ } else if(field_values[2] == &vector_query_sentinel_value) {+ scores[2] = float_to_int64_t(2.0f);+ try {+ const auto& values = sort_fields[2].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[2].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[2].vector_query.query.values.data(), values.data(), &sort_fields[2].vector_query.vector_index->num_dim);++ scores[2] = float_to_int64_t(dist);+ } catch(...) {+ // probably not found+ // do nothing+ }} else {auto it = field_values[2]->find(seq_id);scores[2] = (it == field_values[2]->end()) ? default_score : it->second;@@@ -5381,22 -5200,12 +5416,22 @@@ Option<bool> Index::search_wildcard(filsearch_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;- search_cutoff = parent_search_cutoff;+ search_cutoff = false;- size_t filter_index = 0;+ std::vector<uint32_t> filter_indexes;++ std::vector<group_by_field_it_t> group_by_field_it_vec;+ if (group_limit != 0) {+ group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);+ }++ for(size_t i = 0; i < batch_result->count; i++) {+ const uint32_t seq_id = batch_result->docs[i];+ std::map<basic_string<char>, reference_filter_result_t> references;+ if (batch_result->coll_to_references != nullptr) {+ references = std::move(batch_result->coll_to_references[i]);+ }- for(size_t i = 0; i < batch_res_len; i++) {- const uint32_t seq_id = batch_result_ids[i];int64_t match_score = 0;score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,@@@ -5521,22 -5288,15 +5556,24 @@@ void Index::populate_sort_mapping(int*field_values[i] = &seq_id_sentinel_value;} else if (sort_fields_std[i].name == sort_field_const::eval) {field_values[i] = &eval_sentinel_value;- filter_result_t result;- recursive_filter(sort_fields_std[i].eval.filter_tree_root, result);- sort_fields_std[i].eval.ids = result.docs;- sort_fields_std[i].eval.size = result.count;- result.docs = nullptr;+ auto& eval_exp = sort_fields_std[i].eval;+ auto count = sort_fields_std[i].eval_expressions.size();+ for (uint32_t j = 0; j < count; j++) {+ auto filter_result_iterator = filter_result_iterator_t("", this, &eval_exp.filter_trees[j]);+ auto filter_init_op = filter_result_iterator.init_status();+ if (!filter_init_op.ok()) {+ return;+ }+ uint32_t* eval_ids = nullptr;+ auto eval_ids_count = filter_result_iterator.to_filter_id_array(eval_ids);++ eval_exp.eval_ids_vec.push_back(eval_ids);+ eval_exp.eval_ids_count_vec.push_back(eval_ids_count);+ }} else if(sort_fields_std[i].name == sort_field_const::vector_distance) {field_values[i] = &vector_distance_sentinel_value;+ } else if(sort_fields_std[i].name == sort_field_const::vector_query) {+ field_values[i] = &vector_query_sentinel_value;} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {geopoint_indices.push_back(i);diff --cc test/collection_sorting_test.cppindex 2465a481,8d51a079..366f2f93--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@@ -2469,4 -2387,84 +2469,83 @@@ TEST_F(CollectionSortingTest, InvalidVeASSERT_FALSE(results.ok());ASSERT_EQ("sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.", results.error());- }+ }+--TEST_F(CollectionSortingTest, TestSortByVectorQuery) {++TEST_F(CollectionSortingTest, DISABLED_TestSortByVectorQuery) {+ std::string coll_schema = R"(+ {+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string" },+ {"name": "points", "type": "float[]", "num_dim": 2}+ ]+ }+ )";++ nlohmann::json schema = nlohmann::json::parse(coll_schema);+ auto create_coll = collectionManager.create_collection(schema);+ ASSERT_TRUE(create_coll.ok());++ auto coll = create_coll.get();++ std::vector<std::vector<float>> points = {+ {7.0, 8.0},+ {8.0, 15.0},+ {5.0, 12.0},+ };++ for(size_t i = 0; i < points.size(); i++) {+ nlohmann::json doc;+ doc["name"] = "Title " + std::to_string(i);+ doc["points"] = points[i];+ ASSERT_TRUE(coll->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = {};++ auto results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "").get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"]);+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);+ ASSERT_EQ("0", results["hits"][2]["document"]["id"]);++ sort_fields = {+ sort_by("_vector_query(points:([5.0, 5.0]))", "asc"),+ };++ results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "").get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"]);+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);+ ASSERT_EQ("2", results["hits"][2]["document"]["id"]);++ sort_fields = {+ sort_by("_vector_query(points:([5.0, 5.0]))", "desc"),+ };++ results = coll->search("title", {"name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "").get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"]);+ ASSERT_EQ("1", results["hits"][1]["document"]["id"]);+ ASSERT_EQ("0", results["hits"][2]["document"]["id"]);-}++}diff --cc test/collection_vector_search_test.cppindex bfdb2f78,04cd0096..70e0a15b--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -2666,3 -2649,84 +2666,84 @@@ TEST_F(CollectionVectorTest, TestHybrid"`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());}++ TEST_F(CollectionVectorTest, TestSearchNonIndexedEmbeddingField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "index": false,+ "optional": true,+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto search_res = coll->search("soccer", {"name", "embedding"}, "", {}, {}, {0});+ ASSERT_FALSE(search_res.ok());++ ASSERT_EQ("Field `embedding` is marked as a non-indexed field in the schema.", search_res.error());+ }++ TEST_F(CollectionVectorTest, TestSearchNonIndexedVectorField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "vec",+ "type": "float[]",+ "index": false,+ "optional": true,+ "num_dim": 2+ }+ ]+ })"_json;++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "vec": [0.1, 0.2]+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto search_result = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "vec:([0.96826, 0.94])");++ ASSERT_FALSE(search_result.ok());+ ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());-}++}