Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor vector index lookup & normalization

ozanarmagandfbb1ebfb43c
diff --git a/include/field.h b/include/field.h
index 0b5d98c8..b6f245eb 100644
--- a/include/field.h
+++ b/include/field.h
@@ -665,6 +665,13 @@ namespace sort_field_const {
static const std::string vector_query = "_vector_query";
}
+struct hnsw_index_t;
+
+struct sort_vector_query_t {
+ vector_query_t query;
+ hnsw_index_t* vector_index;
+};
+
struct sort_by {
enum missing_values_t {
first,
@@ -692,7 +699,7 @@ struct sort_by {
missing_values_t missing_values;
eval_t eval;
- vector_query_t vector_query;
+ sort_vector_query_t vector_query;
sort_by(const std::string & name, const std::string & order):
name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
diff --git a/src/collection.cpp b/src/collection.cpp
index a7d993f9..997a47a6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -17,6 +17,7 @@
#include "thread_local_vars.h"
#include "vector_query_ops.h"
#include "text_embedder_manager.h"
+#include "field.h"
const std::string override_t::MATCH_EXACT = "exact";
const std::string override_t::MATCH_CONTAINS = "contains";
@@ -804,20 +805,20 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
}
- auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query,
+ auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query.query,
is_wildcard_query, this, true);
if(!parse_vector_op.ok()) {
return Option<bool>(400, parse_vector_op.error());
}
- auto vector_field_it = search_schema.find(sort_field_std.vector_query.field_name);
+ auto vector_field_it = search_schema.find(sort_field_std.vector_query.query.field_name);
if(vector_field_it == search_schema.end() || vector_field_it.value().num_dim == 0) {
- return Option<bool>(400, "Field `" + sort_field_std.vector_query.field_name + "` does not have a vector query index.");
+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector query index.");
}
-
-
- if(sort_field_std.vector_query.values.empty() && embedding_fields.find(sort_field_std.vector_query.field_name) != embedding_fields.end()) {
+
+ if(sort_field_std.vector_query.query.values.empty() && embedding_fields.find(sort_field_std.vector_query.query.field_name) != embedding_fields.end()) {
// generate embeddings for the query
+
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
auto embedder_op = embedder_manager.get_text_embedder(vector_field_it.value().embed[fields::model_config]);
if(!embedder_op.ok()) {
@@ -842,14 +843,28 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
}
}
- sort_field_std.vector_query.values = embedding_op.embedding;
+ sort_field_std.vector_query.query.values = embedding_op.embedding;
+ }
+
+ const auto& vector_index_map = index->_get_vector_index();
+ if(vector_index_map.find(sort_field_std.vector_query.query.field_name) == vector_index_map.end()) {
+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector index.");
}
- if(vector_field_it.value().num_dim != sort_field_std.vector_query.values.size()) {
- return Option<bool>(400, "Query field `" + sort_field_std.vector_query.field_name + "` must have " +
+
+ if(vector_field_it.value().num_dim != sort_field_std.vector_query.query.values.size()) {
+ return Option<bool>(400, "Query field `" + sort_field_std.vector_query.query.field_name + "` must have " +
std::to_string(vector_field_it.value().num_dim) + " dimensions.");
}
+ sort_field_std.vector_query.vector_index = vector_index_map.at(sort_field_std.vector_query.query.field_name);
+
+ if(sort_field_std.vector_query.vector_index->distance_type == cosine) {
+ std::vector<float> normalized_values(sort_field_std.vector_query.query.values.size());
+ hnsw_index_t::normalize_vector(sort_field_std.vector_query.query.values, normalized_values);
+ sort_field_std.vector_query.query.values = normalized_values;
+ }
+
sort_field_std.name = actual_field_name;
} else {
diff --git a/src/index.cpp b/src/index.cpp
index c7a9c8a7..65ae04ee 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -4399,25 +4399,15 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else if(field_values[0] == &vector_query_sentinel_value) {
scores[0] = float_to_int64_t(2.0f);
try {
- auto& field_vector_index = vector_index.at(sort_fields[0].vector_query.field_name);
- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
- const auto& dist_func = field_vector_index->space->get_dist_func();
- float dist = 2.0f;
- if(field_vector_index->distance_type == cosine) {
- std::vector<float> normalized_values(sort_fields[0].vector_query.values.size());
- hnsw_index_t::normalize_vector(sort_fields[0].vector_query.values, normalized_values);
- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);
-
- } else {
- dist = dist_func(sort_fields[0].vector_query.values.data(), values.data(), &field_vector_index->num_dim);
- }
+ const auto& values = sort_fields[0].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[0].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[0].vector_query.query.values.data(), values.data(), &sort_fields[0].vector_query.vector_index->num_dim);
scores[0] = float_to_int64_t(dist);
} catch(...) {
// probably not found
// do nothing
}
-
} else {
auto it = field_values[0]->find(seq_id);
scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
@@ -4479,18 +4469,9 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else if(field_values[1] == &vector_query_sentinel_value) {
scores[1] = float_to_int64_t(2.0f);
try {
- auto& field_vector_index = vector_index.at(sort_fields[1].vector_query.field_name);
- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
- const auto& dist_func = field_vector_index->space->get_dist_func();
- float dist = 2.0f;
- if(field_vector_index->distance_type == cosine) {
- std::vector<float> normalized_values(sort_fields[1].vector_query.values.size());
- hnsw_index_t::normalize_vector(sort_fields[1].vector_query.values, normalized_values);
- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);
-
- } else {
- dist = dist_func(sort_fields[1].vector_query.values.data(), values.data(), &field_vector_index->num_dim);
- }
+ const auto& values = sort_fields[1].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[1].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[1].vector_query.query.values.data(), values.data(), &sort_fields[1].vector_query.vector_index->num_dim);
scores[1] = float_to_int64_t(dist);
} catch(...) {
@@ -4555,25 +4536,15 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else if(field_values[2] == &vector_query_sentinel_value) {
scores[2] = float_to_int64_t(2.0f);
try {
- auto& field_vector_index = vector_index.at(sort_fields[2].vector_query.field_name);
- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
- const auto& dist_func = field_vector_index->space->get_dist_func();
- float dist = 2.0f;
- if(field_vector_index->distance_type == cosine) {
- std::vector<float> normalized_values(sort_fields[2].vector_query.values.size());
- hnsw_index_t::normalize_vector(sort_fields[2].vector_query.values, normalized_values);
- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);
-
- } else {
- dist = dist_func(sort_fields[2].vector_query.values.data(), values.data(), &field_vector_index->num_dim);
- }
+ const auto& values = sort_fields[2].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);
+ const auto& dist_func = sort_fields[2].vector_query.vector_index->space->get_dist_func();
+ float dist = dist_func(sort_fields[2].vector_query.query.values.data(), values.data(), &sort_fields[2].vector_query.vector_index->num_dim);
scores[2] = float_to_int64_t(dist);
} catch(...) {
// probably not found
// do nothing
}
-
} else {
auto it = field_values[2]->find(seq_id);
scores[2] = (it == field_values[2]->end()) ? default_score : it->second;