Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor vector index lookup & normalization
ozanarmagandfbb1ebfb43c
diff --git a/include/field.h b/include/field.hindex 0b5d98c8..b6f245eb 100644--- a/include/field.h+++ b/include/field.h@@ -665,6 +665,13 @@ namespace sort_field_const {static const std::string vector_query = "_vector_query";}+struct hnsw_index_t;++struct sort_vector_query_t {+ vector_query_t query;+ hnsw_index_t* vector_index;+};+struct sort_by {enum missing_values_t {first,@@ -692,7 +699,7 @@ struct sort_by {missing_values_t missing_values;eval_t eval;- vector_query_t vector_query;+ sort_vector_query_t vector_query;sort_by(const std::string & name, const std::string & order):name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),diff --git a/src/collection.cpp b/src/collection.cppindex a7d993f9..997a47a6 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -17,6 +17,7 @@#include "thread_local_vars.h"#include "vector_query_ops.h"#include "text_embedder_manager.h"+#include "field.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@ -804,20 +805,20 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<}- auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query,+ auto parse_vector_op = VectorQueryOps::parse_vector_query_str(vector_query_str, sort_field_std.vector_query.query,is_wildcard_query, this, true);if(!parse_vector_op.ok()) {return Option<bool>(400, parse_vector_op.error());}- auto vector_field_it = search_schema.find(sort_field_std.vector_query.field_name);+ auto vector_field_it = search_schema.find(sort_field_std.vector_query.query.field_name);if(vector_field_it == search_schema.end() || vector_field_it.value().num_dim == 0) {- return Option<bool>(400, "Field `" + sort_field_std.vector_query.field_name + "` does not have a vector query index.");+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector query index.");}--- if(sort_field_std.vector_query.values.empty() && embedding_fields.find(sort_field_std.vector_query.field_name) != embedding_fields.end()) {++ if(sort_field_std.vector_query.query.values.empty() && embedding_fields.find(sort_field_std.vector_query.query.field_name) != embedding_fields.end()) {// generate embeddings for the query+TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();auto embedder_op = embedder_manager.get_text_embedder(vector_field_it.value().embed[fields::model_config]);if(!embedder_op.ok()) {@@ -842,14 +843,28 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<}}- sort_field_std.vector_query.values = embedding_op.embedding;+ sort_field_std.vector_query.query.values = embedding_op.embedding;+ }++ const auto& vector_index_map = index->_get_vector_index();+ if(vector_index_map.find(sort_field_std.vector_query.query.field_name) == vector_index_map.end()) {+ return Option<bool>(400, "Field `" + sort_field_std.vector_query.query.field_name + "` does not have a vector index.");}- if(vector_field_it.value().num_dim != sort_field_std.vector_query.values.size()) {- return Option<bool>(400, "Query field `" + sort_field_std.vector_query.field_name + "` must have " +++ if(vector_field_it.value().num_dim != sort_field_std.vector_query.query.values.size()) {+ return Option<bool>(400, "Query field `" + sort_field_std.vector_query.query.field_name + "` must have " +std::to_string(vector_field_it.value().num_dim) + " dimensions.");}+ sort_field_std.vector_query.vector_index = vector_index_map.at(sort_field_std.vector_query.query.field_name);++ if(sort_field_std.vector_query.vector_index->distance_type == cosine) {+ std::vector<float> normalized_values(sort_field_std.vector_query.query.values.size());+ hnsw_index_t::normalize_vector(sort_field_std.vector_query.query.values, normalized_values);+ sort_field_std.vector_query.query.values = normalized_values;+ }+sort_field_std.name = actual_field_name;} else {diff --git a/src/index.cpp b/src/index.cppindex c7a9c8a7..65ae04ee 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4399,25 +4399,15 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else if(field_values[0] == &vector_query_sentinel_value) {scores[0] = float_to_int64_t(2.0f);try {- auto& field_vector_index = vector_index.at(sort_fields[0].vector_query.field_name);- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);- const auto& dist_func = field_vector_index->space->get_dist_func();- float dist = 2.0f;- if(field_vector_index->distance_type == cosine) {- std::vector<float> normalized_values(sort_fields[0].vector_query.values.size());- hnsw_index_t::normalize_vector(sort_fields[0].vector_query.values, normalized_values);- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);-- } else {- dist = dist_func(sort_fields[0].vector_query.values.data(), values.data(), &field_vector_index->num_dim);- }+ const auto& values = sort_fields[0].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[0].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[0].vector_query.query.values.data(), values.data(), &sort_fields[0].vector_query.vector_index->num_dim);scores[0] = float_to_int64_t(dist);} catch(...) {// probably not found// do nothing}-} else {auto it = field_values[0]->find(seq_id);scores[0] = (it == field_values[0]->end()) ? default_score : it->second;@@ -4479,18 +4469,9 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else if(field_values[1] == &vector_query_sentinel_value) {scores[1] = float_to_int64_t(2.0f);try {- auto& field_vector_index = vector_index.at(sort_fields[1].vector_query.field_name);- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);- const auto& dist_func = field_vector_index->space->get_dist_func();- float dist = 2.0f;- if(field_vector_index->distance_type == cosine) {- std::vector<float> normalized_values(sort_fields[1].vector_query.values.size());- hnsw_index_t::normalize_vector(sort_fields[1].vector_query.values, normalized_values);- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);-- } else {- dist = dist_func(sort_fields[1].vector_query.values.data(), values.data(), &field_vector_index->num_dim);- }+ const auto& values = sort_fields[1].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[1].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[1].vector_query.query.values.data(), values.data(), &sort_fields[1].vector_query.vector_index->num_dim);scores[1] = float_to_int64_t(dist);} catch(...) {@@ -4555,25 +4536,15 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else if(field_values[2] == &vector_query_sentinel_value) {scores[2] = float_to_int64_t(2.0f);try {- auto& field_vector_index = vector_index.at(sort_fields[2].vector_query.field_name);- const auto& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);- const auto& dist_func = field_vector_index->space->get_dist_func();- float dist = 2.0f;- if(field_vector_index->distance_type == cosine) {- std::vector<float> normalized_values(sort_fields[2].vector_query.values.size());- hnsw_index_t::normalize_vector(sort_fields[2].vector_query.values, normalized_values);- dist = dist_func(normalized_values.data(), values.data(), &field_vector_index->num_dim);-- } else {- dist = dist_func(sort_fields[2].vector_query.values.data(), values.data(), &field_vector_index->num_dim);- }+ const auto& values = sort_fields[2].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);+ const auto& dist_func = sort_fields[2].vector_query.vector_index->space->get_dist_func();+ float dist = dist_func(sort_fields[2].vector_query.query.values.data(), values.data(), &sort_fields[2].vector_query.vector_index->num_dim);scores[2] = float_to_int64_t(dist);} catch(...) {// probably not found// do nothing}-} else {auto it = field_values[2]->find(seq_id);scores[2] = (it == field_values[2]->end()) ? default_score : it->second;