Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Flat search cutoff for vector search.
Kishore Nallan4808d1610a74
diff --git a/include/field.h b/include/field.hindex bff58b4b..ec9156c9 100644--- a/include/field.h+++ b/include/field.h@@ -604,6 +604,7 @@ struct sort_by {struct vector_query_t {std::string field_name;size_t k = 0;+ size_t flat_search_cutoff = 1000;std::vector<float> values;void _reset() {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex acc05db5..57972b20 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1454,6 +1454,14 @@ bool CollectionManager::parse_vector_query_str(std::string vector_query_str, vecvector_query.k = std::stoul(param_kv[1]);}++ if(param_kv[0] == "flat_search_cutoff") {+ if(!StringUtils::is_uint32_t(param_kv[1])) {+ return false;+ }++ vector_query.flat_search_cutoff = std::stoi(param_kv[1]);+ }}return true;diff --git a/src/index.cpp b/src/index.cppindex 7fd85292..eba9f9f1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2458,8 +2458,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v// for phrase query, parser will set field_query_tokens to "*", need to handle thatif (is_wildcard_query) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);- bool no_filters_provided =- (filter_tree_root != nullptr && filter_ids_length == 0);+ bool no_filters_provided = (filter_tree_root == nullptr && filter_ids_length == 0);if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&@@ -2513,13 +2512,32 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vstd::vector<std::pair<float, size_t>> dist_labels;+ if(!no_filters_provided && filter_ids_length < vector_query.flat_search_cutoff) {+ for(size_t i = 0; i < filter_ids_length; i++) {+ auto seq_id = filter_ids[i];+ const std::vector<float>& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);+ float dist;++ if(field_vector_index->distance_type == cosine) {+ std::vector<float> normalized_q(vector_query.values.size());+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);+ dist = field_vector_index->space->get_dist_func()(normalized_q.data(), values.data(),+ &field_vector_index->num_dim);+ } else {+ dist = field_vector_index->space->get_dist_func()(vector_query.values.data(), values.data(),+ &field_vector_index->num_dim);+ }- if(field_vector_index->distance_type == cosine) {- std::vector<float> normalized_q(vector_query.values.size());- hnsw_index_t::normalize_vector(vector_query.values, normalized_q);- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);+ dist_labels.emplace_back(dist, seq_id);+ }} else {- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);+ if(field_vector_index->distance_type == cosine) {+ std::vector<float> normalized_q(vector_query.values.size());+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);+ } else {+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);+ }}std::vector<uint32_t> nearest_ids;@@ -2547,11 +2565,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v}if(!nearest_ids.empty()) {- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, nearest_ids.data(),- nearest_ids.size(), &new_all_result_ids);- delete [] all_result_ids;- all_result_ids = new_all_result_ids;+ all_result_ids = new uint32[nearest_ids.size()];+ std::copy(nearest_ids.begin(), nearest_ids.end(), all_result_ids);+ all_result_ids_len = nearest_ids.size();}} else {search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,@@ -4244,14 +4260,6 @@ void Index::curate_filtered_ids(filter_node_t const* const& filter_tree_root, coconst uint32_t* exclude_token_ids, size_t exclude_token_ids_size,uint32_t*& filter_ids, uint32_t& filter_ids_length,const std::vector<uint32_t>& curated_ids_sorted) const {-- // if filtered results are not available, use the seq_ids index to generate- // the list of all document ids- if (filter_tree_root == nullptr && filter_ids_length == 0) {- filter_ids_length = seq_ids->num_ids();- filter_ids = seq_ids->uncompress();- }-if(!curated_ids.empty()) {uint32_t *excluded_result_ids = nullptr;filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 32ce7c36..d4ad9ac3 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -86,7 +86,22 @@ TEST_F(CollectionVectorTest, BasicVectorQuerying) {"", 10, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ // with filtering + flat search+ results = coll1->search("*", {}, "points:[0,1]", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(2, results["hits"].size());@@ -250,7 +265,7 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {ASSERT_EQ(num_docs, results["found"].get<size_t>());ASSERT_EQ(num_docs, results["hits"].size());- // with points:<10+ // with points:<10, non-flat-searchresults = coll1->search("*", {}, "points:<10", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),@@ -259,7 +274,20 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,fallback,4, {off}, 32767, 32767, 2,- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();++ ASSERT_EQ(10, results["found"].get<size_t>());+ ASSERT_EQ(10, results["hits"].size());++ // with points:<10, flat-search+ results = coll1->search("*", {}, "points:<10", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();ASSERT_EQ(10, results["found"].get<size_t>());ASSERT_EQ(10, results["hits"].size());@@ -273,7 +301,19 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,fallback,4, {off}, 32767, 32767, 2,- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ results = coll1->search("*", {}, "points:1", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());