Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Flat search cutoff for vector search.

Kishore Nallan4808d1610a74
diff --git a/include/field.h b/include/field.h
index bff58b4b..ec9156c9 100644
--- a/include/field.h
+++ b/include/field.h
@@ -604,6 +604,7 @@ struct sort_by {
struct vector_query_t {
std::string field_name;
size_t k = 0;
+ size_t flat_search_cutoff = 1000;
std::vector<float> values;
void _reset() {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index acc05db5..57972b20 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1454,6 +1454,14 @@ bool CollectionManager::parse_vector_query_str(std::string vector_query_str, vec
vector_query.k = std::stoul(param_kv[1]);
}
+
+ if(param_kv[0] == "flat_search_cutoff") {
+ if(!StringUtils::is_uint32_t(param_kv[1])) {
+ return false;
+ }
+
+ vector_query.flat_search_cutoff = std::stoi(param_kv[1]);
+ }
}
return true;
diff --git a/src/index.cpp b/src/index.cpp
index 7fd85292..eba9f9f1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2458,8 +2458,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
// for phrase query, parser will set field_query_tokens to "*", need to handle that
if (is_wildcard_query) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- bool no_filters_provided =
- (filter_tree_root != nullptr && filter_ids_length == 0);
+ bool no_filters_provided = (filter_tree_root == nullptr && filter_ids_length == 0);
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
@@ -2513,13 +2512,32 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::vector<std::pair<float, size_t>> dist_labels;
+ if(!no_filters_provided && filter_ids_length < vector_query.flat_search_cutoff) {
+ for(size_t i = 0; i < filter_ids_length; i++) {
+ auto seq_id = filter_ids[i];
+ const std::vector<float>& values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
+ float dist;
+
+ if(field_vector_index->distance_type == cosine) {
+ std::vector<float> normalized_q(vector_query.values.size());
+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
+ dist = field_vector_index->space->get_dist_func()(normalized_q.data(), values.data(),
+ &field_vector_index->num_dim);
+ } else {
+ dist = field_vector_index->space->get_dist_func()(vector_query.values.data(), values.data(),
+ &field_vector_index->num_dim);
+ }
- if(field_vector_index->distance_type == cosine) {
- std::vector<float> normalized_q(vector_query.values.size());
- hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);
+ dist_labels.emplace_back(dist, seq_id);
+ }
} else {
- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);
+ if(field_vector_index->distance_type == cosine) {
+ std::vector<float> normalized_q(vector_query.values.size());
+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);
+ } else {
+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);
+ }
}
std::vector<uint32_t> nearest_ids;
@@ -2547,11 +2565,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
}
if(!nearest_ids.empty()) {
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, nearest_ids.data(),
- nearest_ids.size(), &new_all_result_ids);
- delete [] all_result_ids;
- all_result_ids = new_all_result_ids;
+ all_result_ids = new uint32[nearest_ids.size()];
+ std::copy(nearest_ids.begin(), nearest_ids.end(), all_result_ids);
+ all_result_ids_len = nearest_ids.size();
}
} else {
search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
@@ -4244,14 +4260,6 @@ void Index::curate_filtered_ids(filter_node_t const* const& filter_tree_root, co
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
uint32_t*& filter_ids, uint32_t& filter_ids_length,
const std::vector<uint32_t>& curated_ids_sorted) const {
-
- // if filtered results are not available, use the seq_ids index to generate
- // the list of all document ids
- if (filter_tree_root == nullptr && filter_ids_length == 0) {
- filter_ids_length = seq_ids->num_ids();
- filter_ids = seq_ids->uncompress();
- }
-
if(!curated_ids.empty()) {
uint32_t *excluded_result_ids = nullptr;
filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 32ce7c36..d4ad9ac3 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -86,7 +86,22 @@ TEST_F(CollectionVectorTest, BasicVectorQuerying) {
"", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ // with filtering + flat search
+ results = coll1->search("*", {}, "points:[0,1]", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
@@ -250,7 +265,7 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {
ASSERT_EQ(num_docs, results["found"].get<size_t>());
ASSERT_EQ(num_docs, results["hits"].size());
- // with points:<10
+ // with points:<10, non-flat-search
results = coll1->search("*", {}, "points:<10", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
@@ -259,7 +274,20 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
fallback,
4, {off}, 32767, 32767, 2,
- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();
+
+ ASSERT_EQ(10, results["found"].get<size_t>());
+ ASSERT_EQ(10, results["hits"].size());
+
+ // with points:<10, flat-search
+ results = coll1->search("*", {}, "points:<10", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(10, results["hits"].size());
@@ -273,7 +301,19 @@ TEST_F(CollectionVectorTest, VecSearchWithFiltering) {
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
fallback,
4, {off}, 32767, 32767, 2,
- false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488])").get();
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 0)").get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ results = coll1->search("*", {}, "points:1", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.96826, 0.94, 0.39557, 0.306488], flat_search_cutoff: 1000)").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());