Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Make HNSW params configurable (#1521)

Ozan Armağan18306e3dd268
diff --git a/WORKSPACE b/WORKSPACE
index d090cccf..3ed3bf77 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -186,7 +186,7 @@ new_git_repository(
new_git_repository(
name = "hnsw",
build_file = "//bazel:hnsw.BUILD",
- commit = "5100d3fe41da45601875b3f395f508398cb12b8a",
+ commit = "2fec56c161548ae8b2ab6bacfe404e4040729cd7",
remote = "https://github.com/typesense/hnswlib.git",
)
diff --git a/include/field.h b/include/field.h
index 49f35f84..a2b6a56f 100644
--- a/include/field.h
+++ b/include/field.h
@@ -77,6 +77,8 @@ namespace fields {
static const std::string REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";
static const std::string store = "store";
+
+ static const std::string hnsw_params = "hnsw_params";
}
enum vector_distance_type_t {
@@ -117,16 +119,18 @@ struct field {
bool stem = false;
std::shared_ptr<Stemmer> stemmer;
+
+ nlohmann::json hnsw_params;
field() {}
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,
int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine,
- std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false) :
+ std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false, const nlohmann::json hnsw_params = nlohmann::json()) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),
nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference),
- embed(embed), range_index(range_index), store(store), stem(stem) {
+ embed(embed), range_index(range_index), store(store), stem(stem), hnsw_params(hnsw_params) {
set_computed_defaults(sort, infix);
diff --git a/include/index.h b/include/index.h
index 3147d4a1..3e2c3f22 100644
--- a/include/index.h
+++ b/include/index.h
@@ -313,9 +313,9 @@ struct hnsw_index_t {
// ensures that this index is not dropped when it's being repaired
std::mutex repair_m;
- hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type):
+ hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type, size_t M = 16, size_t ef_construction = 200) :
space(new hnswlib::InnerProductSpace(num_dim)),
- vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, 16, 200, 100, true)),
+ vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, M, ef_construction, 100, true)),
num_dim(num_dim), distance_type(distance_type) {
}
diff --git a/include/vector_query_ops.h b/include/vector_query_ops.h
index 120c1d0b..9c78c2be 100644
--- a/include/vector_query_ops.h
+++ b/include/vector_query_ops.h
@@ -17,6 +17,8 @@ struct vector_query_t {
bool query_doc_given = false;
float alpha = 0.3;
+ uint32_t ef = 10;
+
std::vector<std::string> queries;
std::vector<float> query_weights;
diff --git a/src/collection.cpp b/src/collection.cpp
index 7b2ba723..9e1fa4cd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -435,6 +435,10 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::infix] = coll_field.infix;
field_json[fields::locale] = coll_field.locale;
field_json[fields::stem] = coll_field.stem;
+ // no need to sned hnsw_params for text fields
+ if(coll_field.num_dim > 0) {
+ field_json[fields::hnsw_params] = coll_field.hnsw_params;
+ }
if(coll_field.embed.count(fields::from) != 0) {
field_json[fields::embed] = coll_field.embed;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a6cf8274..12bf5077 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -70,6 +70,25 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
if(field_obj.count(fields::model_config) == 0) {
field_obj[fields::model_config] = nlohmann::json::object();
}
+
+ if(field_obj.count(fields::hnsw_params) == 0) {
+ field_obj[fields::hnsw_params] = nlohmann::json::object();
+ field_obj[fields::hnsw_params]["ef_construction"] = 200;
+ field_obj[fields::hnsw_params]["M"] = 16;
+ }
+
+ if(field_obj.count(fields::stem) == 0) {
+ field_obj[fields::stem] = false;
+ }
+
+ if(field_obj.count(fields::range_index) == 0) {
+ field_obj[fields::range_index] = false;
+ }
+
+ if(field_obj.count(fields::store) == 0) {
+ field_obj[fields::store] = true;
+ }
+
vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;
if(field_obj.count(fields::vec_dist) != 0) {
@@ -97,7 +116,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],
- field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed]);
+ field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed], field_obj[fields::range_index], field_obj[fields::store], field_obj[fields::stem], field_obj[fields::hnsw_params]);
// value of `sort` depends on field type
if(field_obj.count(fields::sort) == 0) {
diff --git a/src/field.cpp b/src/field.cpp
index cc2e4e56..931bc419 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -313,6 +313,47 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
+ if(field_json.count(fields::hnsw_params) != 0) {
+ if(!field_json[fields::hnsw_params].is_object()) {
+ return Option<bool>(400, "Property `" + fields::hnsw_params + "` must be an object.");
+ }
+
+ if(field_json[fields::hnsw_params].count("ef_construction") != 0 &&
+ (!field_json[fields::hnsw_params]["ef_construction"].is_number_unsigned() ||
+ field_json[fields::hnsw_params]["ef_construction"] == 0)) {
+ return Option<bool>(400, "Property `" + fields::hnsw_params + ".ef_construction` must be a positive integer.");
+ }
+
+ if(field_json[fields::hnsw_params].count("M") != 0 &&
+ (!field_json[fields::hnsw_params]["M"].is_number_unsigned() ||
+ field_json[fields::hnsw_params]["M"] == 0)) {
+ return Option<bool>(400, "Property `" + fields::hnsw_params + ".M` must be a positive integer.");
+ }
+
+ // remove unrelated properties except for m ef_construction and M
+ auto it = field_json[fields::hnsw_params].begin();
+ while(it != field_json[fields::hnsw_params].end()) {
+ if(it.key() != "max_elements" && it.key() != "ef_construction" && it.key() != "M" && it.key() != "ef") {
+ it = field_json[fields::hnsw_params].erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ if(field_json[fields::hnsw_params].count("ef_construction") == 0) {
+ field_json[fields::hnsw_params]["ef_construction"] = 200;
+ }
+
+ if(field_json[fields::hnsw_params].count("M") == 0) {
+ field_json[fields::hnsw_params]["M"] = 16;
+ }
+ } else {
+ field_json[fields::hnsw_params] = R"({
+ "M": 16,
+ "ef_construction": 200
+ })"_json;
+ }
+
if(field_json.count(fields::optional) == 0) {
// dynamic type fields are always optional
bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
@@ -364,7 +405,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,
- field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store], field_json[fields::stem])
+ field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index],
+ field_json[fields::store], field_json[fields::stem], field_json[fields::hnsw_params])
);
if (!field_json[fields::reference].get<std::string>().empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index 139bf3d0..cef54ef7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -68,7 +68,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
}
if(a_field.num_dim > 0) {
- auto hnsw_index = new hnsw_index_t(a_field.num_dim, 1024, a_field.vec_dist);
+ auto hnsw_index = new hnsw_index_t(a_field.num_dim, 1024, a_field.vec_dist, a_field.hnsw_params["M"].get<uint32_t>(), a_field.hnsw_params["ef_construction"].get<uint32_t>());
vector_index.emplace(a_field.name, hnsw_index);
continue;
}
@@ -3048,9 +3048,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
- pairs = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, &filterFunctor);
+ pairs = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, vector_query.ef, &filterFunctor);
} else {
- pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);
+ pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, vector_query.ef, &filterFunctor);
}
std::sort(pairs.begin(), pairs.end(), [](auto& x, auto& y) {
@@ -3420,13 +3420,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// use k as 100 by default for ensuring results stability in pagination
size_t default_k = 100;
auto k = vector_query.k == 0 ? std::max<size_t>(fetch_size, default_k) : vector_query.k;
-
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, &filterFunctor);
+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, vector_query.ef, &filterFunctor);
} else {
- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);
+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, vector_query.ef, &filterFunctor);
}
filter_result_iterator->reset();
search_cutoff = search_cutoff || filter_result_iterator->validity == filter_result_iterator_t::timed_out;
@@ -7108,7 +7107,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
search_schema.emplace(new_field.name, new_field);
if(new_field.type == field_types::FLOAT_ARRAY && new_field.num_dim > 0) {
- auto hnsw_index = new hnsw_index_t(new_field.num_dim, 1024, new_field.vec_dist);
+ auto hnsw_index = new hnsw_index_t(new_field.num_dim, 1024, new_field.vec_dist, new_field.hnsw_params["M"].get<uint32_t>(), new_field.hnsw_params["ef_construction"].get<uint32_t>());
vector_index.emplace(new_field.name, hnsw_index);
continue;
}
diff --git a/src/vector_query_ops.cpp b/src/vector_query_ops.cpp
index d4996432..dfa1dda4 100644
--- a/src/vector_query_ops.cpp
+++ b/src/vector_query_ops.cpp
@@ -182,6 +182,14 @@ Option<bool> VectorQueryOps::parse_vector_query_str(const std::string& vector_qu
vector_query.alpha = std::stof(param_kv[1]);
}
+ if(param_kv[0] == "ef") {
+ if(!StringUtils::is_uint32_t(param_kv[1]) || std::stoul(param_kv[1]) == 0) {
+ return Option<bool>(400, "Malformed vector query string: `ef` parameter must be a positive integer.");
+ }
+
+ vector_query.ef = std::stoul(param_kv[1]);
+ }
+
if(param_kv[0] == "queries") {
if(param_kv[1].front() != '[' || param_kv[1].back() != ']') {
return Option<bool>(400, "Malformed vector query string: "
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 29ad2a43..b6dadcdb 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -3850,4 +3850,219 @@ TEST_F(CollectionVectorTest, TestInvalidVoiceQuery) {
ASSERT_FALSE(results.ok());
ASSERT_EQ("Invalid audio format. Please provide a 16-bit 16kHz wav file.", results.error());
+}
+
+TEST_F(CollectionVectorTest, TestInvalidHNSWParams) {
+ nlohmann::json schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": "aaa",
+ "M": 16
+ }
+ }
+ ]
+ })"_json;
+
+
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_FALSE(collection_create_op.ok());
+
+ ASSERT_EQ("Property `hnsw_params.ef_construction` must be a positive integer.", collection_create_op.error());
+
+ schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": -100,
+ "M": 16
+ }
+ }
+ ]
+ })"_json;
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_FALSE(collection_create_op.ok());
+
+ ASSERT_EQ("Property `hnsw_params.ef_construction` must be a positive integer.", collection_create_op.error());
+
+
+ schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": 100,
+ "M": "aaa"
+ }
+ }
+ ]
+ })"_json;
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_FALSE(collection_create_op.ok());
+ ASSERT_EQ("Property `hnsw_params.M` must be a positive integer.", collection_create_op.error());
+
+
+ schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": 100,
+ "M": -100
+ }
+ }
+ ]
+ })"_json;
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_FALSE(collection_create_op.ok());
+ ASSERT_EQ("Property `hnsw_params.M` must be a positive integer.", collection_create_op.error());
+
+
+ schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": 100,
+ "M": 16
+ }
+ }
+ ]
+ })"_json;
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto collection = collection_create_op.get();
+
+
+ auto results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:aaa)");
+
+ ASSERT_FALSE(results.ok());
+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());
+
+ results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:-100)");
+
+ ASSERT_FALSE(results.ok());
+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());
+
+ results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:0)");
+
+ ASSERT_FALSE(results.ok());
+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());
+
+ results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:100)");
+
+ ASSERT_TRUE(results.ok());
+
+}
+
+TEST_F(CollectionVectorTest, TestHNSWParamsSummaryJSON) {
+ nlohmann::json schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "vector",
+ "type": "float[]",
+ "embed": {
+ "from": ["name"],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ },
+ "hnsw_params": {
+ "ef_construction": 100,
+ "M": 16
+ }
+ }
+ ]
+ })"_json;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto collection = collection_create_op.get();
+
+ auto summary = collection->get_summary_json();
+
+ ASSERT_TRUE(summary["fields"][1]["hnsw_params"].is_object());
+ ASSERT_EQ(100, summary["fields"][1]["hnsw_params"]["ef_construction"].get<uint32_t>());
+ ASSERT_EQ(16, summary["fields"][1]["hnsw_params"]["M"].get<uint32_t>());
+ ASSERT_EQ(0, summary["fields"][0].count("hnsw_params"));
}
\ No newline at end of file