Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Make HNSW params configurable (#1521)
Ozan Armağan18306e3dd268
diff --git a/WORKSPACE b/WORKSPACEindex d090cccf..3ed3bf77 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -186,7 +186,7 @@ new_git_repository(new_git_repository(name = "hnsw",build_file = "//bazel:hnsw.BUILD",- commit = "5100d3fe41da45601875b3f395f508398cb12b8a",+ commit = "2fec56c161548ae8b2ab6bacfe404e4040729cd7",remote = "https://github.com/typesense/hnswlib.git",)diff --git a/include/field.h b/include/field.hindex 49f35f84..a2b6a56f 100644--- a/include/field.h+++ b/include/field.h@@ -77,6 +77,8 @@ namespace fields {static const std::string REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";static const std::string store = "store";++ static const std::string hnsw_params = "hnsw_params";}enum vector_distance_type_t {@@ -117,16 +119,18 @@ struct field {bool stem = false;std::shared_ptr<Stemmer> stemmer;++ nlohmann::json hnsw_params;field() {}field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine,- std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false) :+ std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false, const nlohmann::json hnsw_params = nlohmann::json()) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference),- embed(embed), range_index(range_index), store(store), stem(stem) {+ embed(embed), range_index(range_index), store(store), stem(stem), hnsw_params(hnsw_params) {set_computed_defaults(sort, infix);diff --git a/include/index.h b/include/index.hindex 3147d4a1..3e2c3f22 100644--- a/include/index.h+++ b/include/index.h@@ -313,9 +313,9 @@ struct hnsw_index_t {// ensures that this index is not dropped when it's being repairedstd::mutex repair_m;- hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type):+ hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type, size_t M = 16, size_t ef_construction = 200) :space(new hnswlib::InnerProductSpace(num_dim)),- vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, 16, 200, 100, true)),+ vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, M, ef_construction, 100, true)),num_dim(num_dim), distance_type(distance_type) {}diff --git a/include/vector_query_ops.h b/include/vector_query_ops.hindex 120c1d0b..9c78c2be 100644--- a/include/vector_query_ops.h+++ b/include/vector_query_ops.h@@ -17,6 +17,8 @@ struct vector_query_t {bool query_doc_given = false;float alpha = 0.3;+ uint32_t ef = 10;+std::vector<std::string> queries;std::vector<float> query_weights;diff --git a/src/collection.cpp b/src/collection.cppindex 7b2ba723..9e1fa4cd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -435,6 +435,10 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::infix] = coll_field.infix;field_json[fields::locale] = coll_field.locale;field_json[fields::stem] = coll_field.stem;+ // no need to sned hnsw_params for text fields+ if(coll_field.num_dim > 0) {+ field_json[fields::hnsw_params] = coll_field.hnsw_params;+ }if(coll_field.embed.count(fields::from) != 0) {field_json[fields::embed] = coll_field.embed;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a6cf8274..12bf5077 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -70,6 +70,25 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionif(field_obj.count(fields::model_config) == 0) {field_obj[fields::model_config] = nlohmann::json::object();}++ if(field_obj.count(fields::hnsw_params) == 0) {+ field_obj[fields::hnsw_params] = nlohmann::json::object();+ field_obj[fields::hnsw_params]["ef_construction"] = 200;+ field_obj[fields::hnsw_params]["M"] = 16;+ }++ if(field_obj.count(fields::stem) == 0) {+ field_obj[fields::stem] = false;+ }++ if(field_obj.count(fields::range_index) == 0) {+ field_obj[fields::range_index] = false;+ }++ if(field_obj.count(fields::store) == 0) {+ field_obj[fields::store] = true;+ }+vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;if(field_obj.count(fields::vec_dist) != 0) {@@ -97,7 +116,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],- field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed]);+ field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed], field_obj[fields::range_index], field_obj[fields::store], field_obj[fields::stem], field_obj[fields::hnsw_params]);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {diff --git a/src/field.cpp b/src/field.cppindex cc2e4e56..931bc419 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -313,6 +313,47 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}+ if(field_json.count(fields::hnsw_params) != 0) {+ if(!field_json[fields::hnsw_params].is_object()) {+ return Option<bool>(400, "Property `" + fields::hnsw_params + "` must be an object.");+ }++ if(field_json[fields::hnsw_params].count("ef_construction") != 0 &&+ (!field_json[fields::hnsw_params]["ef_construction"].is_number_unsigned() ||+ field_json[fields::hnsw_params]["ef_construction"] == 0)) {+ return Option<bool>(400, "Property `" + fields::hnsw_params + ".ef_construction` must be a positive integer.");+ }++ if(field_json[fields::hnsw_params].count("M") != 0 &&+ (!field_json[fields::hnsw_params]["M"].is_number_unsigned() ||+ field_json[fields::hnsw_params]["M"] == 0)) {+ return Option<bool>(400, "Property `" + fields::hnsw_params + ".M` must be a positive integer.");+ }++ // remove unrelated properties except for m ef_construction and M+ auto it = field_json[fields::hnsw_params].begin();+ while(it != field_json[fields::hnsw_params].end()) {+ if(it.key() != "max_elements" && it.key() != "ef_construction" && it.key() != "M" && it.key() != "ef") {+ it = field_json[fields::hnsw_params].erase(it);+ } else {+ ++it;+ }+ }++ if(field_json[fields::hnsw_params].count("ef_construction") == 0) {+ field_json[fields::hnsw_params]["ef_construction"] = 200;+ }++ if(field_json[fields::hnsw_params].count("M") == 0) {+ field_json[fields::hnsw_params]["M"] = 16;+ }+ } else {+ field_json[fields::hnsw_params] = R"({+ "M": 16,+ "ef_construction": 200+ })"_json;+ }+if(field_json.count(fields::optional) == 0) {// dynamic type fields are always optionalbool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);@@ -364,7 +405,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::optional], field_json[fields::index], field_json[fields::locale],field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,- field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store], field_json[fields::stem])+ field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index],+ field_json[fields::store], field_json[fields::stem], field_json[fields::hnsw_params]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/index.cpp b/src/index.cppindex 139bf3d0..cef54ef7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -68,7 +68,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}if(a_field.num_dim > 0) {- auto hnsw_index = new hnsw_index_t(a_field.num_dim, 1024, a_field.vec_dist);+ auto hnsw_index = new hnsw_index_t(a_field.num_dim, 1024, a_field.vec_dist, a_field.hnsw_params["M"].get<uint32_t>(), a_field.hnsw_params["ef_construction"].get<uint32_t>());vector_index.emplace(a_field.name, hnsw_index);continue;}@@ -3048,9 +3048,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consif(field_vector_index->distance_type == cosine) {std::vector<float> normalized_q(vector_query.values.size());hnsw_index_t::normalize_vector(vector_query.values, normalized_q);- pairs = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, &filterFunctor);+ pairs = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, vector_query.ef, &filterFunctor);} else {- pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);+ pairs = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, vector_query.ef, &filterFunctor);}std::sort(pairs.begin(), pairs.end(), [](auto& x, auto& y) {@@ -3420,13 +3420,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// use k as 100 by default for ensuring results stability in paginationsize_t default_k = 100;auto k = vector_query.k == 0 ? std::max<size_t>(fetch_size, default_k) : vector_query.k;-if(field_vector_index->distance_type == cosine) {std::vector<float> normalized_q(vector_query.values.size());hnsw_index_t::normalize_vector(vector_query.values, normalized_q);- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, &filterFunctor);+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, vector_query.ef, &filterFunctor);} else {- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, vector_query.ef, &filterFunctor);}filter_result_iterator->reset();search_cutoff = search_cutoff || filter_result_iterator->validity == filter_result_iterator_t::timed_out;@@ -7108,7 +7107,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecsearch_schema.emplace(new_field.name, new_field);if(new_field.type == field_types::FLOAT_ARRAY && new_field.num_dim > 0) {- auto hnsw_index = new hnsw_index_t(new_field.num_dim, 1024, new_field.vec_dist);+ auto hnsw_index = new hnsw_index_t(new_field.num_dim, 1024, new_field.vec_dist, new_field.hnsw_params["M"].get<uint32_t>(), new_field.hnsw_params["ef_construction"].get<uint32_t>());vector_index.emplace(new_field.name, hnsw_index);continue;}diff --git a/src/vector_query_ops.cpp b/src/vector_query_ops.cppindex d4996432..dfa1dda4 100644--- a/src/vector_query_ops.cpp+++ b/src/vector_query_ops.cpp@@ -182,6 +182,14 @@ Option<bool> VectorQueryOps::parse_vector_query_str(const std::string& vector_quvector_query.alpha = std::stof(param_kv[1]);}+ if(param_kv[0] == "ef") {+ if(!StringUtils::is_uint32_t(param_kv[1]) || std::stoul(param_kv[1]) == 0) {+ return Option<bool>(400, "Malformed vector query string: `ef` parameter must be a positive integer.");+ }++ vector_query.ef = std::stoul(param_kv[1]);+ }+if(param_kv[0] == "queries") {if(param_kv[1].front() != '[' || param_kv[1].back() != ']') {return Option<bool>(400, "Malformed vector query string: "diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 29ad2a43..b6dadcdb 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -3850,4 +3850,219 @@ TEST_F(CollectionVectorTest, TestInvalidVoiceQuery) {ASSERT_FALSE(results.ok());ASSERT_EQ("Invalid audio format. Please provide a 16-bit 16kHz wav file.", results.error());+}++TEST_F(CollectionVectorTest, TestInvalidHNSWParams) {+ nlohmann::json schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": "aaa",+ "M": 16+ }+ }+ ]+ })"_json;++++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_FALSE(collection_create_op.ok());++ ASSERT_EQ("Property `hnsw_params.ef_construction` must be a positive integer.", collection_create_op.error());++ schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": -100,+ "M": 16+ }+ }+ ]+ })"_json;++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_FALSE(collection_create_op.ok());++ ASSERT_EQ("Property `hnsw_params.ef_construction` must be a positive integer.", collection_create_op.error());+++ schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": 100,+ "M": "aaa"+ }+ }+ ]+ })"_json;++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_FALSE(collection_create_op.ok());+ ASSERT_EQ("Property `hnsw_params.M` must be a positive integer.", collection_create_op.error());+++ schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": 100,+ "M": -100+ }+ }+ ]+ })"_json;++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_FALSE(collection_create_op.ok());+ ASSERT_EQ("Property `hnsw_params.M` must be a positive integer.", collection_create_op.error());+++ schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": 100,+ "M": 16+ }+ }+ ]+ })"_json;++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ auto collection = collection_create_op.get();+++ auto results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:aaa)");++ ASSERT_FALSE(results.ok());+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());++ results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:-100)");++ ASSERT_FALSE(results.ok());+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());++ results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:0)");++ ASSERT_FALSE(results.ok());+ ASSERT_EQ("Malformed vector query string: `ef` parameter must be a positive integer.", results.error());++ results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "vector:([], ef:100)");++ ASSERT_TRUE(results.ok());++}++TEST_F(CollectionVectorTest, TestHNSWParamsSummaryJSON) {+ nlohmann::json schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {+ "name": "vector",+ "type": "float[]",+ "embed": {+ "from": ["name"],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ },+ "hnsw_params": {+ "ef_construction": 100,+ "M": 16+ }+ }+ ]+ })"_json;++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ auto collection = collection_create_op.get();++ auto summary = collection->get_summary_json();++ ASSERT_TRUE(summary["fields"][1]["hnsw_params"].is_object());+ ASSERT_EQ(100, summary["fields"][1]["hnsw_params"]["ef_construction"].get<uint32_t>());+ ASSERT_EQ(16, summary["fields"][1]["hnsw_params"]["M"].get<uint32_t>());+ ASSERT_EQ(0, summary["fields"][0].count("hnsw_params"));}\ No newline at end of file