Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Resize hnsw graph dynamically.

Kishore Nallan914a4013129d
diff --git a/cmake/hnsw.cmake b/cmake/hnsw.cmake
index 0196a52a..ccf6ad57 100644
--- a/cmake/hnsw.cmake
+++ b/cmake/hnsw.cmake
@@ -1,6 +1,6 @@
# Download hnsw (header-only)
-set(HNSW_VERSION b87f6230dbe59e874b3099cfcab689b42e887a20)
+set(HNSW_VERSION cc69672b90ff7150016ceeba7d547d8a9608db41)
set(HNSW_NAME hnswlib-${HNSW_VERSION})
set(HNSW_TAR_PATH ${DEP_ROOT_DIR}/${HNSW_NAME}.tar.gz)
diff --git a/src/index.cpp b/src/index.cpp
index cab8e595..17e95c4a 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -932,6 +932,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(afield.type == field_types::FLOAT_ARRAY && afield.num_dim > 0) {
const std::vector<float>& float_vals = record.doc[afield.name].get<std::vector<float>>();
+ auto vec_index = vector_index[afield.name]->vecdex;
+ size_t curr_ele_count = vec_index->getCurrentElementCount();
+ if(curr_ele_count == vec_index->getMaxElements()) {
+ vec_index->resizeIndex(curr_ele_count * 1.3);
+ }
vector_index[afield.name]->vecdex->addPoint(float_vals.data(), (size_t)seq_id);
}
});
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 6ed87bce..14a91472 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -154,3 +154,49 @@ TEST_F(CollectionVectorTest, BasicVectorQuerying) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionVectorTest, Index10KVectors) {
+ // tests the dynamic resizing of graph
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "points", "type": "int32"},
+ {"name": "vec", "type": "float[]", "num_dim": 4}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ size_t d = 4;
+ size_t n = 10 * 1000;
+
+ std::mt19937 rng;
+ rng.seed(47);
+ std::uniform_real_distribution<> distrib;
+
+ for (size_t i = 0; i < n; i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["title"] = std::to_string(i) + " title";
+ doc["points"] = i;
+
+ std::vector<float> values;
+ for (size_t j = 0; j < d; j++) {
+ values.push_back(distrib(rng));
+ }
+ doc["vec"] = values;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "").get();
+
+ ASSERT_EQ(10000, results["found"].get<size_t>());
+}