Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Resize hnsw graph dynamically.
Kishore Nallan914a4013129d
diff --git a/cmake/hnsw.cmake b/cmake/hnsw.cmakeindex 0196a52a..ccf6ad57 100644--- a/cmake/hnsw.cmake+++ b/cmake/hnsw.cmake@@ -1,6 +1,6 @@# Download hnsw (header-only)-set(HNSW_VERSION b87f6230dbe59e874b3099cfcab689b42e887a20)+set(HNSW_VERSION cc69672b90ff7150016ceeba7d547d8a9608db41)set(HNSW_NAME hnswlib-${HNSW_VERSION})set(HNSW_TAR_PATH ${DEP_ROOT_DIR}/${HNSW_NAME}.tar.gz)diff --git a/src/index.cpp b/src/index.cppindex cab8e595..17e95c4a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -932,6 +932,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(afield.type == field_types::FLOAT_ARRAY && afield.num_dim > 0) {const std::vector<float>& float_vals = record.doc[afield.name].get<std::vector<float>>();+ auto vec_index = vector_index[afield.name]->vecdex;+ size_t curr_ele_count = vec_index->getCurrentElementCount();+ if(curr_ele_count == vec_index->getMaxElements()) {+ vec_index->resizeIndex(curr_ele_count * 1.3);+ }vector_index[afield.name]->vecdex->addPoint(float_vals.data(), (size_t)seq_id);}});diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 6ed87bce..14a91472 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -154,3 +154,49 @@ TEST_F(CollectionVectorTest, BasicVectorQuerying) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionVectorTest, Index10KVectors) {+ // tests the dynamic resizing of graph+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "points", "type": "int32"},+ {"name": "vec", "type": "float[]", "num_dim": 4}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ size_t d = 4;+ size_t n = 10 * 1000;++ std::mt19937 rng;+ rng.seed(47);+ std::uniform_real_distribution<> distrib;++ for (size_t i = 0; i < n; i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["title"] = std::to_string(i) + " title";+ doc["points"] = i;++ std::vector<float> values;+ for (size_t j = 0; j < d; j++) {+ values.push_back(distrib(rng));+ }+ doc["vec"] = values;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "").get();++ ASSERT_EQ(10000, results["found"].get<size_t>());+}