Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix duplicate results from vector index (#1536)
Ozan Armağan559b2de337d3
diff --git a/WORKSPACE b/WORKSPACEindex 3ed3bf77..80e6a65c 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -186,7 +186,7 @@ new_git_repository(new_git_repository(name = "hnsw",build_file = "//bazel:hnsw.BUILD",- commit = "2fec56c161548ae8b2ab6bacfe404e4040729cd7",+ commit = "687d981753f8bafcd16421cbd2a166d0b62bc520",remote = "https://github.com/typesense/hnswlib.git",)diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex b6dadcdb..73569a40 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -4065,4 +4065,86 @@ TEST_F(CollectionVectorTest, TestHNSWParamsSummaryJSON) {ASSERT_EQ(100, summary["fields"][1]["hnsw_params"]["ef_construction"].get<uint32_t>());ASSERT_EQ(16, summary["fields"][1]["hnsw_params"]["M"].get<uint32_t>());ASSERT_EQ(0, summary["fields"][0].count("hnsw_params"));+}++TEST_F(CollectionVectorTest, TestUpdatingSameDocument){+ nlohmann::json schema_json = R"({+ "name": "test",+ "fields": [+ {"name": "vector", "type": "float[]", "num_dim": 10}+ ]+ })"_json;++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ auto collection = collection_create_op.get();++ std::mt19937 rng;+ std::uniform_real_distribution<float> dist;++ // generate 100 random documents+ for (int i = 0; i < 100; i++) {+ std::vector<float> vector(10);+ std::generate(vector.begin(), vector.end(), [&](){ return dist(rng); });++ nlohmann::json doc = {+ {"vector", vector}+ };+ auto op = collection->add(doc.dump());+ ASSERT_TRUE(op.ok());+ }++ std::vector<float> query_vector(10);+ std::generate(query_vector.begin(), query_vector.end(), [&](){ return dist(rng); });+ std::string query_vector_str = "vector:([";+ for (int i = 0; i < 10; i++) {+ query_vector_str += std::to_string(query_vector[i]);+ if (i != 9) {+ query_vector_str += ", ";+ }+ }+ query_vector_str += "], k:10)";++ auto results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, query_vector_str);+ ASSERT_TRUE(results.ok());+ auto results_json = results.get();+ ASSERT_EQ(results_json["found"].get<size_t>(), results_json["hits"].size());++ // delete half of the documents+ for (int i = 50; i < 99; i++) {+ auto op = collection->remove(std::to_string(i));+ ASSERT_TRUE(op.ok());+ }++ // update document with id 11 for 100 times+ for (int i = 0; i < 100; i++) {+ std::vector<float> vector(10);+ std::generate(vector.begin(), vector.end(), [&](){ return dist(rng); });++ nlohmann::json doc = {+ {"vector", vector}+ };+ auto op = collection->add(doc.dump(), index_operation_t::UPDATE, "11");+ ASSERT_TRUE(op.ok());+ }+++ results = collection->search("*", {}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>{"vector"}, 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, query_vector_str);+ ASSERT_TRUE(results.ok());++ results_json = results.get();+ ASSERT_EQ(results_json["found"].get<size_t>(), results_json["hits"].size());}\ No newline at end of file