Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix duplicate results from vector index (#1536)

Ozan Armağan559b2de337d3
diff --git a/WORKSPACE b/WORKSPACE
index 3ed3bf77..80e6a65c 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -186,7 +186,7 @@ new_git_repository(
new_git_repository(
name = "hnsw",
build_file = "//bazel:hnsw.BUILD",
- commit = "2fec56c161548ae8b2ab6bacfe404e4040729cd7",
+ commit = "687d981753f8bafcd16421cbd2a166d0b62bc520",
remote = "https://github.com/typesense/hnswlib.git",
)
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index b6dadcdb..73569a40 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -4065,4 +4065,86 @@ TEST_F(CollectionVectorTest, TestHNSWParamsSummaryJSON) {
ASSERT_EQ(100, summary["fields"][1]["hnsw_params"]["ef_construction"].get<uint32_t>());
ASSERT_EQ(16, summary["fields"][1]["hnsw_params"]["M"].get<uint32_t>());
ASSERT_EQ(0, summary["fields"][0].count("hnsw_params"));
+}
+
+TEST_F(CollectionVectorTest, TestUpdatingSameDocument){
+ nlohmann::json schema_json = R"({
+ "name": "test",
+ "fields": [
+ {"name": "vector", "type": "float[]", "num_dim": 10}
+ ]
+ })"_json;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto collection = collection_create_op.get();
+
+ std::mt19937 rng;
+ std::uniform_real_distribution<float> dist;
+
+ // generate 100 random documents
+ for (int i = 0; i < 100; i++) {
+ std::vector<float> vector(10);
+ std::generate(vector.begin(), vector.end(), [&](){ return dist(rng); });
+
+ nlohmann::json doc = {
+ {"vector", vector}
+ };
+ auto op = collection->add(doc.dump());
+ ASSERT_TRUE(op.ok());
+ }
+
+ std::vector<float> query_vector(10);
+ std::generate(query_vector.begin(), query_vector.end(), [&](){ return dist(rng); });
+ std::string query_vector_str = "vector:([";
+ for (int i = 0; i < 10; i++) {
+ query_vector_str += std::to_string(query_vector[i]);
+ if (i != 9) {
+ query_vector_str += ", ";
+ }
+ }
+ query_vector_str += "], k:10)";
+
+ auto results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, query_vector_str);
+ ASSERT_TRUE(results.ok());
+ auto results_json = results.get();
+ ASSERT_EQ(results_json["found"].get<size_t>(), results_json["hits"].size());
+
+ // delete half of the documents
+ for (int i = 50; i < 99; i++) {
+ auto op = collection->remove(std::to_string(i));
+ ASSERT_TRUE(op.ok());
+ }
+
+ // update document with id 11 for 100 times
+ for (int i = 0; i < 100; i++) {
+ std::vector<float> vector(10);
+ std::generate(vector.begin(), vector.end(), [&](){ return dist(rng); });
+
+ nlohmann::json doc = {
+ {"vector", vector}
+ };
+ auto op = collection->add(doc.dump(), index_operation_t::UPDATE, "11");
+ ASSERT_TRUE(op.ok());
+ }
+
+
+ results = collection->search("*", {}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>{"vector"}, 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, query_vector_str);
+ ASSERT_TRUE(results.ok());
+
+ results_json = results.get();
+ ASSERT_EQ(results_json["found"].get<size_t>(), results_json["hits"].size());
}
\ No newline at end of file