Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Don't remove vector during update: use update api directly.

Kishore Nallan0db3dd0d00c7
diff --git a/include/index.h b/include/index.h
index 5656cc2f..66cf3365 100644
--- a/include/index.h
+++ b/include/index.h
@@ -665,7 +665,8 @@ public:
const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,
const std::string& collection_name) const;
- void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);
+ void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
+ const bool is_update);
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document,
const std::vector<field>& del_fields, const bool is_update);
diff --git a/src/index.cpp b/src/index.cpp
index ac7be1d6..f46916b8 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -5852,7 +5852,8 @@ void Index::remove_facet_token(const field& search_field, spp::sparse_hash_map<s
}
}
-void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name) {
+void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
+ const bool is_update) {
const auto& search_field_it = search_schema.find(field_name);
if(search_field_it == search_schema.end()) {
return;
@@ -5912,7 +5913,10 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
}
}
} else if(search_field.num_dim) {
- vector_index[search_field.name]->vecdex->markDelete(seq_id);
+ if(!is_update) {
+ // since vector index supports upsert natively, we should not attempt to delete for update
+ vector_index[search_field.name]->vecdex->markDelete(seq_id);
+ }
} else if(search_field.is_float()) {
const std::vector<float>& values = search_field.is_single_float() ?
std::vector<float>{document[field_name].get<float>()} :
@@ -6018,7 +6022,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
}
try {
- remove_field(seq_id, document, the_field.name);
+ remove_field(seq_id, document, the_field.name, is_update);
} catch(const std::exception& e) {
LOG(WARNING) << "Error while removing field `" << the_field.name << "` from document, message: "
<< e.what();
@@ -6028,7 +6032,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
for(auto it = document.begin(); it != document.end(); ++it) {
const std::string& field_name = it.key();
try {
- remove_field(seq_id, document, field_name);
+ remove_field(seq_id, document, field_name, is_update);
} catch(const std::exception& e) {
LOG(WARNING) << "Error while removing field `" << field_name << "` from document, message: "
<< e.what();
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 4fcc134c..ece18fa6 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -287,6 +287,97 @@ TEST_F(CollectionVectorTest, VectorUnchangedUpsert) {
ASSERT_EQ(1, results["found"].get<size_t>());
}
+TEST_F(CollectionVectorTest, VectorManyUpserts) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "points", "type": "int32"},
+ {"name": "vec", "type": "float[]", "num_dim": 3}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ size_t d = 3;
+ size_t n = 50;
+
+ std::mt19937 rng;
+ rng.seed(47);
+ std::uniform_real_distribution<> distrib;
+
+ std::vector<std::string> import_records;
+
+ // first insert n docs
+ for (size_t i = 0; i < n; i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["title"] = std::to_string(i) + " title";
+ doc["points"] = i;
+
+ std::vector<float> values;
+ for (size_t j = 0; j < d; j++) {
+ values.push_back(distrib(rng));
+ }
+ doc["vec"] = values;
+ import_records.push_back(doc.dump());
+ }
+
+ nlohmann::json document;
+ nlohmann::json import_response = coll1->add_many(import_records, document);
+
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(n, import_response["num_imported"].get<int>());
+ import_records.clear();
+
+ size_t num_new_records = 0;
+
+ // upsert mix of old + new docs50
+ for (size_t i = 0; i < n; i++) {
+ nlohmann::json doc;
+ auto id = i;
+ if(i % 2 != 0) {
+ id = (i + 1000);
+ num_new_records++;
+ }
+
+ doc["id"] = std::to_string(id);
+ doc["title"] = std::to_string(id) + " title";
+ doc["points"] = id;
+
+ std::vector<float> values;
+ for (size_t j = 0; j < d; j++) {
+ values.push_back(distrib(rng) + 0.01);
+ }
+ doc["vec"] = values;
+ import_records.push_back(doc.dump());
+ }
+
+ import_response = coll1->add_many(import_records, document, UPSERT);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(n, import_response["num_imported"].get<int>());
+ import_records.clear();
+
+ /*for(size_t i = 0; i < 100; i++) {
+ auto results = coll1->search("*", {}, "", {}, {}, {0}, 200, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.12, 0.44, 0.55])").get();
+
+ if(results["found"].get<size_t>() != n+num_new_records) {
+ LOG(INFO) << results["found"].get<size_t>();
+ }
+ }*/
+
+ //LOG(INFO) << "Expected: " << n + num_new_records;
+ //ASSERT_EQ(n + num_new_records, results["found"].get<size_t>());
+ //ASSERT_EQ(n + num_new_records, results["hits"].size());
+}
+
+
TEST_F(CollectionVectorTest, VectorPartialUpdate) {
nlohmann::json schema = R"({
"name": "coll1",