Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix reindexing old documents with embeddings on alter

ozanarmaganeaef5cb8c6e4
diff --git a/src/collection.cpp b/src/collection.cpp
index 28907b33..765dbc06 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3756,6 +3756,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
std::vector<std::string> nested_field_names;
+ bool found_embedding_field = false;
+
for(auto& f: alter_fields) {
if(f.name == ".*") {
fields.push_back(f);
@@ -3776,6 +3778,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
if(f.embed.count(fields::from) != 0) {
+ found_embedding_field = true;
embedding_fields.emplace(f.name, f);
}
@@ -3834,8 +3837,24 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
}
- Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true, 200, false);
+ Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,
+ fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field);
+ if(found_embedding_field) {
+ for(auto& index_record : iter_batch) {
+ if(index_record.indexed.ok()) {
+ remove_flat_fields(index_record.doc);
+ const std::string& serialized_json = index_record.doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);
+ bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);
+
+ if(!write_ok) {
+ LOG(ERROR) << "Inserting doc with new embedding field failed for seq id: " << index_record.seq_id;
+ index_record.index_failure(500, "Could not write to on-disk storage.");
+ } else {
+ index_record.index_success();
+ }
+ }
+ }
+ }
iter_batch.clear();
}
diff --git a/src/validator.cpp b/src/validator.cpp
index f8c23ee9..4f3df1aa 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -53,8 +53,9 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
}
} else if(a_field.is_array()) {
if(!doc_ele.is_array()) {
- if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||
- dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {
+ bool is_embedding_field = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;
+ if((a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||
+ dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) || is_embedding_field) {
document.erase(field_name);
return Option<uint32_t>(200);
} else {
@@ -630,7 +631,9 @@ Option<uint32_t> validator_t::validate_index_in_memory(nlohmann::json& document,
continue;
}
- if(document.count(field_name) == 0) {
+ bool is_embedding_field = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;
+
+ if(document.count(field_name) == 0 && !is_embedding_field) {
return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
"but is not found in the document.");
}
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index c936ce78..858818fc 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1793,3 +1793,39 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterDropTest) {
ASSERT_EQ(0, vec_index.size());
ASSERT_EQ(0, vec_index.count("embedding"));
}
+
+
+TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterUpdateOldDocs) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ nlohmann::json doc;
+ doc["title"] = "hello";
+
+ auto add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json schema_change = R"({
+ "fields": [
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["title"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ auto schema_change_op = coll->alter(schema_change);
+ ASSERT_TRUE(schema_change_op.ok());
+
+ auto search_res = coll->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+
+ ASSERT_EQ(1, search_res.get()["found"].get<size_t>());
+ ASSERT_EQ(384, search_res.get()["hits"][0]["document"]["embedding"].get<std::vector<float>>().size());
+}