Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix reindexing old documents with embeddings on alter
ozanarmaganeaef5cb8c6e4
diff --git a/src/collection.cpp b/src/collection.cppindex 28907b33..765dbc06 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3756,6 +3756,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsstd::vector<std::string> nested_field_names;+ bool found_embedding_field = false;+for(auto& f: alter_fields) {if(f.name == ".*") {fields.push_back(f);@@ -3776,6 +3778,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}if(f.embed.count(fields::from) != 0) {+ found_embedding_field = true;embedding_fields.emplace(f.name, f);}@@ -3834,8 +3837,24 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}}- Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true, 200, false);+ Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,+ fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field);+ if(found_embedding_field) {+ for(auto& index_record : iter_batch) {+ if(index_record.indexed.ok()) {+ remove_flat_fields(index_record.doc);+ const std::string& serialized_json = index_record.doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);+ bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);++ if(!write_ok) {+ LOG(ERROR) << "Inserting doc with new embedding field failed for seq id: " << index_record.seq_id;+ index_record.index_failure(500, "Could not write to on-disk storage.");+ } else {+ index_record.index_success();+ }+ }+ }+ }iter_batch.clear();}diff --git a/src/validator.cpp b/src/validator.cppindex f8c23ee9..4f3df1aa 100644--- a/src/validator.cpp+++ b/src/validator.cpp@@ -53,8 +53,9 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso}} else if(a_field.is_array()) {if(!doc_ele.is_array()) {- if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||- dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {+ bool is_embedding_field = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;+ if((a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||+ dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) || is_embedding_field) {document.erase(field_name);return Option<uint32_t>(200);} else {@@ -630,7 +631,9 @@ Option<uint32_t> validator_t::validate_index_in_memory(nlohmann::json& document,continue;}- if(document.count(field_name) == 0) {+ bool is_embedding_field = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;++ if(document.count(field_name) == 0 && !is_embedding_field) {return Option<>(400, "Field `" + field_name + "` has been declared in the schema, ""but is not found in the document.");}diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex c936ce78..858818fc 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1793,3 +1793,39 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterDropTest) {ASSERT_EQ(0, vec_index.size());ASSERT_EQ(0, vec_index.count("embedding"));}+++TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterUpdateOldDocs) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ nlohmann::json doc;+ doc["title"] = "hello";++ auto add_op = coll->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ nlohmann::json schema_change = R"({+ "fields": [+ {"name": "embedding", "type":"float[]", "embed":{"from": ["title"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ auto schema_change_op = coll->alter(schema_change);+ ASSERT_TRUE(schema_change_op.ok());++ auto search_res = coll->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);++ ASSERT_EQ(1, search_res.get()["found"].get<size_t>());+ ASSERT_EQ(384, search_res.get()["hits"][0]["document"]["embedding"].get<std::vector<float>>().size());+}