Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ignore null optional fields while generating embedding

ozanarmaganabff0e0cb67f
diff --git a/src/index.cpp b/src/index.cpp
index a8008a2d..dd1f5335 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -6495,6 +6495,9 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,
auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
for(const auto& field_name : embed_from) {
auto field_it = search_schema.find(field_name);
+ if(document->count(field_name) == 0) {
+ continue;
+ }
if(field_it.value().type == field_types::STRING) {
text += (*document)[field_name].get<std::string>() + " ";
} else if(field_it.value().type == field_types::STRING_ARRAY) {
@@ -6511,7 +6514,7 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,
if(texts_to_embed.empty()) {
continue;
}
-
+
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
auto embedder_op = embedder_manager.get_text_embedder(field.embed[fields::model_config]);
diff --git a/src/validator.cpp b/src/validator.cpp
index b8516130..58a443fe 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -677,7 +677,7 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
return Option<bool>(400, "Field `" + field.name + "` has invalid fields to create embeddings from.");
}
if(doc_field_it == document.end()) {
- if(error_if_field_not_found) {
+ if(error_if_field_not_found && schema_field_it->optional == false) {
return Option<bool>(400, "Field `" + field_name + "` is needed to create embedding.");
} else {
continue;