Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add support for optional embedding field

ozanarmagan94d7b54b8b33
diff --git a/src/validator.cpp b/src/validator.cpp
index 58a443fe..5fe9bab3 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -669,7 +669,9 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
const tsl::htrie_map<char, field> & search_schema,
const bool& error_if_field_not_found) {
for(const auto& field : embedding_fields) {
- auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
+ const auto& embed_from = field.embed[fields::from].get<std::vector<std::string>>();
+ // flag to check if all fields to embed from are optional and null
+ bool all_optional_and_null = true;
for(const auto& field_name : embed_from) {
auto schema_field_it = search_schema.find(field_name);
auto doc_field_it = document.find(field_name);
@@ -677,12 +679,13 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
return Option<bool>(400, "Field `" + field.name + "` has invalid fields to create embeddings from.");
}
if(doc_field_it == document.end()) {
- if(error_if_field_not_found && schema_field_it->optional == false) {
+ if(error_if_field_not_found && !schema_field_it->optional) {
return Option<bool>(400, "Field `" + field_name + "` is needed to create embedding.");
} else {
continue;
}
}
+ all_optional_and_null = false;
if((schema_field_it.value().type == field_types::STRING && !doc_field_it.value().is_string()) ||
(schema_field_it.value().type == field_types::STRING_ARRAY && !doc_field_it.value().is_array())) {
return Option<bool>(400, "Field `" + field_name + "` has malformed data.");
@@ -695,6 +698,9 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
}
}
}
+ if(all_optional_and_null && !field.optional) {
+ return Option<bool>(400, "No valid fields found to create embedding for `" + field.name + "`, please provide at least one valid field or make the embedding field optional.");
+ }
}
return Option<bool>(true);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index bb1faa88..8211f962 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4870,8 +4870,7 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {
doc["names"].push_back("butterball");
auto add_op = coll->add(doc.dump());
- ASSERT_FALSE(add_op.ok());
- ASSERT_EQ("Field `category` is needed to create embedding.", add_op.error());
+ ASSERT_TRUE(add_op.ok());
}
TEST_F(CollectionTest, WrongTypeInEmbedFrom) {
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index b726a06a..7f1e7eef 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -987,4 +987,52 @@ TEST_F(CollectionVectorTest, HybridSearchSortByGeopoint) {
ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());
ASSERT_EQ("butterball", search_res["hits"][1]["document"]["name"].get<std::string>());
ASSERT_EQ("butterfly", search_res["hits"][2]["document"]["name"].get<std::string>());
+}
+
+
+TEST_F(CollectionVectorTest, EmbedFromOptionalNullField) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "text", "type": "string", "optional": true},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["text"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(op.ok());
+ auto coll = op.get();
+
+ nlohmann::json doc = R"({
+ })"_json;
+
+ auto add_op = coll->add(doc.dump());
+
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("No valid fields found to create embedding for `embedding`, please provide at least one valid field or make the embedding field optional.", add_op.error());
+
+ doc["text"] = "butter";
+ add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ // drop the embedding field and reindex
+
+ nlohmann::json alter_schema = R"({
+ "fields": [
+ {"name": "embedding", "drop": true},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["text"], "model_config": {"model_name": "ts/e5-small"}}, "optional": true}
+ ]
+ })"_json;
+
+ auto update_op = coll->alter(alter_schema);
+ ASSERT_TRUE(update_op.ok());
+
+
+ doc = R"({
+ })"_json;
+ add_op = coll->add(doc.dump());
+
+ ASSERT_TRUE(add_op.ok());
}
\ No newline at end of file