Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review Changes III

ozanarmagana50fe99a7c47
diff --git a/include/collection.h b/include/collection.h
index dce59d0b..8cc8b15a 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -162,7 +162,7 @@ private:
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
- void process_remove_field_for_embedding_fields(const field& the_field);
+ std::vector<field> process_remove_field_for_embedding_fields(const field& the_field);
void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
diff --git a/include/field.h b/include/field.h
index febb2a1e..d4361db9 100644
--- a/include/field.h
+++ b/include/field.h
@@ -450,7 +450,7 @@ struct field {
for(const auto& field : fields_json) {
if(field[fields::name] == embed_from_field) {
if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed_from + "` can only have string or string array fields.");
+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");
}
flag = true;
break;
@@ -460,7 +460,7 @@ struct field {
for(const auto& field : the_fields) {
if(field.name == embed_from_field) {
if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed_from + "` can only have used with string or string array fields.");
+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");
}
flag = true;
break;
@@ -468,7 +468,7 @@ struct field {
}
}
if(!flag) {
- return Option<bool>(400, "Property `" + fields::embed_from + "` can only be used with string or string array fields.");
+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");
}
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 676628d2..457e155d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3736,7 +3736,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
LOG(INFO) << "Finished altering " << num_found_docs << " document(s).";
-
+ std::vector<field> garbage_embedding_fields_vec;
for(auto& del_field: del_fields) {
search_schema.erase(del_field.name);
auto new_end = std::remove_if(fields.begin(), fields.end(), [&del_field](const field& f) {
@@ -3765,10 +3765,13 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
default_sorting_field = "";
}
- process_remove_field_for_embedding_fields(del_field);
+ auto garbage_embedding_fields = process_remove_field_for_embedding_fields(del_field);
+ garbage_embedding_fields_vec.insert(garbage_embedding_fields_vec.end(), garbage_embedding_fields.begin(),
+ garbage_embedding_fields.end());
}
index->refresh_schemas({}, del_fields);
+ index->refresh_schemas({}, garbage_embedding_fields_vec);
auto persist_op = persist_collection_meta();
if(!persist_op.ok()) {
@@ -4741,26 +4744,25 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_ha
return populate_include_exclude_fields(include_fields, exclude_fields, include_fields_full, exclude_fields_full);
}
-
-void Collection::process_remove_field_for_embedding_fields(const field& the_field) {
- std::vector<std::vector<field>::iterator> empty_fields;
- for(auto& embedding_field : embedding_fields) {
- const auto& actual_field = std::find_if(fields.begin(), fields.end(), [&embedding_field] (field other_field) {
- return other_field.name == embedding_field.name;
- });
- actual_field->embed_from.erase(std::remove_if(actual_field->embed_from.begin(), actual_field->embed_from.end(), [&the_field](std::string field_name) {
+// Removes the dropped field from embed_from of all embedding fields.
+std::vector<field> Collection::process_remove_field_for_embedding_fields(const field& the_field) {
+ std::vector<field> garbage_fields;
+ for(auto& field : fields) {
+ if(field.embed_from.empty()) {
+ continue;
+ }
+ field.embed_from.erase(std::remove_if(field.embed_from.begin(), field.embed_from.end(), [&the_field](std::string field_name) {
return the_field.name == field_name;
}));
- embedding_field = *actual_field;
- // store to remove embedding field if it has no field names in 'embed_from' anymore.
- if(embedding_field.embed_from.empty()) {
- empty_fields.push_back(actual_field);
+ embedding_fields[field.name] = field;
+
+ // mark this embedding field as "garbage" if it has no more embed_from fields
+ if(field.embed_from.empty()) {
+ embedding_fields.erase(field.name);
+ garbage_fields.push_back(field);
}
}
- for(const auto& empty_field : empty_fields) {
- search_schema.erase(empty_field->name);
- embedding_fields.erase(empty_field->name);
- fields.erase(empty_field);
- }
+ // return garbage embedding fields
+ return garbage_fields;
}
\ No newline at end of file
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 9269c858..3acc794c 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1592,7 +1592,7 @@ TEST_F(CollectionAllFieldsTest, FieldNameMatchingRegexpShouldNotBeIndexedInNonAu
ASSERT_EQ(1, results["hits"].size());
}
-TEST_F(CollectionAllFieldsTest, CreateFromFieldJSONInvalidField) {
+TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {
TextEmbedderManager::model_dir = "/tmp/models";
nlohmann::json field_json;
field_json["name"] = "embedding";
@@ -1607,10 +1607,10 @@ TEST_F(CollectionAllFieldsTest, CreateFromFieldJSONInvalidField) {
auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
ASSERT_FALSE(field_op.ok());
- ASSERT_EQ("Property `embed_from` can only be used with string or string array fields.", field_op.error());
+ ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());
}
-TEST_F(CollectionAllFieldsTest, CreateFromFieldNoModelDir) {
+TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {
TextEmbedderManager::model_dir = std::string();
nlohmann::json field_json;
field_json["name"] = "embedding";
@@ -1628,7 +1628,7 @@ TEST_F(CollectionAllFieldsTest, CreateFromFieldNoModelDir) {
ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());
}
-TEST_F(CollectionAllFieldsTest, CreateFromNotArray) {
+TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {
TextEmbedderManager::model_dir = "/tmp/models";
nlohmann::json field_json;
field_json["name"] = "embedding";
@@ -1646,7 +1646,7 @@ TEST_F(CollectionAllFieldsTest, CreateFromNotArray) {
ASSERT_EQ("Property `embed_from` must be an array.", field_op.error());
}
-TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {
+TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {
TextEmbedderManager::model_dir = "/tmp/models";
nlohmann::json field_json;
field_json["name"] = "embedding";
@@ -1664,7 +1664,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {
}
-TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {
+TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
TextEmbedderManager::download_default_model();
@@ -1690,3 +1690,21 @@ TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {
}
+TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {
+ TextEmbedderManager::model_dir = "/tmp/models";
+ nlohmann::json field_json;
+ field_json["name"] = "embedding";
+ field_json["type"] = "float[]";
+ field_json["embed_from"] = {"age"};
+
+ std::vector<field> fields;
+ std::string fallback_field_type;
+ auto arr = nlohmann::json::array();
+ arr.push_back(field_json);
+ field_json["name"] = "age";
+ field_json["type"] = "int32";
+ arr.push_back(field_json);
+ auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
+ ASSERT_FALSE(field_op.ok());
+ ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());
+}
\ No newline at end of file
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 82ccd65a..1a301446 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1497,21 +1497,29 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
]
})"_json;
- LOG(INFO) << "Dropping field";
auto embedding_fields = coll->get_embedding_fields();
ASSERT_EQ(2, embedding_fields["embedding"].embed_from.size());
- LOG(INFO) << "Before alter";
-
auto alter_op = coll->alter(schema_changes);
ASSERT_TRUE(alter_op.ok());
- LOG(INFO) << "After alter";
-
embedding_fields = coll->get_embedding_fields();
ASSERT_EQ(1, embedding_fields["embedding"].embed_from.size());
ASSERT_EQ("category", embedding_fields["embedding"].embed_from[0]);
+
+ schema_changes = R"({
+ "fields": [
+ {"name": "category", "drop": true}
+ ]
+ })"_json;
+
+ alter_op = coll->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ embedding_fields = coll->get_embedding_fields();
+ ASSERT_EQ(0, embedding_fields.size());
+ ASSERT_EQ(0, coll->_get_index()->_get_vector_index().size());
}
TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 2a15e531..2bd77929 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4666,7 +4666,6 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {
object["name"] = "apple";
auto add_op = coll->add(object.dump());
ASSERT_TRUE(add_op.ok());
- LOG(INFO) << "add_op.get(): " << add_op.get().dump();
ASSERT_EQ("apple", add_op.get()["name"]);
ASSERT_EQ(384, add_op.get()["embedding"].size());
@@ -4899,6 +4898,55 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {
ASSERT_EQ("Field `category` is needed to create embedding.", add_op.error());
}
+
+TEST_F(CollectionTest, WrongTypeForEmbedding) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed_from": ["category"]}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ TextEmbedderManager::download_default_model();
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ nlohmann::json doc;
+ doc["category"] = 1;
+
+ auto add_op = validator_t::validate_embed_fields(doc, coll->get_embedding_fields(), coll->get_schema(), true);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `category` has malformed data.", add_op.error());
+}
+
+TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "category", "type": "string[]"},
+ {"name": "embedding", "type":"float[]", "embed_from": ["category"]}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ TextEmbedderManager::download_default_model();
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ nlohmann::json doc;
+ doc["category"].push_back(33);
+
+ auto add_op = validator_t::validate_embed_fields(doc, coll->get_embedding_fields(), coll->get_schema(), true);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `category` has malformed data.", add_op.error());
+}
+
TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
nlohmann::json schema = R"({
"name": "objects",
@@ -4944,4 +4992,4 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
// check if the embedding field is updated
ASSERT_NE(embedding_field, updated_embedding_field);
-}
\ No newline at end of file
+}
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index e1dc5d10..c78e004e 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -775,5 +775,38 @@ TEST_F(CollectionVectorTest, DistanceThresholdTest) {
ASSERT_FLOAT_EQ(0.7, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[1]);
ASSERT_FLOAT_EQ(0.8, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[2]);
+}
+
+TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ TextEmbedderManager::download_default_model();
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ auto& vec_index = coll->_get_index()->_get_vector_index();
+ ASSERT_EQ(1, vec_index.size());
+ ASSERT_EQ(1, vec_index.count("embedding"));
+
+
+ nlohmann::json schema_change = R"({
+ "fields": [
+ {"name": "embedding", "drop": true}
+ ]
+ })"_json;
+
+ auto schema_change_op = coll->alter(schema_change);
+ ASSERT_TRUE(schema_change_op.ok());
+ ASSERT_EQ(0, vec_index.size());
+ ASSERT_EQ(0, vec_index.count("embedding"));
}
\ No newline at end of file