Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review Changes III
ozanarmagana50fe99a7c47
diff --git a/include/collection.h b/include/collection.hindex dce59d0b..8cc8b15a 100644--- a/include/collection.h+++ b/include/collection.h@@ -162,7 +162,7 @@ private:void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);- void process_remove_field_for_embedding_fields(const field& the_field);+ std::vector<field> process_remove_field_for_embedding_fields(const field& the_field);void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,const std::map<size_t, std::vector<std::string>>& pinned_hits,diff --git a/include/field.h b/include/field.hindex febb2a1e..d4361db9 100644--- a/include/field.h+++ b/include/field.h@@ -450,7 +450,7 @@ struct field {for(const auto& field : fields_json) {if(field[fields::name] == embed_from_field) {if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed_from + "` can only have string or string array fields.");+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");}flag = true;break;@@ -460,7 +460,7 @@ struct field {for(const auto& field : the_fields) {if(field.name == embed_from_field) {if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed_from + "` can only have used with string or string array fields.");+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");}flag = true;break;@@ -468,7 +468,7 @@ struct field {}}if(!flag) {- return Option<bool>(400, "Property `" + fields::embed_from + "` can only be used with string or string array fields.");+ return Option<bool>(400, "Property `" + fields::embed_from + "` can only refer to string or string array fields.");}}}diff --git a/src/collection.cpp b/src/collection.cppindex 676628d2..457e155d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3736,7 +3736,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}LOG(INFO) << "Finished altering " << num_found_docs << " document(s).";-+ std::vector<field> garbage_embedding_fields_vec;for(auto& del_field: del_fields) {search_schema.erase(del_field.name);auto new_end = std::remove_if(fields.begin(), fields.end(), [&del_field](const field& f) {@@ -3765,10 +3765,13 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsdefault_sorting_field = "";}- process_remove_field_for_embedding_fields(del_field);+ auto garbage_embedding_fields = process_remove_field_for_embedding_fields(del_field);+ garbage_embedding_fields_vec.insert(garbage_embedding_fields_vec.end(), garbage_embedding_fields.begin(),+ garbage_embedding_fields.end());}index->refresh_schemas({}, del_fields);+ index->refresh_schemas({}, garbage_embedding_fields_vec);auto persist_op = persist_collection_meta();if(!persist_op.ok()) {@@ -4741,26 +4744,25 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_hareturn populate_include_exclude_fields(include_fields, exclude_fields, include_fields_full, exclude_fields_full);}--void Collection::process_remove_field_for_embedding_fields(const field& the_field) {- std::vector<std::vector<field>::iterator> empty_fields;- for(auto& embedding_field : embedding_fields) {- const auto& actual_field = std::find_if(fields.begin(), fields.end(), [&embedding_field] (field other_field) {- return other_field.name == embedding_field.name;- });- actual_field->embed_from.erase(std::remove_if(actual_field->embed_from.begin(), actual_field->embed_from.end(), [&the_field](std::string field_name) {+// Removes the dropped field from embed_from of all embedding fields.+std::vector<field> Collection::process_remove_field_for_embedding_fields(const field& the_field) {+ std::vector<field> garbage_fields;+ for(auto& field : fields) {+ if(field.embed_from.empty()) {+ continue;+ }+ field.embed_from.erase(std::remove_if(field.embed_from.begin(), field.embed_from.end(), [&the_field](std::string field_name) {return the_field.name == field_name;}));- embedding_field = *actual_field;- // store to remove embedding field if it has no field names in 'embed_from' anymore.- if(embedding_field.embed_from.empty()) {- empty_fields.push_back(actual_field);+ embedding_fields[field.name] = field;++ // mark this embedding field as "garbage" if it has no more embed_from fields+ if(field.embed_from.empty()) {+ embedding_fields.erase(field.name);+ garbage_fields.push_back(field);}}- for(const auto& empty_field : empty_fields) {- search_schema.erase(empty_field->name);- embedding_fields.erase(empty_field->name);- fields.erase(empty_field);- }+ // return garbage embedding fields+ return garbage_fields;}\ No newline at end of filediff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 9269c858..3acc794c 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1592,7 +1592,7 @@ TEST_F(CollectionAllFieldsTest, FieldNameMatchingRegexpShouldNotBeIndexedInNonAuASSERT_EQ(1, results["hits"].size());}-TEST_F(CollectionAllFieldsTest, CreateFromFieldJSONInvalidField) {+TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {TextEmbedderManager::model_dir = "/tmp/models";nlohmann::json field_json;field_json["name"] = "embedding";@@ -1607,10 +1607,10 @@ TEST_F(CollectionAllFieldsTest, CreateFromFieldJSONInvalidField) {auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);ASSERT_FALSE(field_op.ok());- ASSERT_EQ("Property `embed_from` can only be used with string or string array fields.", field_op.error());+ ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());}-TEST_F(CollectionAllFieldsTest, CreateFromFieldNoModelDir) {+TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {TextEmbedderManager::model_dir = std::string();nlohmann::json field_json;field_json["name"] = "embedding";@@ -1628,7 +1628,7 @@ TEST_F(CollectionAllFieldsTest, CreateFromFieldNoModelDir) {ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());}-TEST_F(CollectionAllFieldsTest, CreateFromNotArray) {+TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {TextEmbedderManager::model_dir = "/tmp/models";nlohmann::json field_json;field_json["name"] = "embedding";@@ -1646,7 +1646,7 @@ TEST_F(CollectionAllFieldsTest, CreateFromNotArray) {ASSERT_EQ("Property `embed_from` must be an array.", field_op.error());}-TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {+TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {TextEmbedderManager::model_dir = "/tmp/models";nlohmann::json field_json;field_json["name"] = "embedding";@@ -1664,7 +1664,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {}-TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {+TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {TextEmbedderManager::model_dir = "/tmp/typesense_test/models";TextEmbedderManager::download_default_model();@@ -1690,3 +1690,21 @@ TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {}+TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {+ TextEmbedderManager::model_dir = "/tmp/models";+ nlohmann::json field_json;+ field_json["name"] = "embedding";+ field_json["type"] = "float[]";+ field_json["embed_from"] = {"age"};++ std::vector<field> fields;+ std::string fallback_field_type;+ auto arr = nlohmann::json::array();+ arr.push_back(field_json);+ field_json["name"] = "age";+ field_json["type"] = "int32";+ arr.push_back(field_json);+ auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);+ ASSERT_FALSE(field_op.ok());+ ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());+}\ No newline at end of filediff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 82ccd65a..1a301446 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1497,21 +1497,29 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {]})"_json;- LOG(INFO) << "Dropping field";auto embedding_fields = coll->get_embedding_fields();ASSERT_EQ(2, embedding_fields["embedding"].embed_from.size());- LOG(INFO) << "Before alter";-auto alter_op = coll->alter(schema_changes);ASSERT_TRUE(alter_op.ok());- LOG(INFO) << "After alter";-embedding_fields = coll->get_embedding_fields();ASSERT_EQ(1, embedding_fields["embedding"].embed_from.size());ASSERT_EQ("category", embedding_fields["embedding"].embed_from[0]);++ schema_changes = R"({+ "fields": [+ {"name": "category", "drop": true}+ ]+ })"_json;++ alter_op = coll->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ embedding_fields = coll->get_embedding_fields();+ ASSERT_EQ(0, embedding_fields.size());+ ASSERT_EQ(0, coll->_get_index()->_get_vector_index().size());}TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 2a15e531..2bd77929 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4666,7 +4666,6 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {object["name"] = "apple";auto add_op = coll->add(object.dump());ASSERT_TRUE(add_op.ok());- LOG(INFO) << "add_op.get(): " << add_op.get().dump();ASSERT_EQ("apple", add_op.get()["name"]);ASSERT_EQ(384, add_op.get()["embedding"].size());@@ -4899,6 +4898,55 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {ASSERT_EQ("Field `category` is needed to create embedding.", add_op.error());}++TEST_F(CollectionTest, WrongTypeForEmbedding) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed_from": ["category"]}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ TextEmbedderManager::download_default_model();++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ nlohmann::json doc;+ doc["category"] = 1;++ auto add_op = validator_t::validate_embed_fields(doc, coll->get_embedding_fields(), coll->get_schema(), true);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `category` has malformed data.", add_op.error());+}++TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "category", "type": "string[]"},+ {"name": "embedding", "type":"float[]", "embed_from": ["category"]}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ TextEmbedderManager::download_default_model();++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ nlohmann::json doc;+ doc["category"].push_back(33);++ auto add_op = validator_t::validate_embed_fields(doc, coll->get_embedding_fields(), coll->get_schema(), true);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `category` has malformed data.", add_op.error());+}+TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {nlohmann::json schema = R"({"name": "objects",@@ -4944,4 +4992,4 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {// check if the embedding field is updatedASSERT_NE(embedding_field, updated_embedding_field);-}\ No newline at end of file+}diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex e1dc5d10..c78e004e 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -775,5 +775,38 @@ TEST_F(CollectionVectorTest, DistanceThresholdTest) {ASSERT_FLOAT_EQ(0.7, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[1]);ASSERT_FLOAT_EQ(0.8, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[2]);+}++TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ TextEmbedderManager::download_default_model();++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ auto& vec_index = coll->_get_index()->_get_vector_index();+ ASSERT_EQ(1, vec_index.size());+ ASSERT_EQ(1, vec_index.count("embedding"));+++ nlohmann::json schema_change = R"({+ "fields": [+ {"name": "embedding", "drop": true}+ ]+ })"_json;++ auto schema_change_op = coll->alter(schema_change);+ ASSERT_TRUE(schema_change_op.ok());+ ASSERT_EQ(0, vec_index.size());+ ASSERT_EQ(0, vec_index.count("embedding"));}\ No newline at end of file