Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review Changes III
ozanarmaganfb8c507e18a6
diff --git a/include/field.h b/include/field.hindex ba70be31..33672180 100644--- a/include/field.h+++ b/include/field.h@@ -86,8 +86,7 @@ struct field {int nested_array;size_t num_dim;- std::vector<std::string> embed_from;- nlohmann::json model_config;+ nlohmann::json embed;vector_distance_type_t vec_dist;static constexpr int VAL_UNKNOWN = 2;@@ -98,10 +97,9 @@ struct field {field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {},- const nlohmann::json& model_config = nlohmann::json()) :+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const nlohmann::json& embed = nlohmann::json()) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_config(model_config) {+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed(embed) {set_computed_defaults(sort, infix);}@@ -315,6 +313,10 @@ struct field {field_val[fields::infix] = field.infix;field_val[fields::locale] = field.locale;++ if(field.embed.count(fields::from) != 0) {+ field_val[fields::embed] = field.embed;+ }field_val[fields::nested] = field.nested;if(field.nested) {@@ -329,11 +331,7 @@ struct field {if (!field.reference.empty()) {field_val[fields::reference] = field.reference;}- if(!field.embed_from.empty()) {- field_val[fields::embed] = nlohmann::json::object();- field_val[fields::embed][fields::from] = field.embed_from;- field_val[fields::embed][fields::model_config] = field.model_config;- }+fields_json.push_back(field_val);if(!field.has_valid_type()) {diff --git a/src/collection.cpp b/src/collection.cppindex a51a8fff..fe1cbb63 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -52,7 +52,7 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, coindex(init_index()) {for (auto const& field: fields) {- if (!field.embed_from.empty()) {+ if (field.embed.count(fields::from) != 0) {embedding_fields.emplace(field.name, field);}}@@ -239,20 +239,20 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::sort] = coll_field.sort;field_json[fields::infix] = coll_field.infix;field_json[fields::locale] = coll_field.locale;- field_json[fields::embed] = nlohmann::json::object();-- if(!coll_field.embed_from.empty()) {- field_json[fields::embed][fields::from] = coll_field.embed_from;- }-- if(coll_field.model_config.size() > 0) {- field_json[fields::embed][fields::model_config] = coll_field.model_config;- // Hide OpenAI API key from the response.- if(field_json[fields::embed][fields::model_config].count(fields::api_key) != 0) {- field_json[fields::embed][fields::model_config][fields::api_key] = "<hidden>";+ if(coll_field.embed.count(fields::from) != 0) {+ field_json[fields::embed] = coll_field.embed;++ if(field_json[fields::embed].count(fields::api_key) != 0) {+ // hide api key with * except first 3 chars+ std::string api_key = field_json[fields::embed][fields::api_key];+ if(api_key.size() > 3) {+ field_json[fields::embed][fields::api_key] = api_key.replace(3, api_key.size() - 3, api_key.size() - 3, '*');+ } else {+ field_json[fields::embed][fields::api_key] = api_key.replace(0, api_key.size(), api_key.size(), '*');+ }}}-+if(coll_field.num_dim > 0) {field_json[fields::num_dim] = coll_field.num_dim;}@@ -1001,7 +1001,7 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {bool exact_key_match = (kv.key().size() == field_name.size());bool exact_primitive_match = exact_key_match && !kv.value().is_object();- bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && !kv.value().embed_from.empty();+ bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().embed.count(fields::from) != 0;if(extract_only_string_fields && !kv.value().is_string() && !text_embedding) {if(exact_primitive_match && !is_wildcard) {@@ -1189,9 +1189,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(search_field.model_config);+ auto embedder = embedder_manager.get_text_embedder(search_field.embed[fields::model_config]);- std::string embed_query = embedder_manager.get_query_prefix(search_field.model_config) + raw_query;+ std::string embed_query = embedder_manager.get_query_prefix(search_field.embed[fields::model_config]) + raw_query;auto embedding_op = embedder->Embed(embed_query);if(!embedding_op.ok()) {return Option<nlohmann::json>(400, embedding_op.error());@@ -3763,7 +3763,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsnested_fields.erase(del_field.name);}- if(!del_field.embed_from.empty()) {+ if(del_field.embed.count(fields::from) != 0) {embedding_fields.erase(del_field.name);}@@ -4062,7 +4062,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,return Option<bool>(400, "Field `" + field_name + "` is not part of collection schema.");}- if(found_field && !field_it.value().embed_from.empty()) {+ if(found_field && field_it.value().embed.count(fields::from) != 0) {updated_embedding_fields.erase(field_it.key());}@@ -4071,7 +4071,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,updated_search_schema.erase(field_it.key());updated_nested_fields.erase(field_it.key());- if(!field_it.value().embed_from.empty()) {+ if(field_it.value().embed.count(fields::from) != 0) {updated_embedding_fields.erase(field_it.key());}@@ -4085,7 +4085,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,updated_search_schema.erase(prefix_kv.key());updated_nested_fields.erase(prefix_kv.key());- if(!prefix_kv.value().embed_from.empty()) {+ if(prefix_kv.value().embed.count(fields::from) != 0) {updated_embedding_fields.erase(prefix_kv.key());}}@@ -4138,7 +4138,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,addition_fields.push_back(f);}- if(!f.embed_from.empty()) {+ if(f.embed.count(fields::from) != 0) {embedding_fields.emplace(f.name, f);}@@ -4153,7 +4153,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,updated_search_schema.emplace(prefix_kv.key(), prefix_kv.value());updated_nested_fields.emplace(prefix_kv.key(), prefix_kv.value());- if(!prefix_kv.value().embed_from.empty()) {+ if(prefix_kv.value().embed.count(fields::from) != 0) {embedding_fields.emplace(prefix_kv.key(), prefix_kv.value());}@@ -4478,7 +4478,7 @@ Index* Collection::init_index() {nested_fields.emplace(field.name, field);}- if(!field.embed_from.empty()) {+ if(field.embed.count(fields::from) != 0) {embedding_fields.emplace(field.name, field);}@@ -4755,19 +4755,23 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_ha// Removes the dropped field from embed_from of all embedding fields.void Collection::process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields) {for(auto& field : fields) {- if(field.embed_from.empty()) {+ if(field.embed.count(fields::from) == 0) {continue;}- field.embed_from.erase(std::remove_if(field.embed_from.begin(), field.embed_from.end(), [&the_field](std::string field_name) {+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();+ embed_from.erase(std::remove_if(embed_from.begin(), embed_from.end(), [&the_field](std::string field_name) {return the_field.name == field_name;}));+ field.embed[fields::from] = std::move(embed_from);embedding_fields[field.name] = field;// mark this embedding field as "garbage" if it has no more embed_from fields- if(field.embed_from.empty()) {+ if(embed_from.empty()) {embedding_fields.erase(field.name);garbage_fields.push_back(field);}++}}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 220940a8..8ed9eb40 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -58,8 +58,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::reference] = "";}- if(field_obj.count(fields::embed_from) == 0) {- field_obj[fields::embed_from] = std::vector<std::string>();+ if(field_obj.count(fields::embed) == 0) {+ field_obj[fields::embed] = nlohmann::json::object();}if(field_obj.count(fields::model_config) == 0) {@@ -77,8 +77,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],- field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed_from],- field_obj[fields::model_config]);+ field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed]);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {diff --git a/src/field.cpp b/src/field.cppindex 80e52ace..12183c9c 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -672,16 +672,16 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}- if(field_json.count(fields::model_config) > 0 && field_json.count(fields::embed_from) == 0) {- return Option<bool>(400, "Property `" + fields::model_config + "` can only be used with `" + fields::embed_from + "`.");- }-if(field_json.count(fields::embed) != 0) {// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)field_json[fields::num_dim] = static_cast<unsigned int>(384);auto& embed_json = field_json[fields::embed];+ if(embed_json.count(fields::from) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` not found.");+ }+if(embed_json.count(fields::model_config) == 0) {return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");}@@ -725,7 +725,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::num_dim] = num_dim;} else {field_json[fields::embed] = nlohmann::json::object();- field_json[fields::embed][fields::from] = nlohmann::json::array();}auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);@@ -808,8 +807,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::optional], field_json[fields::index], field_json[fields::locale],field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,- field_json[fields::reference], field_json[fields::embed][fields::from].get<std::vector<std::string>>(),- field_json[fields::embed][fields::model_config])+ field_json[fields::reference], field_json[fields::embed]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/index.cpp b/src/index.cppindex 6c5c73f2..013ab0b9 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -6348,9 +6348,11 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,const tsl::htrie_map<char, field> & search_schema) {for(const auto& field : embedding_fields) {std::vector<std::string> text_to_embed;+ auto indexing_prefix = TextEmbedderManager::get_instance().get_indexing_prefix(field.embed[fields::model_config]);for(auto& document : documents) {- std::string text = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_config);- for(const auto& field_name : field.embed_from) {+ std::string text = indexing_prefix;+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();+ for(const auto& field_name : embed_from) {auto field_it = search_schema.find(field_name);if(field_it.value().type == field_types::STRING) {text += (*document)[field_name].get<std::string>() + " ";@@ -6363,7 +6365,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,text_to_embed.push_back(text);}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(field.model_config);+ auto embedder = embedder_manager.get_text_embedder(field.embed[fields::model_config]);auto embedding_op = embedder->batch_embed(text_to_embed);if(!embedding_op.ok()) {diff --git a/src/validator.cpp b/src/validator.cppindex cb36571f..79488321 100644--- a/src/validator.cpp+++ b/src/validator.cpp@@ -595,7 +595,8 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,const tsl::htrie_map<char, field> & search_schema,const bool& error_if_field_not_found) {for(const auto& field : embedding_fields) {- for(const auto& field_name : field.embed_from) {+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();+ for(const auto& field_name : embed_from) {auto schema_field_it = search_schema.find(field_name);auto doc_field_it = document.find(field_name);if(schema_field_it == search_schema.end()) {diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 8e29787e..c3817ad5 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -63,8 +63,6 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {while (std::getline(infile, json_line)) {nlohmann::json document = nlohmann::json::parse(json_line);Option<nlohmann::json> add_op = coll1->add(document.dump());-- LOG(INFO) << "Add op: " << add_op.error();ASSERT_TRUE(add_op.ok());}@@ -1658,8 +1656,8 @@ TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");field embedding = field("embedding", field_types::FLOAT_ARRAY, false);- embedding.embed_from.push_back("name");- embedding.model_config["model_name"] = "ts/e5-small";+ embedding.embed["from"].push_back("name");+ embedding.embed["model_config"]["model_name"] = "ts/e5-small";std::vector<field> fields = {field("name", field_types::STRING, false),embedding};auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 272c82cd..7b486a03 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1509,14 +1509,14 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {auto embedding_fields = coll->get_embedding_fields();- ASSERT_EQ(2, embedding_fields["embedding"].embed_from.size());+ ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());auto alter_op = coll->alter(schema_changes);ASSERT_TRUE(alter_op.ok());embedding_fields = coll->get_embedding_fields();- ASSERT_EQ(1, embedding_fields["embedding"].embed_from.size());- ASSERT_EQ("category", embedding_fields["embedding"].embed_from[0]);+ ASSERT_EQ(1, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());+ ASSERT_EQ("category", embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>()[0]);schema_changes = R"({"fields": [@@ -1552,8 +1552,8 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {auto embedding_field_it = embedding_fields_map.find("embedding");ASSERT_TRUE(embedding_field_it != embedding_fields_map.end());ASSERT_EQ("embedding", embedding_field_it.value().name);- ASSERT_EQ(1, embedding_field_it.value().embed_from.size());- ASSERT_EQ("name", embedding_field_it.value().embed_from[0]);+ ASSERT_EQ(1, embedding_field_it.value().embed[fields::from].get<std::vector<std::string>>().size());+ ASSERT_EQ("name", embedding_field_it.value().embed[fields::from].get<std::vector<std::string>>()[0]);// drop the embedding fieldnlohmann::json schema_without_embedding = R"({