Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review Changes III

ozanarmaganfb8c507e18a6
diff --git a/include/field.h b/include/field.h
index ba70be31..33672180 100644
--- a/include/field.h
+++ b/include/field.h
@@ -86,8 +86,7 @@ struct field {
int nested_array;
size_t num_dim;
- std::vector<std::string> embed_from;
- nlohmann::json model_config;
+ nlohmann::json embed;
vector_distance_type_t vec_dist;
static constexpr int VAL_UNKNOWN = 2;
@@ -98,10 +97,9 @@ struct field {
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,
- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {},
- const nlohmann::json& model_config = nlohmann::json()) :
+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const nlohmann::json& embed = nlohmann::json()) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),
- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_config(model_config) {
+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed(embed) {
set_computed_defaults(sort, infix);
}
@@ -315,6 +313,10 @@ struct field {
field_val[fields::infix] = field.infix;
field_val[fields::locale] = field.locale;
+
+ if(field.embed.count(fields::from) != 0) {
+ field_val[fields::embed] = field.embed;
+ }
field_val[fields::nested] = field.nested;
if(field.nested) {
@@ -329,11 +331,7 @@ struct field {
if (!field.reference.empty()) {
field_val[fields::reference] = field.reference;
}
- if(!field.embed_from.empty()) {
- field_val[fields::embed] = nlohmann::json::object();
- field_val[fields::embed][fields::from] = field.embed_from;
- field_val[fields::embed][fields::model_config] = field.model_config;
- }
+
fields_json.push_back(field_val);
if(!field.has_valid_type()) {
diff --git a/src/collection.cpp b/src/collection.cpp
index a51a8fff..fe1cbb63 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -52,7 +52,7 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, co
index(init_index()) {
for (auto const& field: fields) {
- if (!field.embed_from.empty()) {
+ if (field.embed.count(fields::from) != 0) {
embedding_fields.emplace(field.name, field);
}
}
@@ -239,20 +239,20 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::sort] = coll_field.sort;
field_json[fields::infix] = coll_field.infix;
field_json[fields::locale] = coll_field.locale;
- field_json[fields::embed] = nlohmann::json::object();
-
- if(!coll_field.embed_from.empty()) {
- field_json[fields::embed][fields::from] = coll_field.embed_from;
- }
-
- if(coll_field.model_config.size() > 0) {
- field_json[fields::embed][fields::model_config] = coll_field.model_config;
- // Hide OpenAI API key from the response.
- if(field_json[fields::embed][fields::model_config].count(fields::api_key) != 0) {
- field_json[fields::embed][fields::model_config][fields::api_key] = "<hidden>";
+ if(coll_field.embed.count(fields::from) != 0) {
+ field_json[fields::embed] = coll_field.embed;
+
+ if(field_json[fields::embed].count(fields::api_key) != 0) {
+ // hide api key with * except first 3 chars
+ std::string api_key = field_json[fields::embed][fields::api_key];
+ if(api_key.size() > 3) {
+ field_json[fields::embed][fields::api_key] = api_key.replace(3, api_key.size() - 3, api_key.size() - 3, '*');
+ } else {
+ field_json[fields::embed][fields::api_key] = api_key.replace(0, api_key.size(), api_key.size(), '*');
+ }
}
}
-
+
if(coll_field.num_dim > 0) {
field_json[fields::num_dim] = coll_field.num_dim;
}
@@ -1001,7 +1001,7 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {
bool exact_key_match = (kv.key().size() == field_name.size());
bool exact_primitive_match = exact_key_match && !kv.value().is_object();
- bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && !kv.value().embed_from.empty();
+ bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().embed.count(fields::from) != 0;
if(extract_only_string_fields && !kv.value().is_string() && !text_embedding) {
if(exact_primitive_match && !is_wildcard) {
@@ -1189,9 +1189,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
- auto embedder = embedder_manager.get_text_embedder(search_field.model_config);
+ auto embedder = embedder_manager.get_text_embedder(search_field.embed[fields::model_config]);
- std::string embed_query = embedder_manager.get_query_prefix(search_field.model_config) + raw_query;
+ std::string embed_query = embedder_manager.get_query_prefix(search_field.embed[fields::model_config]) + raw_query;
auto embedding_op = embedder->Embed(embed_query);
if(!embedding_op.ok()) {
return Option<nlohmann::json>(400, embedding_op.error());
@@ -3763,7 +3763,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
nested_fields.erase(del_field.name);
}
- if(!del_field.embed_from.empty()) {
+ if(del_field.embed.count(fields::from) != 0) {
embedding_fields.erase(del_field.name);
}
@@ -4062,7 +4062,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
return Option<bool>(400, "Field `" + field_name + "` is not part of collection schema.");
}
- if(found_field && !field_it.value().embed_from.empty()) {
+ if(found_field && field_it.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(field_it.key());
}
@@ -4071,7 +4071,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
updated_search_schema.erase(field_it.key());
updated_nested_fields.erase(field_it.key());
- if(!field_it.value().embed_from.empty()) {
+ if(field_it.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(field_it.key());
}
@@ -4085,7 +4085,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
updated_search_schema.erase(prefix_kv.key());
updated_nested_fields.erase(prefix_kv.key());
- if(!prefix_kv.value().embed_from.empty()) {
+ if(prefix_kv.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(prefix_kv.key());
}
}
@@ -4138,7 +4138,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
addition_fields.push_back(f);
}
- if(!f.embed_from.empty()) {
+ if(f.embed.count(fields::from) != 0) {
embedding_fields.emplace(f.name, f);
}
@@ -4153,7 +4153,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
updated_search_schema.emplace(prefix_kv.key(), prefix_kv.value());
updated_nested_fields.emplace(prefix_kv.key(), prefix_kv.value());
- if(!prefix_kv.value().embed_from.empty()) {
+ if(prefix_kv.value().embed.count(fields::from) != 0) {
embedding_fields.emplace(prefix_kv.key(), prefix_kv.value());
}
@@ -4478,7 +4478,7 @@ Index* Collection::init_index() {
nested_fields.emplace(field.name, field);
}
- if(!field.embed_from.empty()) {
+ if(field.embed.count(fields::from) != 0) {
embedding_fields.emplace(field.name, field);
}
@@ -4755,19 +4755,23 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_ha
// Removes the dropped field from embed_from of all embedding fields.
void Collection::process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields) {
for(auto& field : fields) {
- if(field.embed_from.empty()) {
+ if(field.embed.count(fields::from) == 0) {
continue;
}
- field.embed_from.erase(std::remove_if(field.embed_from.begin(), field.embed_from.end(), [&the_field](std::string field_name) {
+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
+ embed_from.erase(std::remove_if(embed_from.begin(), embed_from.end(), [&the_field](std::string field_name) {
return the_field.name == field_name;
}));
+ field.embed[fields::from] = std::move(embed_from);
embedding_fields[field.name] = field;
// mark this embedding field as "garbage" if it has no more embed_from fields
- if(field.embed_from.empty()) {
+ if(embed_from.empty()) {
embedding_fields.erase(field.name);
garbage_fields.push_back(field);
}
+
+
}
}
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 220940a8..8ed9eb40 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -58,8 +58,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::reference] = "";
}
- if(field_obj.count(fields::embed_from) == 0) {
- field_obj[fields::embed_from] = std::vector<std::string>();
+ if(field_obj.count(fields::embed) == 0) {
+ field_obj[fields::embed] = nlohmann::json::object();
}
if(field_obj.count(fields::model_config) == 0) {
@@ -77,8 +77,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],
- field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed_from],
- field_obj[fields::model_config]);
+ field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed]);
// value of `sort` depends on field type
if(field_obj.count(fields::sort) == 0) {
diff --git a/src/field.cpp b/src/field.cpp
index 80e52ace..12183c9c 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -672,16 +672,16 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
- if(field_json.count(fields::model_config) > 0 && field_json.count(fields::embed_from) == 0) {
- return Option<bool>(400, "Property `" + fields::model_config + "` can only be used with `" + fields::embed_from + "`.");
- }
-
if(field_json.count(fields::embed) != 0) {
// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)
field_json[fields::num_dim] = static_cast<unsigned int>(384);
auto& embed_json = field_json[fields::embed];
+ if(embed_json.count(fields::from) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` not found.");
+ }
+
if(embed_json.count(fields::model_config) == 0) {
return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");
}
@@ -725,7 +725,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::num_dim] = num_dim;
} else {
field_json[fields::embed] = nlohmann::json::object();
- field_json[fields::embed][fields::from] = nlohmann::json::array();
}
auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);
@@ -808,8 +807,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,
- field_json[fields::reference], field_json[fields::embed][fields::from].get<std::vector<std::string>>(),
- field_json[fields::embed][fields::model_config])
+ field_json[fields::reference], field_json[fields::embed])
);
if (!field_json[fields::reference].get<std::string>().empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index 6c5c73f2..013ab0b9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -6348,9 +6348,11 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
const tsl::htrie_map<char, field> & search_schema) {
for(const auto& field : embedding_fields) {
std::vector<std::string> text_to_embed;
+ auto indexing_prefix = TextEmbedderManager::get_instance().get_indexing_prefix(field.embed[fields::model_config]);
for(auto& document : documents) {
- std::string text = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_config);
- for(const auto& field_name : field.embed_from) {
+ std::string text = indexing_prefix;
+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
+ for(const auto& field_name : embed_from) {
auto field_it = search_schema.find(field_name);
if(field_it.value().type == field_types::STRING) {
text += (*document)[field_name].get<std::string>() + " ";
@@ -6363,7 +6365,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
text_to_embed.push_back(text);
}
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
- auto embedder = embedder_manager.get_text_embedder(field.model_config);
+ auto embedder = embedder_manager.get_text_embedder(field.embed[fields::model_config]);
auto embedding_op = embedder->batch_embed(text_to_embed);
if(!embedding_op.ok()) {
diff --git a/src/validator.cpp b/src/validator.cpp
index cb36571f..79488321 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -595,7 +595,8 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
const tsl::htrie_map<char, field> & search_schema,
const bool& error_if_field_not_found) {
for(const auto& field : embedding_fields) {
- for(const auto& field_name : field.embed_from) {
+ auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
+ for(const auto& field_name : embed_from) {
auto schema_field_it = search_schema.find(field_name);
auto doc_field_it = document.find(field_name);
if(schema_field_it == search_schema.end()) {
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 8e29787e..c3817ad5 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -63,8 +63,6 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
while (std::getline(infile, json_line)) {
nlohmann::json document = nlohmann::json::parse(json_line);
Option<nlohmann::json> add_op = coll1->add(document.dump());
-
- LOG(INFO) << "Add op: " << add_op.error();
ASSERT_TRUE(add_op.ok());
}
@@ -1658,8 +1656,8 @@ TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
- embedding.embed_from.push_back("name");
- embedding.model_config["model_name"] = "ts/e5-small";
+ embedding.embed["from"].push_back("name");
+ embedding.embed["model_config"]["model_name"] = "ts/e5-small";
std::vector<field> fields = {field("name", field_types::STRING, false),
embedding};
auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 272c82cd..7b486a03 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1509,14 +1509,14 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
auto embedding_fields = coll->get_embedding_fields();
- ASSERT_EQ(2, embedding_fields["embedding"].embed_from.size());
+ ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());
auto alter_op = coll->alter(schema_changes);
ASSERT_TRUE(alter_op.ok());
embedding_fields = coll->get_embedding_fields();
- ASSERT_EQ(1, embedding_fields["embedding"].embed_from.size());
- ASSERT_EQ("category", embedding_fields["embedding"].embed_from[0]);
+ ASSERT_EQ(1, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());
+ ASSERT_EQ("category", embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>()[0]);
schema_changes = R"({
"fields": [
@@ -1552,8 +1552,8 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {
auto embedding_field_it = embedding_fields_map.find("embedding");
ASSERT_TRUE(embedding_field_it != embedding_fields_map.end());
ASSERT_EQ("embedding", embedding_field_it.value().name);
- ASSERT_EQ(1, embedding_field_it.value().embed_from.size());
- ASSERT_EQ("name", embedding_field_it.value().embed_from[0]);
+ ASSERT_EQ(1, embedding_field_it.value().embed[fields::from].get<std::vector<std::string>>().size());
+ ASSERT_EQ("name", embedding_field_it.value().embed[fields::from].get<std::vector<std::string>>()[0]);
// drop the embedding field
nlohmann::json schema_without_embedding = R"({