Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix tokenizing XLM-RoBERTa models with 3 inputs and altering embedding fields
ozanarmagan29613ad05454
diff --git a/src/collection.cpp b/src/collection.cppindex ba00b5ab..6168c8f7 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4300,6 +4300,27 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,auto validation_op = field::validate_and_init_embed_fields(embed_json_field_indices, search_schema,schema_changes["fields"], diff_fields);++ for(auto index : embed_json_field_indices) {+ auto& field = diff_fields[index.second];+ auto is_reindex = (delete_field_names.count(field.name) != 0);+ if(is_reindex) {+ for(auto& reindex_field: reindex_fields) {+ if(reindex_field.name == field.name) {+ reindex_field.num_dim = field.num_dim;+ break;+ }+ }+ } else {+ for(auto& add_field: addition_fields) {+ if(add_field.name == field.name) {+ add_field.num_dim = field.num_dim;+ break;+ }+ }+ }+ }+if(!validation_op.ok()) {return validation_op;}diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 61244054..126655f9 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -117,6 +117,11 @@ embedding_res_t TextEmbedder::Embed(const std::string& text, const size_t remoteinput_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});if(session_->GetInputCount() == 3) {+ // edge case: xlm_roberta does not have token_type_ids, but if the model has it as input, we need to fill it with 0s+ if(encoded_input.token_type_ids.size() == 0) {+ encoded_input.token_type_ids.resize(encoded_input.input_ids.size(), 0);+ }+input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});}input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));