Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Added support for XLMRoBERTa tokenizer
ozanarmagancfcf95d5ddf2
diff --git a/BUILD b/BUILDindex a2780d5d..da734dde 100644--- a/BUILD+++ b/BUILD@@ -52,6 +52,8 @@ cc_library("@rocksdb","@s2geometry","@hnsw",+ # "@zip",+ "@sentencepiece"],)diff --git a/WORKSPACE b/WORKSPACEindex 098015ff..acc498b7 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -63,6 +63,22 @@ new_git_repository(tag = "v1.2.12",)+# new_git_repository(+# name = "zip",+# build_file = "//bazel:zip.BUILD",+# branch = "master",+# remote = "https://github.com/kuba--/zip.git",+# )++new_git_repository(+ name = "sentencepiece",+ build_file = "//bazel:sentencepiece.BUILD",+ tag = "v0.1.98",+ remote = "https://github.com/google/sentencepiece",+ patches = ["//bazel:sentencepiece.patch"],+ patch_args = ["-p1"]+)+git_repository(name = "rules_perl",remote = "https://github.com/bazelbuild/rules_perl.git",diff --git a/bazel/sentencepiece.BUILD b/bazel/sentencepiece.BUILDnew file mode 100644index 00000000..75d0ba17--- /dev/null+++ b/bazel/sentencepiece.BUILD@@ -0,0 +1,31 @@+load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")++package(default_visibility = ["//visibility:public"])+++filegroup(+ name = "sentencepiece_src",+ srcs = glob(["**"]),+ visibility = ["//visibility:public"],+)++++cmake(+ name = "sentencepiece",+ lib_source = "//:sentencepiece_src",+ out_static_libs = ["libsentencepiece.a"],+ out_include_dir = "include",+ build_args = [+ "--config Release",+ "--target sentencepiece-static",+ ],+ install = False,+ cache_entries = {+ 'SPM_USE_BUILTIN_PROTOBUF': 'OFF',+ },+ postfix_script= """+echo "Intstalling sentencepiece"+cp $BUILD_TMPDIR/src/libsentencepiece.a $INSTALLDIR/lib/libsentencepiece.a+"""+)diff --git a/bazel/sentencepiece.patch b/bazel/sentencepiece.patchnew file mode 100644index 00000000..ba8e1152--- /dev/null+++ b/bazel/sentencepiece.patch@@ -0,0 +1,380 @@+diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h+index 14b1e8c..331c2a5 100644+--- a/src/sentencepiece_processor.h++++ b/src/sentencepiece_processor.h+@@ -22,12 +22,6 @@+ #include <utility>+ #include <vector>++-#ifndef SWIG+-namespace absl {+-using std::string_view;+-} // namespace absl+-#endif // SWIG+-+ namespace sentencepiece {+ namespace util {++@@ -55,7 +49,7 @@ class Status {+ public:+ Status();+ ~Status();+- Status(StatusCode code, absl::string_view error_message);++ Status(StatusCode code, std::string_view error_message);+ Status(const Status &s);+ void operator=(const Status &s);+ bool operator==(const Status &s) const;+@@ -241,11 +235,11 @@ class SentencePieceProcessor {++ // Loads model from `filename`.+ // Returns false if `filename` cannot be loaded.+- virtual util::Status Load(absl::string_view filename);++ virtual util::Status Load(std::string_view filename);++ // Loads model from `filename`.+ // Crash if `filename` cannot be loaded.+- virtual void LoadOrDie(absl::string_view filename);++ virtual void LoadOrDie(std::string_view filename);++ // Loads model from `model_proto`.+ // `model_proto` is copied.+@@ -257,16 +251,16 @@ class SentencePieceProcessor {++ // Loads model from `serialized`, which is a string-serialized model proto.+ // Useful to load the model from a platform independent blob object.+- virtual util::Status LoadFromSerializedProto(absl::string_view serialized);++ virtual util::Status LoadFromSerializedProto(std::string_view serialized);++ // Returns the status. Encode/Decode methods are valid when status is OK.+ virtual util::Status status() const;++ // Sets encode extra_option sequence.+- virtual util::Status SetEncodeExtraOptions(absl::string_view extra_option);++ virtual util::Status SetEncodeExtraOptions(std::string_view extra_option);++ // Sets decode extra_option sequence.+- virtual util::Status SetDecodeExtraOptions(absl::string_view extra_option);++ virtual util::Status SetDecodeExtraOptions(std::string_view extra_option);++ //////////////////////////////////////////////////////////////+ // Vocabulary restriction.+@@ -276,7 +270,7 @@ class SentencePieceProcessor {+ // Restricts the vocabulary set.+ // The input sentences are encoded into the tokens in `valid_vocab`.+ virtual util::Status SetVocabulary(+- const std::vector<absl::string_view> &valid_vocab);++ const std::vector<std::string_view> &valid_vocab);++ // Reverts the vocabulary restriction.+ virtual util::Status ResetVocabulary();+@@ -284,18 +278,18 @@ class SentencePieceProcessor {+ // Loads the valid vocabulary set from `filename` in TSV format.+ // Format: <token> <tab> <freq>.+ // Any token with frequency < threshold will be treated as OOV.+- virtual util::Status LoadVocabulary(absl::string_view filename,++ virtual util::Status LoadVocabulary(std::string_view filename,+ int threshold);++ //////////////////////////////////////////////////////////////+ // Simple Encode and Decode API.+ //+ // Given a UTF8 input, encodes it into a sequence of sentence pieces.+- virtual util::Status Encode(absl::string_view input,++ virtual util::Status Encode(std::string_view input,+ std::vector<std::string> *pieces) const;++ // Given a UTF8 input, encodes it into a sequence of ids.+- virtual util::Status Encode(absl::string_view input,++ virtual util::Status Encode(std::string_view input,+ std::vector<int> *ids) const;++ // Given a sequence of pieces, decodes it into a detokenized output.+@@ -303,7 +297,7 @@ class SentencePieceProcessor {+ std::string *detokenized) const;++ // Given a sequence of pieces, decodes it into a detokenized output.+- virtual util::Status Decode(const std::vector<absl::string_view> &pieces,++ virtual util::Status Decode(const std::vector<std::string_view> &pieces,+ std::string *detokenized) const;++ // Given a sequence of ids, decodes it into a detokenized output.+@@ -315,11 +309,11 @@ class SentencePieceProcessor {+ //+ // Same as Encode, but returns nbest results.+ virtual util::Status NBestEncode(+- absl::string_view input, int nbest_size,++ std::string_view input, int nbest_size,+ std::vector<std::vector<std::string>> *pieces) const;++ // Same as Encode, but returns nbest results.+- virtual util::Status NBestEncode(absl::string_view input, int nbest_size,++ virtual util::Status NBestEncode(std::string_view input, int nbest_size,+ std::vector<std::vector<int>> *ids) const;++ //////////////////////////////////////////////////////////////+@@ -342,12 +336,12 @@ class SentencePieceProcessor {+ // `alpha`: The dropout probability `p` of bpe merge operations in+ // https://arxiv.org/abs/1910.13267 Nbest-based sampling is not supported so+ // nbest_size parameter is ignored in BPE.+- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,+ float alpha,+ std::vector<std::string> *pieces) const;++ // Same as above, but returns a sequence of ids.+- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,+ float alpha, std::vector<int> *ids) const;++ //////////////////////////////////////////////////////////////+@@ -366,13 +360,13 @@ class SentencePieceProcessor {+ // included in the sample, and the remaining elements are sampled excluding+ // the best.+ virtual util::Status SampleEncodeAndScore(+- absl::string_view input, int num_samples, float alpha, bool wor,++ std::string_view input, int num_samples, float alpha, bool wor,+ bool include_best,+ std::vector<std::pair<std::vector<std::string>, float>> *pieces) const;++ // Same as above, but returns a sequence of ids.+ virtual util::Status SampleEncodeAndScore(+- absl::string_view input, int num_samples, float alpha, bool wor,++ std::string_view input, int num_samples, float alpha, bool wor,+ bool include_best,+ std::vector<std::pair<std::vector<int>, float>> *ids) const;++@@ -381,7 +375,7 @@ class SentencePieceProcessor {+ //+ // This only available in model_type=unigram.+ // Calculate entropy of possible tokenisations+- virtual util::Status CalculateEntropy(absl::string_view input, float alpha,++ virtual util::Status CalculateEntropy(std::string_view input, float alpha,+ float *entropy) const;++ //////////////////////////////////////////////////////////////+@@ -397,24 +391,24 @@ class SentencePieceProcessor {+ // ImmutableSentencePieceText spt;+ // Encode("hello", spt.mutable_proto()).IgnoreError();+ // std::cout << spt.pieces_size() << std::endl;+- virtual util::Status Encode(absl::string_view input,++ virtual util::Status Encode(std::string_view input,+ SentencePieceText *spt) const;++- virtual util::Status NBestEncode(absl::string_view input, int nbest_size,++ virtual util::Status NBestEncode(std::string_view input, int nbest_size,+ NBestSentencePieceText *nbest_spt) const;++- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,+ float alpha, SentencePieceText *spt) const;++ virtual util::Status SampleEncodeAndScore(+- absl::string_view input, int num_samples, float alpha, bool wor,++ std::string_view input, int num_samples, float alpha, bool wor,+ bool include_best, NBestSentencePieceText *samples_spt) const;++ // DEPRECATED: Remove this API and use std::vector<std::string_view>+ virtual util::Status Decode(const std::vector<std::string> &pieces,+ SentencePieceText *spt) const;++- virtual util::Status Decode(const std::vector<absl::string_view> &pieces,++ virtual util::Status Decode(const std::vector<std::string_view> &pieces,+ SentencePieceText *spt) const;++ virtual util::Status Decode(const std::vector<int> &ids,+@@ -450,34 +444,34 @@ class SentencePieceProcessor {+ // Handy methods that return the result directly.+ // These functions ignore internal errors.+ virtual std::vector<std::string> EncodeAsPieces(+- absl::string_view input) const {++ std::string_view input) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<std::string>, input);+ }++- virtual std::vector<int> EncodeAsIds(absl::string_view input) const {++ virtual std::vector<int> EncodeAsIds(std::string_view input) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<int>, input);+ }++ virtual std::vector<std::vector<std::string>> NBestEncodeAsPieces(+- absl::string_view input, int nbest_size) const {++ std::string_view input, int nbest_size) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(+ NBestEncode, std::vector<std::vector<std::string>>, input, nbest_size);+ }++ virtual std::vector<std::vector<int>> NBestEncodeAsIds(+- absl::string_view input, int nbest_size) const {++ std::string_view input, int nbest_size) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(NBestEncode, std::vector<std::vector<int>>,+ input, nbest_size);+ }++- virtual std::vector<std::string> SampleEncodeAsPieces(absl::string_view input,++ virtual std::vector<std::string> SampleEncodeAsPieces(std::string_view input,+ int nbest_size,+ float alpha) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<std::string>, input,+ nbest_size, alpha);+ }++- virtual std::vector<int> SampleEncodeAsIds(absl::string_view input,++ virtual std::vector<int> SampleEncodeAsIds(std::string_view input,+ int nbest_size,+ float alpha) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<int>, input,+@@ -485,7 +479,7 @@ class SentencePieceProcessor {+ }++ virtual std::vector<std::pair<std::vector<std::string>, float>>+- SampleEncodeAndScoreAsPieces(absl::string_view input, int num_samples,++ SampleEncodeAndScoreAsPieces(std::string_view input, int num_samples,+ float alpha, bool wor, bool include_best) const {+ using _T = std::vector<std::pair<std::vector<std::string>, float>>;+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,+@@ -493,7 +487,7 @@ class SentencePieceProcessor {+ }++ virtual std::vector<std::pair<std::vector<int>, float>>+- SampleEncodeAndScoreAsIds(absl::string_view input, int num_samples,++ SampleEncodeAndScoreAsIds(std::string_view input, int num_samples,+ float alpha, bool wor, bool include_best) const {+ using _T = std::vector<std::pair<std::vector<int>, float>>;+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,+@@ -507,7 +501,7 @@ class SentencePieceProcessor {+ }++ virtual std::string DecodePieces(+- const std::vector<absl::string_view> &pieces) const {++ const std::vector<std::string_view> &pieces) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, pieces);+ }++@@ -515,7 +509,7 @@ class SentencePieceProcessor {+ DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, ids);+ }++- virtual float CalculateEntropy(absl::string_view text, float alpha) const {++ virtual float CalculateEntropy(std::string_view text, float alpha) const {+ DEFINE_SPP_DIRECT_FUNC_IMPL(CalculateEntropy, float, text, alpha);+ }++@@ -524,25 +518,25 @@ class SentencePieceProcessor {+ // They are used in Python interface. Returns serialized proto.+ // In python module, we can get access to the full Proto after+ // deserialzing the returned byte sequence.+- virtual util::bytes EncodeAsSerializedProto(absl::string_view input) const {++ virtual util::bytes EncodeAsSerializedProto(std::string_view input) const {+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);+ }++- virtual util::bytes SampleEncodeAsSerializedProto(absl::string_view input,++ virtual util::bytes SampleEncodeAsSerializedProto(std::string_view input,+ int nbest_size,+ float alpha) const {+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,+ input, nbest_size, alpha);+ }++- virtual util::bytes NBestEncodeAsSerializedProto(absl::string_view input,++ virtual util::bytes NBestEncodeAsSerializedProto(std::string_view input,+ int nbest_size) const {+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(+ NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);+ }++ virtual util::bytes SampleEncodeAndScoreAsSerializedProto(+- absl::string_view input, int num_samples, float alpha, bool wor,++ std::string_view input, int num_samples, float alpha, bool wor,+ bool include_best) const {+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncodeAndScore,+ ImmutableNBestSentencePieceText, input,+@@ -557,7 +551,7 @@ class SentencePieceProcessor {+ }++ virtual util::bytes DecodePiecesAsSerializedProto(+- const std::vector<absl::string_view> &pieces) const {++ const std::vector<std::string_view> &pieces) const {+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(Decode, ImmutableSentencePieceText,+ pieces);+ }+@@ -570,24 +564,24 @@ class SentencePieceProcessor {+ //////////////////////////////////////////////////////////////+ // ImmutableProto API.+ virtual ImmutableSentencePieceText EncodeAsImmutableProto(+- absl::string_view input) const {++ std::string_view input) const {+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);+ }++ virtual ImmutableSentencePieceText SampleEncodeAsImmutableProto(+- absl::string_view input, int nbest_size, float alpha) const {++ std::string_view input, int nbest_size, float alpha) const {+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,+ input, nbest_size, alpha);+ }++ virtual ImmutableNBestSentencePieceText NBestEncodeAsImmutableProto(+- absl::string_view input, int nbest_size) const {++ std::string_view input, int nbest_size) const {+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(+ NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);+ }++ virtual ImmutableNBestSentencePieceText SampleEncodeAndScoreAsImmutableProto(+- absl::string_view input, int num_samples, float alpha, bool wor,++ std::string_view input, int num_samples, float alpha, bool wor,+ bool include_best) const {+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncodeAndScore,+ ImmutableNBestSentencePieceText, input,+@@ -601,7 +595,7 @@ class SentencePieceProcessor {+ }++ virtual ImmutableSentencePieceText DecodePiecesAsImmutableProto(+- const std::vector<absl::string_view> &pieces) const {++ const std::vector<std::string_view> &pieces) const {+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Decode, ImmutableSentencePieceText, pieces);+ }++@@ -623,7 +617,7 @@ class SentencePieceProcessor {++ // Returns the vocab id of `piece`.+ // Returns UNK(0) if `piece` is unknown.+- virtual int PieceToId(absl::string_view piece) const;++ virtual int PieceToId(std::string_view piece) const;++ // Returns the string representation of vocab with `id`.+ virtual const std::string &IdToPiece(int id) const;+@@ -680,16 +674,16 @@ class SentencePieceProcessor {+ private:+ enum ExtraOption { REVERSE, BOS, EOS, UNK_PIECE };++- util::Status ParseExtraOptions(absl::string_view extra_option,++ util::Status ParseExtraOptions(std::string_view extra_option,+ std::vector<ExtraOption> *extra_options) const;++ util::Status ApplyExtraOptions(const std::vector<ExtraOption> &extra_options,+ SentencePieceText *spt) const;++ util::Status PopulateSentencePieceText(+- absl::string_view input, absl::string_view normalized,++ std::string_view input, std::string_view normalized,+ const std::vector<size_t> &norm_to_orig,+- const std::vector<std::pair<absl::string_view, int>> &result,++ const std::vector<std::pair<std::string_view, int>> &result,+ SentencePieceText *spt) const;++ std::unique_ptr<ModelInterface> model_;+@@ -718,10 +712,10 @@ namespace io {+ // io::LoadModelProto("//path/spm.model", model_proto.get());+ // SentencePieceProcessor sp;+ // CHECK_OK(sp.Load(std::move(model_proto)));+-util::Status LoadModelProto(absl::string_view, ModelProto *model_proto);++util::Status LoadModelProto(std::string_view, ModelProto *model_proto);++ // Saves `model_proto` as `filename`.+-util::Status SaveModelProto(absl::string_view, const ModelProto &model_proto);++util::Status SaveModelProto(std::string_view, const ModelProto &model_proto);+ } // namespace io+ } // namespace sentencepiece+ #endif // SENTENCEPIECE_PROCESSOR_H_diff --git a/include/text_embedder.h b/include/text_embedder.hindex 3ca0d3d0..3b585b71 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -1,20 +1,16 @@#pragma once+#include <sentencepiece_processor.h>#include <core/session/onnxruntime_cxx_api.h>#include <tokenizer/bert_tokenizer.hpp>#include <vector>#include "option.h"--struct encoded_input_t {- std::vector<int64_t> input_ids;- std::vector<int64_t> token_type_ids;- std::vector<int64_t> attention_mask;-};+#include "text_embedder_tokenizer.h"class TextEmbedder {public:- TextEmbedder(const std::string& model_path);+ TextEmbedder(const std::string& model_path, TokenizerType tokenizer_type = TokenizerType::bert);TextEmbedder(const std::string& openai_model_path, const std::string& api_key);~TextEmbedder();Option<std::vector<float>> Embed(const std::string& text);@@ -30,7 +26,7 @@ class TextEmbedder {std::unique_ptr<Ort::Session> session_;Ort::Env env_;encoded_input_t Encode(const std::string& text);- std::unique_ptr<BertTokenizer> tokenizer_;+ std::unique_ptr<TextEmbeddingTokenizer> tokenizer_;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;std::string api_key;diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex dfad1b10..7789dec2 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -15,6 +15,9 @@ struct text_embedding_model {std::string model_md5;std::string vocab_url;std::string vocab_md5;+ TokenizerType tokenizer_type;+ std::string indexing_prefix = "";+ std::string query_prefix = "";};@@ -33,16 +36,22 @@ public:TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {std::unique_lock<std::mutex> lock(text_embedders_mutex);+ LOG(INFO) << "Getting text embedder for model: " << model_parameters.dump();const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");if(text_embedders[model_name] == nullptr) {if(model_parameters.count("api_key") == 0) {- // download the model if it doesn't exist+ TokenizerType tokenizer_type;if(is_public_model(model_name)) {+ tokenizer_type = public_models[model_name].tokenizer_type;+ // download the model if it doesn't existdownload_public_model(model_name);+ } else {+ tokenizer_type = get_tokenizer_type(model_parameters);}- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);+ LOG(INFO) << "Creating text embedder for model: " << model_name;+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, tokenizer_type);} else {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key"));+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());}}return text_embedders[model_name].get();@@ -60,8 +69,28 @@ public:text_embedders.clear();}- static const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {- std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_INDEXING_PREFIX : model_parameters["indexing_prefix"].get<std::string>();+ static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters) {+ if(model_parameters.count("tokenizer_type") == 0) {+ return TokenizerType::bert;+ } else {+ std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();+ if(tokenizer_type == "distilBert") {+ return TokenizerType::distilBert;+ } else if(tokenizer_type == "xlm-roberta") {+ return TokenizerType::xlm_roberta;+ } else {+ return TokenizerType::bert;+ }+ }+ }++ const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {+ std::string val;+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {+ val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;+ } else {+ val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();+ }if(!val.empty()) {val += " ";}@@ -69,8 +98,13 @@ public:return val;}- static const std::string get_query_prefix(const nlohmann::json& model_parameters) {- std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_QUERY_PREFIX : model_parameters["query_prefix"].get<std::string>();+ const std::string get_query_prefix(const nlohmann::json& model_parameters) {+ std::string val;+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {+ val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;+ } else {+ val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();+ }if(!val.empty()) {val += " ";}@@ -99,34 +133,21 @@ public:static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";inline static std::string model_dir = "";inline static const std::string get_absolute_model_path(const std::string& model_name) {- if(model_dir.back() != '/') {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + "/" + model_name)) {- std::filesystem::create_directories(model_dir + "/" + model_name);- }- return model_dir + "/" + model_name + "/" + "model.onnx";- } else {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + model_name)) {- std::filesystem::create_directories(model_dir + model_name);- }- return model_dir + model_name + "/" + "model.onnx";- }- };+ return get_model_subdir(model_name) + "/model.onnx";+ }inline static const std::string get_absolute_vocab_path(const std::string& model_name) {- if(model_dir.back() != '/') {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + "/" + model_name)) {- std::filesystem::create_directories(model_dir + "/" + model_name);- }- return model_dir + "/" + model_name + "/" + "vocab.txt";- } else {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + model_name)) {- std::filesystem::create_directories(model_dir + model_name);+ return get_model_subdir(model_name) + "/vocab.txt";+ }+ inline static const std::string get_absolute_sentencepiece_model_path(const std::string& model_name) {+ auto model_subdir = get_model_subdir(model_name);+ // look for any file that ends with .model+ for (const auto & entry : std::filesystem::directory_iterator(model_subdir)) {+ if(entry.path().extension() == ".model") {+ return entry.path().string();}- return model_dir + model_name + "/" + "vocab.txt";}++ return model_subdir + "/sentencepiece.model";}inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {std::ifstream stream(file_path);@@ -152,11 +173,23 @@ public:LOG(INFO) << "Failed to download public model " << model_name << ": " << res;}}- if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {- LOG(INFO) << "Downloading default vocab for model: " << model_name;- long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));- if(res != 200) {- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;++ if(model.tokenizer_type == TokenizerType::bert || model.tokenizer_type == TokenizerType::distilBert) {+ if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {+ LOG(INFO) << "Downloading default vocab for model: " << model_name;+ long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));+ if(res != 200) {+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;+ }+ }+ } else {+ LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;+ if(!check_md5(get_absolute_sentencepiece_model_path(model_name), model.vocab_md5)) {+ LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;+ long res = httpClient.download_file(model.vocab_url, get_absolute_sentencepiece_model_path(model_name));+ if(res != 200) {+ LOG(INFO) << "Failed to download default sentencepiece model " << model_name << ": " << res;+ }}}}@@ -170,21 +203,48 @@ private:"https://models.typesense.org/public/e5-small/model.onnx","3d421dc72859a72368c106415cdebf2","https://models.typesense.org/public/e5-small/vocab.txt",- "6480d5d8528ce344256daf115d4965e"};+ "6480d5d8528ce344256daf115d4965e",+ TokenizerType::bert,+ "passage:",+ "query:"};public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{"https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx","6d196b8f7a8d8abcfb08afcac1704302","https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",- "9d2131a9a433502abeb512978d452ecc"};+ "9d2131a9a433502abeb512978d452ecc",+ TokenizerType::bert};public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{"https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx","91f535cd7ca87359b9bfd1d87f9e452e","https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",- "0f05e1b7420dea2db8ade63eb5b80f7a"};+ "0f05e1b7420dea2db8ade63eb5b80f7a",+ TokenizerType::distilBert};+ public_models["ts-paraphrase-multilingual-mpnet-base-v2"] = text_embedding_model{+ "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/model.onnx",+ "728d3db98e1b7a691a731644867382c5",+ "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/sentencepiece.bpe.model",+ "bf25eb5120ad92ef5c7d8596b5dc4046",+ TokenizerType::xlm_roberta};}std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;std::unordered_map<std::string, text_embedding_model> public_models;std::mutex text_embedders_mutex;++ static const std::string get_model_subdir(const std::string& model_name) {+ if(model_dir.back() != '/') {+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {+ std::filesystem::create_directories(model_dir + "/" + model_name);+ }+ return model_dir + "/" + model_name;+ } else {+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + model_name)) {+ std::filesystem::create_directories(model_dir + model_name);+ }+ return model_dir + model_name;+ }+ }};diff --git a/include/text_embedder_tokenizer.h b/include/text_embedder_tokenizer.hnew file mode 100644index 00000000..d0d66098--- /dev/null+++ b/include/text_embedder_tokenizer.h@@ -0,0 +1,59 @@+#pragma++#include <vector>+#include <unordered_map>+#include <sentencepiece_processor.h>+#include <tokenizer/bert_tokenizer.hpp>+++enum class TokenizerType {+ bert,+ distilBert,+ xlm_roberta+};++struct encoded_input_t {+ std::vector<int64_t> input_ids;+ std::vector<int64_t> token_type_ids;+ std::vector<int64_t> attention_mask;+};++++// Create a base class for all tokenizers to inherit from+class TextEmbeddingTokenizer {+ private:+ public:+ virtual encoded_input_t Encode(const std::string& text) = 0;+};++class BertTokenizerWrapper : public TextEmbeddingTokenizer {+ protected:+ std::unique_ptr<BertTokenizer> bert_tokenizer_;+ public:+ BertTokenizerWrapper(const std::string& vocab_path);+ encoded_input_t Encode(const std::string& text) override;+};++class DistilBertTokenizer : public BertTokenizerWrapper {+ public:+ DistilBertTokenizer(const std::string& vocab_path);+ encoded_input_t Encode(const std::string& text) override;+};++class XLMRobertaTokenizer : public TextEmbeddingTokenizer {+ private:+ inline static constexpr uint8_t fairseq_offset = 1;+ std::unordered_map<std::string, int64_t> fairseq_tokens_to_ids_ = {+ {"<s>", 0},+ {"<pad>", 1},+ {"</s>", 2},+ {"<unk>", 3},+ };+ std::unique_ptr<sentencepiece::SentencePieceProcessor> sentencepiece_tokenizer_;+ const int token_to_id(const std::string& token);+ const std::vector<std::string> tokenize(const std::string& text);+ public:+ XLMRobertaTokenizer(const std::string& model_path);+ encoded_input_t Encode(const std::string& text) override;+};\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex 69b12609..e54312ad 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -6348,7 +6348,7 @@ Option<bool> Index::embed_fields(nlohmann::json& document,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema) {for(const auto& field : embedding_fields) {- std::string text_to_embed = TextEmbedderManager::get_indexing_prefix(field.model_parameters);+ std::string text_to_embed = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);for(const auto& field_name : field.embed_from) {auto field_it = search_schema.find(field_name);if(field_it.value().type == field_types::STRING) {@@ -6378,7 +6378,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,for(const auto& field : embedding_fields) {std::vector<std::string> text_to_embed;for(auto& document : documents) {- std::string text = TextEmbedderManager::get_indexing_prefix(field.model_parameters);+ std::string text = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);for(const auto& field_name : field.embed_from) {auto field_it = search_schema.find(field_name);if(field_it.value().type == field_types::STRING) {diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 419f0a5b..871a679a 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -6,18 +6,19 @@#include <sstream>#include <filesystem>-TextEmbedder::TextEmbedder(const std::string& model_name) {+TextEmbedder::TextEmbedder(const std::string& model_name, TokenizerType tokenizer_type) {// create environmentOrt::SessionOptions session_options;std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);LOG(INFO) << "Loading model from: " << abs_path;session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);- std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path(model_name));- std::stringstream ss;- ss << stream.rdbuf();- auto vocab_ = ss.str();- tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),- ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));+ if(tokenizer_type == TokenizerType::bert) {+ auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);+ tokenizer_ = std::make_unique<BertTokenizerWrapper>(vocab_path);+ } else {+ auto spiece_model_path = TextEmbedderManager::get_absolute_sentencepiece_model_path(model_name);+ tokenizer_ = std::make_unique<XLMRobertaTokenizer>(spiece_model_path);+ }auto output_tensor_count = session_->GetOutputCount();for (size_t i = 0; i < output_tensor_count; i++) {auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();@@ -34,22 +35,6 @@ TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::stri}-encoded_input_t TextEmbedder::Encode(const std::string& text) {-- auto encoded = tokenizer_->Encode(tokenizer_->Tokenize(ustring(text)));- auto input_ids = tokenizer_->AddSpecialToken(encoded);- auto token_type_ids = tokenizer_->GenerateTypeId(encoded);- auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);- // BERT supports max sequence length of 512- if (input_ids.size() > 512) {- input_ids.resize(512);- token_type_ids.resize(512);- attention_mask.resize(512);- }- return {input_ids, token_type_ids, attention_mask};-}--std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<float>>& inputs) {std::vector<float> pooled_output;@@ -82,14 +67,14 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {}return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());} else {- auto encoded_input = Encode(text);+ auto encoded_input = tokenizer_->Encode(text);// create input tensor object from data valuesOrt::AllocatorWithDefaultOptions allocator;Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);std::vector<Ort::Value> input_tensors;std::vector<std::vector<int64_t>> input_shapes;std::vector<const char*> input_node_names = {"input_ids", "attention_mask"};- // If model is DistilBERT, it has 2 inputs, else it has 3 inputs+ // If model is DistilBERT or sentencepiece, it has 2 inputs, else it has 3 inputsif(session_->GetInputCount() == 3) {input_node_names.push_back("token_type_ids");}@@ -171,11 +156,6 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& nTextEmbedderManager::get_instance().download_public_model(model_name);}- std::string vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);- if(!std::filesystem::exists(vocab_path)) {- LOG(ERROR) << "Vocab file not found: " << vocab_path;- return false;- }Ort::SessionOptions session_options;Ort::Env env;@@ -211,8 +191,6 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& nLOG(ERROR) << "Invalid model: token_type_ids tensor not found";return false;}- } else {- LOG(INFO) << "Model is DistilBERT";}auto output_tensor_count = session.GetOutputCount();diff --git a/src/text_embedder_tokenizer.cpp b/src/text_embedder_tokenizer.cppnew file mode 100644index 00000000..f3dee97c--- /dev/null+++ b/src/text_embedder_tokenizer.cpp@@ -0,0 +1,88 @@+#include <fstream>+#include <sstream>+#include "text_embedder_tokenizer.h"+++BertTokenizerWrapper::BertTokenizerWrapper(const std::string& vocab_path) {+ std::ifstream stream(vocab_path);+ std::stringstream ss;+ ss << stream.rdbuf();+ auto vocab_ = ss.str();+ bert_tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),+ ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));+}++encoded_input_t BertTokenizerWrapper::Encode(const std::string& text) {+ auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));+ auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);+ auto token_type_ids = bert_tokenizer_->GenerateTypeId(encoded);+ auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);+ // BERT supports max sequence length of 512+ if (input_ids.size() > 512) {+ input_ids.resize(512);+ token_type_ids.resize(512);+ attention_mask.resize(512);+ }+ return {input_ids, token_type_ids, attention_mask};+}++++DistilBertTokenizer::DistilBertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}+++encoded_input_t DistilBertTokenizer::Encode(const std::string& text) {+ auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));+ auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);+ auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);+ // DistilBERT supports max sequence length of 512+ if (input_ids.size() > 512) {+ input_ids.resize(512);+ attention_mask.resize(512);+ }+ return {input_ids, {}, attention_mask};+}+++XLMRobertaTokenizer::XLMRobertaTokenizer(const std::string& model_path) {+ sentencepiece_tokenizer_ = std::make_unique<sentencepiece::SentencePieceProcessor>();+ sentencepiece_tokenizer_->Load(model_path);+ fairseq_tokens_to_ids_["<mask>"] = sentencepiece_tokenizer_->GetPieceSize() + fairseq_offset;+ sentencepiece_tokenizer_->SetEncodeExtraOptions("bos:eos");+}++const int XLMRobertaTokenizer::token_to_id(const std::string& token) {+ auto it = fairseq_tokens_to_ids_.find(token);+ if (it != fairseq_tokens_to_ids_.end()) {+ return it->second;+ }+ auto spm_id = sentencepiece_tokenizer_->PieceToId(token);+ if (spm_id == 0) {+ return fairseq_tokens_to_ids_["<unk>"];+ }+ return spm_id + fairseq_offset;+}++const std::vector<std::string> XLMRobertaTokenizer::tokenize(const std::string& text) {+ std::vector<std::string> tokens;+ sentencepiece_tokenizer_->Encode(text, &tokens);+ return tokens;+}+++encoded_input_t XLMRobertaTokenizer::Encode(const std::string& text) {+ auto tokens = tokenize(text);+ auto input_ids = std::vector<int64_t>(tokens.size());+ auto attention_mask = std::vector<int64_t>(tokens.size(), 1);+ for (size_t i = 0; i < tokens.size(); i++) {+ input_ids[i] = token_to_id(tokens[i]);+ }+ // XLM-RoBERTa supports max sequence length of 128+ if (input_ids.size() > 128) {+ input_ids.resize(128);+ attention_mask.resize(128);+ input_ids[input_ids.size() - 1] = fairseq_tokens_to_ids_["<eos>"];+ }++ return {input_ids, {}, attention_mask};+}\ No newline at end of file