Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Added support for XLMRoBERTa tokenizer

ozanarmagancfcf95d5ddf2
diff --git a/BUILD b/BUILD
index a2780d5d..da734dde 100644
--- a/BUILD
+++ b/BUILD
@@ -52,6 +52,8 @@ cc_library(
"@rocksdb",
"@s2geometry",
"@hnsw",
+ # "@zip",
+ "@sentencepiece"
],
)
diff --git a/WORKSPACE b/WORKSPACE
index 098015ff..acc498b7 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -63,6 +63,22 @@ new_git_repository(
tag = "v1.2.12",
)
+# new_git_repository(
+# name = "zip",
+# build_file = "//bazel:zip.BUILD",
+# branch = "master",
+# remote = "https://github.com/kuba--/zip.git",
+# )
+
+new_git_repository(
+ name = "sentencepiece",
+ build_file = "//bazel:sentencepiece.BUILD",
+ tag = "v0.1.98",
+ remote = "https://github.com/google/sentencepiece",
+ patches = ["//bazel:sentencepiece.patch"],
+ patch_args = ["-p1"]
+)
+
git_repository(
name = "rules_perl",
remote = "https://github.com/bazelbuild/rules_perl.git",
diff --git a/bazel/sentencepiece.BUILD b/bazel/sentencepiece.BUILD
new file mode 100644
index 00000000..75d0ba17
--- /dev/null
+++ b/bazel/sentencepiece.BUILD
@@ -0,0 +1,31 @@
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
+
+package(default_visibility = ["//visibility:public"])
+
+
+filegroup(
+ name = "sentencepiece_src",
+ srcs = glob(["**"]),
+ visibility = ["//visibility:public"],
+)
+
+
+
+cmake(
+ name = "sentencepiece",
+ lib_source = "//:sentencepiece_src",
+ out_static_libs = ["libsentencepiece.a"],
+ out_include_dir = "include",
+ build_args = [
+ "--config Release",
+ "--target sentencepiece-static",
+ ],
+ install = False,
+ cache_entries = {
+ 'SPM_USE_BUILTIN_PROTOBUF': 'OFF',
+ },
+ postfix_script= """
+echo "Intstalling sentencepiece"
+cp $BUILD_TMPDIR/src/libsentencepiece.a $INSTALLDIR/lib/libsentencepiece.a
+"""
+)
diff --git a/bazel/sentencepiece.patch b/bazel/sentencepiece.patch
new file mode 100644
index 00000000..ba8e1152
--- /dev/null
+++ b/bazel/sentencepiece.patch
@@ -0,0 +1,380 @@
+diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h
+index 14b1e8c..331c2a5 100644
+--- a/src/sentencepiece_processor.h
++++ b/src/sentencepiece_processor.h
+@@ -22,12 +22,6 @@
+ #include <utility>
+ #include <vector>
+
+-#ifndef SWIG
+-namespace absl {
+-using std::string_view;
+-} // namespace absl
+-#endif // SWIG
+-
+ namespace sentencepiece {
+ namespace util {
+
+@@ -55,7 +49,7 @@ class Status {
+ public:
+ Status();
+ ~Status();
+- Status(StatusCode code, absl::string_view error_message);
++ Status(StatusCode code, std::string_view error_message);
+ Status(const Status &s);
+ void operator=(const Status &s);
+ bool operator==(const Status &s) const;
+@@ -241,11 +235,11 @@ class SentencePieceProcessor {
+
+ // Loads model from `filename`.
+ // Returns false if `filename` cannot be loaded.
+- virtual util::Status Load(absl::string_view filename);
++ virtual util::Status Load(std::string_view filename);
+
+ // Loads model from `filename`.
+ // Crash if `filename` cannot be loaded.
+- virtual void LoadOrDie(absl::string_view filename);
++ virtual void LoadOrDie(std::string_view filename);
+
+ // Loads model from `model_proto`.
+ // `model_proto` is copied.
+@@ -257,16 +251,16 @@ class SentencePieceProcessor {
+
+ // Loads model from `serialized`, which is a string-serialized model proto.
+ // Useful to load the model from a platform independent blob object.
+- virtual util::Status LoadFromSerializedProto(absl::string_view serialized);
++ virtual util::Status LoadFromSerializedProto(std::string_view serialized);
+
+ // Returns the status. Encode/Decode methods are valid when status is OK.
+ virtual util::Status status() const;
+
+ // Sets encode extra_option sequence.
+- virtual util::Status SetEncodeExtraOptions(absl::string_view extra_option);
++ virtual util::Status SetEncodeExtraOptions(std::string_view extra_option);
+
+ // Sets decode extra_option sequence.
+- virtual util::Status SetDecodeExtraOptions(absl::string_view extra_option);
++ virtual util::Status SetDecodeExtraOptions(std::string_view extra_option);
+
+ //////////////////////////////////////////////////////////////
+ // Vocabulary restriction.
+@@ -276,7 +270,7 @@ class SentencePieceProcessor {
+ // Restricts the vocabulary set.
+ // The input sentences are encoded into the tokens in `valid_vocab`.
+ virtual util::Status SetVocabulary(
+- const std::vector<absl::string_view> &valid_vocab);
++ const std::vector<std::string_view> &valid_vocab);
+
+ // Reverts the vocabulary restriction.
+ virtual util::Status ResetVocabulary();
+@@ -284,18 +278,18 @@ class SentencePieceProcessor {
+ // Loads the valid vocabulary set from `filename` in TSV format.
+ // Format: <token> <tab> <freq>.
+ // Any token with frequency < threshold will be treated as OOV.
+- virtual util::Status LoadVocabulary(absl::string_view filename,
++ virtual util::Status LoadVocabulary(std::string_view filename,
+ int threshold);
+
+ //////////////////////////////////////////////////////////////
+ // Simple Encode and Decode API.
+ //
+ // Given a UTF8 input, encodes it into a sequence of sentence pieces.
+- virtual util::Status Encode(absl::string_view input,
++ virtual util::Status Encode(std::string_view input,
+ std::vector<std::string> *pieces) const;
+
+ // Given a UTF8 input, encodes it into a sequence of ids.
+- virtual util::Status Encode(absl::string_view input,
++ virtual util::Status Encode(std::string_view input,
+ std::vector<int> *ids) const;
+
+ // Given a sequence of pieces, decodes it into a detokenized output.
+@@ -303,7 +297,7 @@ class SentencePieceProcessor {
+ std::string *detokenized) const;
+
+ // Given a sequence of pieces, decodes it into a detokenized output.
+- virtual util::Status Decode(const std::vector<absl::string_view> &pieces,
++ virtual util::Status Decode(const std::vector<std::string_view> &pieces,
+ std::string *detokenized) const;
+
+ // Given a sequence of ids, decodes it into a detokenized output.
+@@ -315,11 +309,11 @@ class SentencePieceProcessor {
+ //
+ // Same as Encode, but returns nbest results.
+ virtual util::Status NBestEncode(
+- absl::string_view input, int nbest_size,
++ std::string_view input, int nbest_size,
+ std::vector<std::vector<std::string>> *pieces) const;
+
+ // Same as Encode, but returns nbest results.
+- virtual util::Status NBestEncode(absl::string_view input, int nbest_size,
++ virtual util::Status NBestEncode(std::string_view input, int nbest_size,
+ std::vector<std::vector<int>> *ids) const;
+
+ //////////////////////////////////////////////////////////////
+@@ -342,12 +336,12 @@ class SentencePieceProcessor {
+ // `alpha`: The dropout probability `p` of bpe merge operations in
+ // https://arxiv.org/abs/1910.13267 Nbest-based sampling is not supported so
+ // nbest_size parameter is ignored in BPE.
+- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,
+ float alpha,
+ std::vector<std::string> *pieces) const;
+
+ // Same as above, but returns a sequence of ids.
+- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,
+ float alpha, std::vector<int> *ids) const;
+
+ //////////////////////////////////////////////////////////////
+@@ -366,13 +360,13 @@ class SentencePieceProcessor {
+ // included in the sample, and the remaining elements are sampled excluding
+ // the best.
+ virtual util::Status SampleEncodeAndScore(
+- absl::string_view input, int num_samples, float alpha, bool wor,
++ std::string_view input, int num_samples, float alpha, bool wor,
+ bool include_best,
+ std::vector<std::pair<std::vector<std::string>, float>> *pieces) const;
+
+ // Same as above, but returns a sequence of ids.
+ virtual util::Status SampleEncodeAndScore(
+- absl::string_view input, int num_samples, float alpha, bool wor,
++ std::string_view input, int num_samples, float alpha, bool wor,
+ bool include_best,
+ std::vector<std::pair<std::vector<int>, float>> *ids) const;
+
+@@ -381,7 +375,7 @@ class SentencePieceProcessor {
+ //
+ // This only available in model_type=unigram.
+ // Calculate entropy of possible tokenisations
+- virtual util::Status CalculateEntropy(absl::string_view input, float alpha,
++ virtual util::Status CalculateEntropy(std::string_view input, float alpha,
+ float *entropy) const;
+
+ //////////////////////////////////////////////////////////////
+@@ -397,24 +391,24 @@ class SentencePieceProcessor {
+ // ImmutableSentencePieceText spt;
+ // Encode("hello", spt.mutable_proto()).IgnoreError();
+ // std::cout << spt.pieces_size() << std::endl;
+- virtual util::Status Encode(absl::string_view input,
++ virtual util::Status Encode(std::string_view input,
+ SentencePieceText *spt) const;
+
+- virtual util::Status NBestEncode(absl::string_view input, int nbest_size,
++ virtual util::Status NBestEncode(std::string_view input, int nbest_size,
+ NBestSentencePieceText *nbest_spt) const;
+
+- virtual util::Status SampleEncode(absl::string_view input, int nbest_size,
++ virtual util::Status SampleEncode(std::string_view input, int nbest_size,
+ float alpha, SentencePieceText *spt) const;
+
+ virtual util::Status SampleEncodeAndScore(
+- absl::string_view input, int num_samples, float alpha, bool wor,
++ std::string_view input, int num_samples, float alpha, bool wor,
+ bool include_best, NBestSentencePieceText *samples_spt) const;
+
+ // DEPRECATED: Remove this API and use std::vector<std::string_view>
+ virtual util::Status Decode(const std::vector<std::string> &pieces,
+ SentencePieceText *spt) const;
+
+- virtual util::Status Decode(const std::vector<absl::string_view> &pieces,
++ virtual util::Status Decode(const std::vector<std::string_view> &pieces,
+ SentencePieceText *spt) const;
+
+ virtual util::Status Decode(const std::vector<int> &ids,
+@@ -450,34 +444,34 @@ class SentencePieceProcessor {
+ // Handy methods that return the result directly.
+ // These functions ignore internal errors.
+ virtual std::vector<std::string> EncodeAsPieces(
+- absl::string_view input) const {
++ std::string_view input) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<std::string>, input);
+ }
+
+- virtual std::vector<int> EncodeAsIds(absl::string_view input) const {
++ virtual std::vector<int> EncodeAsIds(std::string_view input) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(Encode, std::vector<int>, input);
+ }
+
+ virtual std::vector<std::vector<std::string>> NBestEncodeAsPieces(
+- absl::string_view input, int nbest_size) const {
++ std::string_view input, int nbest_size) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(
+ NBestEncode, std::vector<std::vector<std::string>>, input, nbest_size);
+ }
+
+ virtual std::vector<std::vector<int>> NBestEncodeAsIds(
+- absl::string_view input, int nbest_size) const {
++ std::string_view input, int nbest_size) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(NBestEncode, std::vector<std::vector<int>>,
+ input, nbest_size);
+ }
+
+- virtual std::vector<std::string> SampleEncodeAsPieces(absl::string_view input,
++ virtual std::vector<std::string> SampleEncodeAsPieces(std::string_view input,
+ int nbest_size,
+ float alpha) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<std::string>, input,
+ nbest_size, alpha);
+ }
+
+- virtual std::vector<int> SampleEncodeAsIds(absl::string_view input,
++ virtual std::vector<int> SampleEncodeAsIds(std::string_view input,
+ int nbest_size,
+ float alpha) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncode, std::vector<int>, input,
+@@ -485,7 +479,7 @@ class SentencePieceProcessor {
+ }
+
+ virtual std::vector<std::pair<std::vector<std::string>, float>>
+- SampleEncodeAndScoreAsPieces(absl::string_view input, int num_samples,
++ SampleEncodeAndScoreAsPieces(std::string_view input, int num_samples,
+ float alpha, bool wor, bool include_best) const {
+ using _T = std::vector<std::pair<std::vector<std::string>, float>>;
+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,
+@@ -493,7 +487,7 @@ class SentencePieceProcessor {
+ }
+
+ virtual std::vector<std::pair<std::vector<int>, float>>
+- SampleEncodeAndScoreAsIds(absl::string_view input, int num_samples,
++ SampleEncodeAndScoreAsIds(std::string_view input, int num_samples,
+ float alpha, bool wor, bool include_best) const {
+ using _T = std::vector<std::pair<std::vector<int>, float>>;
+ DEFINE_SPP_DIRECT_FUNC_IMPL(SampleEncodeAndScore, _T, input, num_samples,
+@@ -507,7 +501,7 @@ class SentencePieceProcessor {
+ }
+
+ virtual std::string DecodePieces(
+- const std::vector<absl::string_view> &pieces) const {
++ const std::vector<std::string_view> &pieces) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, pieces);
+ }
+
+@@ -515,7 +509,7 @@ class SentencePieceProcessor {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(Decode, std::string, ids);
+ }
+
+- virtual float CalculateEntropy(absl::string_view text, float alpha) const {
++ virtual float CalculateEntropy(std::string_view text, float alpha) const {
+ DEFINE_SPP_DIRECT_FUNC_IMPL(CalculateEntropy, float, text, alpha);
+ }
+
+@@ -524,25 +518,25 @@ class SentencePieceProcessor {
+ // They are used in Python interface. Returns serialized proto.
+ // In python module, we can get access to the full Proto after
+ // deserialzing the returned byte sequence.
+- virtual util::bytes EncodeAsSerializedProto(absl::string_view input) const {
++ virtual util::bytes EncodeAsSerializedProto(std::string_view input) const {
+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);
+ }
+
+- virtual util::bytes SampleEncodeAsSerializedProto(absl::string_view input,
++ virtual util::bytes SampleEncodeAsSerializedProto(std::string_view input,
+ int nbest_size,
+ float alpha) const {
+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,
+ input, nbest_size, alpha);
+ }
+
+- virtual util::bytes NBestEncodeAsSerializedProto(absl::string_view input,
++ virtual util::bytes NBestEncodeAsSerializedProto(std::string_view input,
+ int nbest_size) const {
+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(
+ NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);
+ }
+
+ virtual util::bytes SampleEncodeAndScoreAsSerializedProto(
+- absl::string_view input, int num_samples, float alpha, bool wor,
++ std::string_view input, int num_samples, float alpha, bool wor,
+ bool include_best) const {
+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(SampleEncodeAndScore,
+ ImmutableNBestSentencePieceText, input,
+@@ -557,7 +551,7 @@ class SentencePieceProcessor {
+ }
+
+ virtual util::bytes DecodePiecesAsSerializedProto(
+- const std::vector<absl::string_view> &pieces) const {
++ const std::vector<std::string_view> &pieces) const {
+ DEFINE_SPP_SERIALIZED_PROTO_IMPL(Decode, ImmutableSentencePieceText,
+ pieces);
+ }
+@@ -570,24 +564,24 @@ class SentencePieceProcessor {
+ //////////////////////////////////////////////////////////////
+ // ImmutableProto API.
+ virtual ImmutableSentencePieceText EncodeAsImmutableProto(
+- absl::string_view input) const {
++ std::string_view input) const {
+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Encode, ImmutableSentencePieceText, input);
+ }
+
+ virtual ImmutableSentencePieceText SampleEncodeAsImmutableProto(
+- absl::string_view input, int nbest_size, float alpha) const {
++ std::string_view input, int nbest_size, float alpha) const {
+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncode, ImmutableSentencePieceText,
+ input, nbest_size, alpha);
+ }
+
+ virtual ImmutableNBestSentencePieceText NBestEncodeAsImmutableProto(
+- absl::string_view input, int nbest_size) const {
++ std::string_view input, int nbest_size) const {
+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(
+ NBestEncode, ImmutableNBestSentencePieceText, input, nbest_size);
+ }
+
+ virtual ImmutableNBestSentencePieceText SampleEncodeAndScoreAsImmutableProto(
+- absl::string_view input, int num_samples, float alpha, bool wor,
++ std::string_view input, int num_samples, float alpha, bool wor,
+ bool include_best) const {
+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(SampleEncodeAndScore,
+ ImmutableNBestSentencePieceText, input,
+@@ -601,7 +595,7 @@ class SentencePieceProcessor {
+ }
+
+ virtual ImmutableSentencePieceText DecodePiecesAsImmutableProto(
+- const std::vector<absl::string_view> &pieces) const {
++ const std::vector<std::string_view> &pieces) const {
+ DEFINE_SPP_IMMUTABLE_PROTO_IMPL(Decode, ImmutableSentencePieceText, pieces);
+ }
+
+@@ -623,7 +617,7 @@ class SentencePieceProcessor {
+
+ // Returns the vocab id of `piece`.
+ // Returns UNK(0) if `piece` is unknown.
+- virtual int PieceToId(absl::string_view piece) const;
++ virtual int PieceToId(std::string_view piece) const;
+
+ // Returns the string representation of vocab with `id`.
+ virtual const std::string &IdToPiece(int id) const;
+@@ -680,16 +674,16 @@ class SentencePieceProcessor {
+ private:
+ enum ExtraOption { REVERSE, BOS, EOS, UNK_PIECE };
+
+- util::Status ParseExtraOptions(absl::string_view extra_option,
++ util::Status ParseExtraOptions(std::string_view extra_option,
+ std::vector<ExtraOption> *extra_options) const;
+
+ util::Status ApplyExtraOptions(const std::vector<ExtraOption> &extra_options,
+ SentencePieceText *spt) const;
+
+ util::Status PopulateSentencePieceText(
+- absl::string_view input, absl::string_view normalized,
++ std::string_view input, std::string_view normalized,
+ const std::vector<size_t> &norm_to_orig,
+- const std::vector<std::pair<absl::string_view, int>> &result,
++ const std::vector<std::pair<std::string_view, int>> &result,
+ SentencePieceText *spt) const;
+
+ std::unique_ptr<ModelInterface> model_;
+@@ -718,10 +712,10 @@ namespace io {
+ // io::LoadModelProto("//path/spm.model", model_proto.get());
+ // SentencePieceProcessor sp;
+ // CHECK_OK(sp.Load(std::move(model_proto)));
+-util::Status LoadModelProto(absl::string_view, ModelProto *model_proto);
++util::Status LoadModelProto(std::string_view, ModelProto *model_proto);
+
+ // Saves `model_proto` as `filename`.
+-util::Status SaveModelProto(absl::string_view, const ModelProto &model_proto);
++util::Status SaveModelProto(std::string_view, const ModelProto &model_proto);
+ } // namespace io
+ } // namespace sentencepiece
+ #endif // SENTENCEPIECE_PROCESSOR_H_
diff --git a/include/text_embedder.h b/include/text_embedder.h
index 3ca0d3d0..3b585b71 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -1,20 +1,16 @@
#pragma once
+#include <sentencepiece_processor.h>
#include <core/session/onnxruntime_cxx_api.h>
#include <tokenizer/bert_tokenizer.hpp>
#include <vector>
#include "option.h"
-
-struct encoded_input_t {
- std::vector<int64_t> input_ids;
- std::vector<int64_t> token_type_ids;
- std::vector<int64_t> attention_mask;
-};
+#include "text_embedder_tokenizer.h"
class TextEmbedder {
public:
- TextEmbedder(const std::string& model_path);
+ TextEmbedder(const std::string& model_path, TokenizerType tokenizer_type = TokenizerType::bert);
TextEmbedder(const std::string& openai_model_path, const std::string& api_key);
~TextEmbedder();
Option<std::vector<float>> Embed(const std::string& text);
@@ -30,7 +26,7 @@ class TextEmbedder {
std::unique_ptr<Ort::Session> session_;
Ort::Env env_;
encoded_input_t Encode(const std::string& text);
- std::unique_ptr<BertTokenizer> tokenizer_;
+ std::unique_ptr<TextEmbeddingTokenizer> tokenizer_;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
std::string api_key;
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index dfad1b10..7789dec2 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -15,6 +15,9 @@ struct text_embedding_model {
std::string model_md5;
std::string vocab_url;
std::string vocab_md5;
+ TokenizerType tokenizer_type;
+ std::string indexing_prefix = "";
+ std::string query_prefix = "";
};
@@ -33,16 +36,22 @@ public:
TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {
std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ LOG(INFO) << "Getting text embedder for model: " << model_parameters.dump();
const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");
if(text_embedders[model_name] == nullptr) {
if(model_parameters.count("api_key") == 0) {
- // download the model if it doesn't exist
+ TokenizerType tokenizer_type;
if(is_public_model(model_name)) {
+ tokenizer_type = public_models[model_name].tokenizer_type;
+ // download the model if it doesn't exist
download_public_model(model_name);
+ } else {
+ tokenizer_type = get_tokenizer_type(model_parameters);
}
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);
+ LOG(INFO) << "Creating text embedder for model: " << model_name;
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, tokenizer_type);
} else {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key"));
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());
}
}
return text_embedders[model_name].get();
@@ -60,8 +69,28 @@ public:
text_embedders.clear();
}
- static const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {
- std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_INDEXING_PREFIX : model_parameters["indexing_prefix"].get<std::string>();
+ static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters) {
+ if(model_parameters.count("tokenizer_type") == 0) {
+ return TokenizerType::bert;
+ } else {
+ std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();
+ if(tokenizer_type == "distilBert") {
+ return TokenizerType::distilBert;
+ } else if(tokenizer_type == "xlm-roberta") {
+ return TokenizerType::xlm_roberta;
+ } else {
+ return TokenizerType::bert;
+ }
+ }
+ }
+
+ const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {
+ std::string val;
+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {
+ val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;
+ } else {
+ val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();
+ }
if(!val.empty()) {
val += " ";
}
@@ -69,8 +98,13 @@ public:
return val;
}
- static const std::string get_query_prefix(const nlohmann::json& model_parameters) {
- std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_QUERY_PREFIX : model_parameters["query_prefix"].get<std::string>();
+ const std::string get_query_prefix(const nlohmann::json& model_parameters) {
+ std::string val;
+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {
+ val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;
+ } else {
+ val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();
+ }
if(!val.empty()) {
val += " ";
}
@@ -99,34 +133,21 @@ public:
static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";
inline static std::string model_dir = "";
inline static const std::string get_absolute_model_path(const std::string& model_name) {
- if(model_dir.back() != '/') {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + "/" + model_name)) {
- std::filesystem::create_directories(model_dir + "/" + model_name);
- }
- return model_dir + "/" + model_name + "/" + "model.onnx";
- } else {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + model_name)) {
- std::filesystem::create_directories(model_dir + model_name);
- }
- return model_dir + model_name + "/" + "model.onnx";
- }
- };
+ return get_model_subdir(model_name) + "/model.onnx";
+ }
inline static const std::string get_absolute_vocab_path(const std::string& model_name) {
- if(model_dir.back() != '/') {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + "/" + model_name)) {
- std::filesystem::create_directories(model_dir + "/" + model_name);
- }
- return model_dir + "/" + model_name + "/" + "vocab.txt";
- } else {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + model_name)) {
- std::filesystem::create_directories(model_dir + model_name);
+ return get_model_subdir(model_name) + "/vocab.txt";
+ }
+ inline static const std::string get_absolute_sentencepiece_model_path(const std::string& model_name) {
+ auto model_subdir = get_model_subdir(model_name);
+ // look for any file that ends with .model
+ for (const auto & entry : std::filesystem::directory_iterator(model_subdir)) {
+ if(entry.path().extension() == ".model") {
+ return entry.path().string();
}
- return model_dir + model_name + "/" + "vocab.txt";
}
+
+ return model_subdir + "/sentencepiece.model";
}
inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {
std::ifstream stream(file_path);
@@ -152,11 +173,23 @@ public:
LOG(INFO) << "Failed to download public model " << model_name << ": " << res;
}
}
- if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {
- LOG(INFO) << "Downloading default vocab for model: " << model_name;
- long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));
- if(res != 200) {
- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
+
+ if(model.tokenizer_type == TokenizerType::bert || model.tokenizer_type == TokenizerType::distilBert) {
+ if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {
+ LOG(INFO) << "Downloading default vocab for model: " << model_name;
+ long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));
+ if(res != 200) {
+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
+ }
+ }
+ } else {
+ LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;
+ if(!check_md5(get_absolute_sentencepiece_model_path(model_name), model.vocab_md5)) {
+ LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;
+ long res = httpClient.download_file(model.vocab_url, get_absolute_sentencepiece_model_path(model_name));
+ if(res != 200) {
+ LOG(INFO) << "Failed to download default sentencepiece model " << model_name << ": " << res;
+ }
}
}
}
@@ -170,21 +203,48 @@ private:
"https://models.typesense.org/public/e5-small/model.onnx",
"3d421dc72859a72368c106415cdebf2",
"https://models.typesense.org/public/e5-small/vocab.txt",
- "6480d5d8528ce344256daf115d4965e"};
+ "6480d5d8528ce344256daf115d4965e",
+ TokenizerType::bert,
+ "passage:",
+ "query:"};
public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{
"https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx",
"6d196b8f7a8d8abcfb08afcac1704302",
"https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",
- "9d2131a9a433502abeb512978d452ecc"};
+ "9d2131a9a433502abeb512978d452ecc",
+ TokenizerType::bert};
public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{
"https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx",
"91f535cd7ca87359b9bfd1d87f9e452e",
"https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",
- "0f05e1b7420dea2db8ade63eb5b80f7a"};
+ "0f05e1b7420dea2db8ade63eb5b80f7a",
+ TokenizerType::distilBert};
+ public_models["ts-paraphrase-multilingual-mpnet-base-v2"] = text_embedding_model{
+ "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/model.onnx",
+ "728d3db98e1b7a691a731644867382c5",
+ "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/sentencepiece.bpe.model",
+ "bf25eb5120ad92ef5c7d8596b5dc4046",
+ TokenizerType::xlm_roberta};
}
std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;
std::unordered_map<std::string, text_embedding_model> public_models;
std::mutex text_embedders_mutex;
+
+ static const std::string get_model_subdir(const std::string& model_name) {
+ if(model_dir.back() != '/') {
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {
+ std::filesystem::create_directories(model_dir + "/" + model_name);
+ }
+ return model_dir + "/" + model_name;
+ } else {
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + model_name)) {
+ std::filesystem::create_directories(model_dir + model_name);
+ }
+ return model_dir + model_name;
+ }
+ }
};
diff --git a/include/text_embedder_tokenizer.h b/include/text_embedder_tokenizer.h
new file mode 100644
index 00000000..d0d66098
--- /dev/null
+++ b/include/text_embedder_tokenizer.h
@@ -0,0 +1,59 @@
+#pragma
+
+#include <vector>
+#include <unordered_map>
+#include <sentencepiece_processor.h>
+#include <tokenizer/bert_tokenizer.hpp>
+
+
+enum class TokenizerType {
+ bert,
+ distilBert,
+ xlm_roberta
+};
+
+struct encoded_input_t {
+ std::vector<int64_t> input_ids;
+ std::vector<int64_t> token_type_ids;
+ std::vector<int64_t> attention_mask;
+};
+
+
+
+// Create a base class for all tokenizers to inherit from
+class TextEmbeddingTokenizer {
+ private:
+ public:
+ virtual encoded_input_t Encode(const std::string& text) = 0;
+};
+
+class BertTokenizerWrapper : public TextEmbeddingTokenizer {
+ protected:
+ std::unique_ptr<BertTokenizer> bert_tokenizer_;
+ public:
+ BertTokenizerWrapper(const std::string& vocab_path);
+ encoded_input_t Encode(const std::string& text) override;
+};
+
+class DistilBertTokenizer : public BertTokenizerWrapper {
+ public:
+ DistilBertTokenizer(const std::string& vocab_path);
+ encoded_input_t Encode(const std::string& text) override;
+};
+
+class XLMRobertaTokenizer : public TextEmbeddingTokenizer {
+ private:
+ inline static constexpr uint8_t fairseq_offset = 1;
+ std::unordered_map<std::string, int64_t> fairseq_tokens_to_ids_ = {
+ {"<s>", 0},
+ {"<pad>", 1},
+ {"</s>", 2},
+ {"<unk>", 3},
+ };
+ std::unique_ptr<sentencepiece::SentencePieceProcessor> sentencepiece_tokenizer_;
+ const int token_to_id(const std::string& token);
+ const std::vector<std::string> tokenize(const std::string& text);
+ public:
+ XLMRobertaTokenizer(const std::string& model_path);
+ encoded_input_t Encode(const std::string& text) override;
+};
\ No newline at end of file
diff --git a/src/index.cpp b/src/index.cpp
index 69b12609..e54312ad 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -6348,7 +6348,7 @@ Option<bool> Index::embed_fields(nlohmann::json& document,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema) {
for(const auto& field : embedding_fields) {
- std::string text_to_embed = TextEmbedderManager::get_indexing_prefix(field.model_parameters);
+ std::string text_to_embed = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);
for(const auto& field_name : field.embed_from) {
auto field_it = search_schema.find(field_name);
if(field_it.value().type == field_types::STRING) {
@@ -6378,7 +6378,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
for(const auto& field : embedding_fields) {
std::vector<std::string> text_to_embed;
for(auto& document : documents) {
- std::string text = TextEmbedderManager::get_indexing_prefix(field.model_parameters);
+ std::string text = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);
for(const auto& field_name : field.embed_from) {
auto field_it = search_schema.find(field_name);
if(field_it.value().type == field_types::STRING) {
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index 419f0a5b..871a679a 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -6,18 +6,19 @@
#include <sstream>
#include <filesystem>
-TextEmbedder::TextEmbedder(const std::string& model_name) {
+TextEmbedder::TextEmbedder(const std::string& model_name, TokenizerType tokenizer_type) {
// create environment
Ort::SessionOptions session_options;
std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);
LOG(INFO) << "Loading model from: " << abs_path;
session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);
- std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path(model_name));
- std::stringstream ss;
- ss << stream.rdbuf();
- auto vocab_ = ss.str();
- tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),
- ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));
+ if(tokenizer_type == TokenizerType::bert) {
+ auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);
+ tokenizer_ = std::make_unique<BertTokenizerWrapper>(vocab_path);
+ } else {
+ auto spiece_model_path = TextEmbedderManager::get_absolute_sentencepiece_model_path(model_name);
+ tokenizer_ = std::make_unique<XLMRobertaTokenizer>(spiece_model_path);
+ }
auto output_tensor_count = session_->GetOutputCount();
for (size_t i = 0; i < output_tensor_count; i++) {
auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
@@ -34,22 +35,6 @@ TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::stri
}
-encoded_input_t TextEmbedder::Encode(const std::string& text) {
-
- auto encoded = tokenizer_->Encode(tokenizer_->Tokenize(ustring(text)));
- auto input_ids = tokenizer_->AddSpecialToken(encoded);
- auto token_type_ids = tokenizer_->GenerateTypeId(encoded);
- auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);
- // BERT supports max sequence length of 512
- if (input_ids.size() > 512) {
- input_ids.resize(512);
- token_type_ids.resize(512);
- attention_mask.resize(512);
- }
- return {input_ids, token_type_ids, attention_mask};
-}
-
-
std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<float>>& inputs) {
std::vector<float> pooled_output;
@@ -82,14 +67,14 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
}
return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());
} else {
- auto encoded_input = Encode(text);
+ auto encoded_input = tokenizer_->Encode(text);
// create input tensor object from data values
Ort::AllocatorWithDefaultOptions allocator;
Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
std::vector<Ort::Value> input_tensors;
std::vector<std::vector<int64_t>> input_shapes;
std::vector<const char*> input_node_names = {"input_ids", "attention_mask"};
- // If model is DistilBERT, it has 2 inputs, else it has 3 inputs
+ // If model is DistilBERT or sentencepiece, it has 2 inputs, else it has 3 inputs
if(session_->GetInputCount() == 3) {
input_node_names.push_back("token_type_ids");
}
@@ -171,11 +156,6 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& n
TextEmbedderManager::get_instance().download_public_model(model_name);
}
- std::string vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);
- if(!std::filesystem::exists(vocab_path)) {
- LOG(ERROR) << "Vocab file not found: " << vocab_path;
- return false;
- }
Ort::SessionOptions session_options;
Ort::Env env;
@@ -211,8 +191,6 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& n
LOG(ERROR) << "Invalid model: token_type_ids tensor not found";
return false;
}
- } else {
- LOG(INFO) << "Model is DistilBERT";
}
auto output_tensor_count = session.GetOutputCount();
diff --git a/src/text_embedder_tokenizer.cpp b/src/text_embedder_tokenizer.cpp
new file mode 100644
index 00000000..f3dee97c
--- /dev/null
+++ b/src/text_embedder_tokenizer.cpp
@@ -0,0 +1,88 @@
+#include <fstream>
+#include <sstream>
+#include "text_embedder_tokenizer.h"
+
+
+BertTokenizerWrapper::BertTokenizerWrapper(const std::string& vocab_path) {
+ std::ifstream stream(vocab_path);
+ std::stringstream ss;
+ ss << stream.rdbuf();
+ auto vocab_ = ss.str();
+ bert_tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),
+ ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));
+}
+
+encoded_input_t BertTokenizerWrapper::Encode(const std::string& text) {
+ auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));
+ auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);
+ auto token_type_ids = bert_tokenizer_->GenerateTypeId(encoded);
+ auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);
+ // BERT supports max sequence length of 512
+ if (input_ids.size() > 512) {
+ input_ids.resize(512);
+ token_type_ids.resize(512);
+ attention_mask.resize(512);
+ }
+ return {input_ids, token_type_ids, attention_mask};
+}
+
+
+
+DistilBertTokenizer::DistilBertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}
+
+
+encoded_input_t DistilBertTokenizer::Encode(const std::string& text) {
+ auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));
+ auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);
+ auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);
+ // DistilBERT supports max sequence length of 512
+ if (input_ids.size() > 512) {
+ input_ids.resize(512);
+ attention_mask.resize(512);
+ }
+ return {input_ids, {}, attention_mask};
+}
+
+
+XLMRobertaTokenizer::XLMRobertaTokenizer(const std::string& model_path) {
+ sentencepiece_tokenizer_ = std::make_unique<sentencepiece::SentencePieceProcessor>();
+ sentencepiece_tokenizer_->Load(model_path);
+ fairseq_tokens_to_ids_["<mask>"] = sentencepiece_tokenizer_->GetPieceSize() + fairseq_offset;
+ sentencepiece_tokenizer_->SetEncodeExtraOptions("bos:eos");
+}
+
+const int XLMRobertaTokenizer::token_to_id(const std::string& token) {
+ auto it = fairseq_tokens_to_ids_.find(token);
+ if (it != fairseq_tokens_to_ids_.end()) {
+ return it->second;
+ }
+ auto spm_id = sentencepiece_tokenizer_->PieceToId(token);
+ if (spm_id == 0) {
+ return fairseq_tokens_to_ids_["<unk>"];
+ }
+ return spm_id + fairseq_offset;
+}
+
+const std::vector<std::string> XLMRobertaTokenizer::tokenize(const std::string& text) {
+ std::vector<std::string> tokens;
+ sentencepiece_tokenizer_->Encode(text, &tokens);
+ return tokens;
+}
+
+
+encoded_input_t XLMRobertaTokenizer::Encode(const std::string& text) {
+ auto tokens = tokenize(text);
+ auto input_ids = std::vector<int64_t>(tokens.size());
+ auto attention_mask = std::vector<int64_t>(tokens.size(), 1);
+ for (size_t i = 0; i < tokens.size(); i++) {
+ input_ids[i] = token_to_id(tokens[i]);
+ }
+ // XLM-RoBERTa supports max sequence length of 128
+ if (input_ids.size() > 128) {
+ input_ids.resize(128);
+ attention_mask.resize(128);
+ input_ids[input_ids.size() - 1] = fairseq_tokens_to_ids_["<eos>"];
+ }
+
+ return {input_ids, {}, attention_mask};
+}
\ No newline at end of file