Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use new CLIPTokenizer
ozanarmaganbfc799c5c065
diff --git a/BUILD b/BUILDindex 0532d776..fe9b36c6 100644--- a/BUILD+++ b/BUILD@@ -54,6 +54,7 @@ cc_library("@rocksdb","@s2geometry","@hnsw",+ "@clip_tokenizer//:clip"# "@zip",],)diff --git a/WORKSPACE b/WORKSPACEindex 80f643c9..47d347d2 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -51,6 +51,13 @@ new_git_repository(patch_cmds= ["git submodule sync && git submodule foreach 'git fetch --tags' && git submodule update --init --remote"])+new_git_repository(+ name="clip_tokenizer",+ branch="master",+ remote="https://github.com/ozanarmagan/clip_tokenizer_cpp",+ build_file = "//bazel:clip_tokenizer.BUILD",+)+new_git_repository(name = "onnx_runtime_extensions",build_file = "//bazel:onnxruntime_extensions.BUILD",diff --git a/bazel/clip_tokenizer.BUILD b/bazel/clip_tokenizer.BUILDnew file mode 100644index 00000000..133ffbbc--- /dev/null+++ b/bazel/clip_tokenizer.BUILD@@ -0,0 +1,11 @@+cc_library(+ name="clip",+ srcs=["clip_tokenizer.cpp"],+ hdrs= glob(["**/*.h"]),+ includes=["."],+ deps=["@icu"],+ visibility=["//visibility:public"],+ linkstatic=1,+ ## compile with -g+ copts=["-g"]+)\ No newline at end of filediff --git a/include/text_embedder_tokenizer.h b/include/text_embedder_tokenizer.hindex 08667806..cb07c50b 100644--- a/include/text_embedder_tokenizer.h+++ b/include/text_embedder_tokenizer.h@@ -4,6 +4,7 @@#include <unordered_map>#include <sentencepiece_processor.h>#include <tokenizer/bert_tokenizer.hpp>+#include <clip_tokenizer.h>#include <core/session/onnxruntime_cxx_api.h>#include <mutex>@@ -74,13 +75,12 @@ class XLMRobertaTokenizer : public TextEmbeddingTokenizer {}};-class CLIPTokenizer : public TextEmbeddingTokenizer {+class CLIPTokenizerWrapper : public TextEmbeddingTokenizer {private:- std::unique_ptr<Ort::Session> session_;- Ort::Env env_;+ std::unique_ptr<CLIPTokenizer> clip_tokenizer_;std::mutex mutex_;public:- CLIPTokenizer(const std::string& model_path);+ CLIPTokenizerWrapper(const std::string& vocab_path);encoded_input_t Encode(const std::string& text) override;virtual TokenizerType get_tokenizer_type() override {return TokenizerType::clip;diff --git a/src/embedder_manager.cpp b/src/embedder_manager.cppindex 81a1c2ed..91e56c0a 100644--- a/src/embedder_manager.cpp+++ b/src/embedder_manager.cpp@@ -62,7 +62,7 @@ Option<bool> EmbedderManager::validate_and_init_local_model(const nlohmann::json}std::string abs_path = EmbedderManager::get_absolute_model_path(- EmbedderManager::get_model_name_without_namespace(model_name));+ EmbedderManager::get_model_name_without_namespace(model_name));if(!std::filesystem::exists(abs_path)) {LOG(ERROR) << "Model file not found: " << abs_path;diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex e81d0577..b6780505 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -46,7 +46,7 @@ TextEmbedder::TextEmbedder(const std::string& model_name) {else if(tokenizer_type == TokenizerType::xlm_roberta) {tokenizer_ = std::make_unique<XLMRobertaTokenizer>(vocab_path);} else if(tokenizer_type == TokenizerType::clip) {- tokenizer_ = std::make_unique<CLIPTokenizer>(EmbedderManager::get_model_subdir(model_name));+ tokenizer_ = std::make_unique<CLIPTokenizerWrapper>(vocab_path);output_tensor_name = "text_embeds";num_dim = 512;return;diff --git a/src/text_embedder_tokenizer.cpp b/src/text_embedder_tokenizer.cppindex 991c833d..764af831 100644--- a/src/text_embedder_tokenizer.cpp+++ b/src/text_embedder_tokenizer.cpp@@ -2,6 +2,7 @@#include <sstream>#include "text_embedder_tokenizer.h"#include "logger.h"+#include <unicode/normalizer2.h>BertTokenizerWrapper::BertTokenizerWrapper(const std::string& vocab_path) {@@ -89,36 +90,21 @@ encoded_input_t XLMRobertaTokenizer::Encode(const std::string& text) {}-CLIPTokenizer::CLIPTokenizer(const std::string& model_path) {- Ort::SessionOptions session_options;- session_options.EnableOrtCustomOps();- auto tokenizer_path= model_path + "/clip_tokenizer.onnx";- LOG(INFO) << "Loading tokenizer from " << tokenizer_path;- session_ = std::make_unique<Ort::Session>(env_, tokenizer_path.c_str(), session_options);+CLIPTokenizerWrapper::CLIPTokenizerWrapper(const std::string& vocab_path) {+ try {+ clip_tokenizer_ = std::make_unique<CLIPTokenizer>(vocab_path);+ } catch (const std::exception& e) {+ LOG(INFO) << "Failed to load CLIP tokenizer: " << e.what();+ throw;+ }}-encoded_input_t CLIPTokenizer::Encode(const std::string& text) {- std::unique_lock<std::mutex> lock(mutex_);-- Ort::AllocatorWithDefaultOptions allocator;- Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);- std::vector<Ort::Value> input_tensors;- std::vector<int64_t> input_shape = {1};- std::vector<const char*> input_names = {"string_input"};- const char* const input_array[] = {text.c_str()};-- Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, input_shape.data(), input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING);- input_tensor.FillStringTensor(input_array, 1U);- input_tensors.push_back(std::move(input_tensor));-- const std::vector<const char*> output_names = {"input_ids", "attention_mask"};- auto output_tensors = session_->Run(Ort::RunOptions{nullptr}, input_names.data(), input_tensors.data(), input_tensors.size(), output_names.data(), output_names.size());- auto input_ids_tensor = output_tensors[0].GetTensorMutableData<int64_t>();- auto attention_mask_tensor = output_tensors[1].GetTensorMutableData<int64_t>();+encoded_input_t CLIPTokenizerWrapper::Encode(const std::string& text) {+ auto res = clip_tokenizer_->tokenize({text});- auto input_ids = std::vector<int64_t>(input_ids_tensor, input_ids_tensor + output_tensors[0].GetTensorTypeAndShapeInfo().GetElementCount());- auto attention_mask = std::vector<int64_t>(attention_mask_tensor, attention_mask_tensor + output_tensors[1].GetTensorTypeAndShapeInfo().GetElementCount());+ // convert vector int to vector int64_t+ std::vector<int64_t> input_ids(res.tokens[0].begin(), res.tokens[0].end());+ std::vector<int64_t> attention_mask(res.attention_mask[0].begin(), res.attention_mask[0].end());-- return {std::move(input_ids), {}, std::move(attention_mask)};+ return {input_ids, {}, attention_mask};}\ No newline at end of filediff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 9c0d8a02..cac7f6ec 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -3073,4 +3073,73 @@ TEST_F(CollectionVectorTest, TestInvalidImage) {ASSERT_EQ(add_op.error(), "Error while processing image");+}+++TEST_F(CollectionVectorTest, TestCLIPTokenizerUnicode) {+ auto schema_json =+ R"({+ "name": "Images",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "image", "type": "image", "store": false},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["image"], "model_config": {"model_name": "ts/clip-vit-b-p32"}}}+ ]+ })"_json;+++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ // test english+ auto results = coll->search("dog", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test chinese+ results = coll->search("狗", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test japanese+ results = coll->search("犬", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test korean+ results = coll->search("개", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test russian+ results = coll->search("собака", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test arabic+ results = coll->search("كلب", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // test turkish+ results = coll->search("kö", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ results = coll->search("öğ", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();+}\ No newline at end of file