Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Auto vector generation & Hybrid Search
ozanarmagan4f921a3040f1
diff --git a/BUILD b/BUILDindex c7fde44a..d16eba97 100644--- a/BUILD+++ b/BUILD@@ -155,7 +155,6 @@ mkdir -p $INSTALLDIR/lib/_deps/pytorch_cpuinfo-build/deps/clogmkdir -p $INSTALLDIR/lib/_deps/google_nsync-buildcp $BUILD_TMPDIR/_deps/onnx-build/libonnx.a $INSTALLDIR/lib/_deps/onnx-buildcp $BUILD_TMPDIR/_deps/onnx-build/libonnx_proto.a $INSTALLDIR/lib/_deps/onnx-build-cp $BUILD_TMPDIR/_deps/protobuf-build/libprotobuf-lite.a $INSTALLDIR/lib/_deps/protobuf-buildcp $BUILD_TMPDIR/_deps/re2-build/libre2.a $INSTALLDIR/lib/_deps/re2-buildcp $BUILD_TMPDIR/_deps/abseil_cpp-build/absl/base/libabsl_base.a $INSTALLDIR/lib/_deps/abseil_cpp-build/absl/basecp $BUILD_TMPDIR/_deps/abseil_cpp-build/absl/base/libabsl_throw_delegate.a $INSTALLDIR/lib/_deps/abseil_cpp-build/absl/base@@ -243,7 +242,6 @@ cmake("libonnxruntime_flatbuffers.a","_deps/onnx-build/libonnx.a","_deps/onnx-build/libonnx_proto.a",- "_deps/protobuf-build/libprotobuf-lite.a","_deps/re2-build/libre2.a","_deps/abseil_cpp-build/absl/base/libabsl_base.a","_deps/abseil_cpp-build/absl/base/libabsl_throw_delegate.a",@@ -260,6 +258,10 @@ cmake(cc_library(name = "onnxruntime_lib",- deps = ["//:onnxruntime", "@onnx_runtime//:ext_headers"],+ linkopts = select({+ "@platforms//os:linux": ["-static-libstdc++", "-static-libgcc"],+ "//conditions:default": [],+ }),+ deps = ["//:onnxruntime", "@onnx_runtime_extensions//:operators"],includes= ["onnxruntime/include/onnxruntime"])diff --git a/CMakeLists.txt b/CMakeLists.txtindex dccc127f..9893882b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -62,6 +62,7 @@ ELSE()ENDIF()include(cmake/onnxruntime.cmake)+include(cmake/onnxruntime_ext.cmake)include(cmake/For.cmake)include(cmake/H2O.cmake)include(cmake/RocksDB.cmake)@@ -110,6 +111,8 @@ include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/include)include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/data)include_directories(${DEP_ROOT_DIR}/${HNSW_NAME})include_directories(${DEP_ROOT_DIR}/${ONNX_NAME}/include/onnxruntime)+include_directories(${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir)+link_directories(/usr/local/lib)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)@@ -236,4 +239,20 @@ target_link_libraries(search ${CORE_LIBS})target_link_libraries(benchmark ${CORE_LIBS})target_link_libraries(typesense-test ${CORE_LIBS} gtest gtest_main)-add_dependencies(typesense-server onnxruntime)\ No newline at end of file+add_dependencies(typesense-server onnxruntime)+add_dependencies(typesense-test onnxruntime)+add_dependencies(benchmark onnxruntime)+add_dependencies(search onnxruntime)++# add source files from ${DEP_ROOT_DIR}/${ONNX_EXT_NAME} directory to targets+set(ONNX_EXT_SRC_FILES ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir/ustring.cc ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir/string_utils_onnx.cc ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir/base64.cc ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir/tokenizer/bert_tokenizer.cc ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}/operators/src_dir/tokenizer/basic_tokenizer.cc)+set_source_files_properties(${ONNX_EXT_SRC_FILES} PROPERTIES GENERATED TRUE)+target_sources(typesense-server PRIVATE ${ONNX_EXT_SRC_FILES})+target_sources(typesense-test PRIVATE ${ONNX_EXT_SRC_FILES})+target_sources(benchmark PRIVATE ${ONNX_EXT_SRC_FILES})+target_sources(search PRIVATE ${ONNX_EXT_SRC_FILES})++add_dependencies(typesense-server onnxruntime_ext)+add_dependencies(typesense-test onnxruntime_ext)+add_dependencies(benchmark onnxruntime_ext)+add_dependencies(search onnxruntime_ext)\ No newline at end of filediff --git a/WORKSPACE b/WORKSPACEindex c8ff47e1..1dc1401d 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -13,10 +13,7 @@ bazel_compdb_deps()http_archive(name = "rules_foreign_cc",- patches = ["//bazel:foreign_cc.patch", "//bazel:foreign_cc_version_compiler.patch"],- patch_args = [- "-p1",- ],+ patches = ["//bazel:foreign_cc.patch"],sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",strip_prefix = "rules_foreign_cc-0.9.0",url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",@@ -41,31 +38,23 @@ git_repository(remote = "https://github.com/apache/incubator-brpc.git",)-_ALL_CONTENT = """\-filegroup(- name = "all_srcs",- srcs = glob(["**"]),- visibility = ["//visibility:public"],-)---cc_library(- name = "ext_headers",- hdrs = glob(["cmake/external/onnxruntime-extensions/includes/**"]),- strip_include_prefix = "cmake/external/onnxruntime-extensions/includes",- visibility = ["//visibility:public"],-)---"""new_git_repository(name="onnx_runtime",- branch= "main",- build_file_content= _ALL_CONTENT,+ branch= "rel-1.14.0",+ build_file = "//bazel:onnxruntime.BUILD",init_submodules= 1,recursive_init_submodules= 1,- remote= "https://github.com/microsoft/onnxruntime"+ remote= "https://github.com/microsoft/onnxruntime",+ patches=["//bazel:onnx.patch"],+)++new_git_repository(+ name = "onnx_runtime_extensions",+ build_file = "//bazel:onnxruntime_extensions.BUILD",+ remote = "https://github.com/microsoft/onnxruntime-extensions",+ commit = "81e7799c69044c745239202085eb0a98f102937b",+ patches=["//bazel:onnx_ext.patch"],)new_git_repository(diff --git a/bazel/foreign_cc.patch b/bazel/foreign_cc.patchindex 9d843971..b7ba6ef8 100644--- a/bazel/foreign_cc.patch+++ b/bazel/foreign_cc.patch@@ -1,5 +1,5 @@---- a/foreign_cc/private/configure_script.bzl-+++ b/foreign_cc/private/configure_script.bzl+--- foreign_cc/private/configure_script.bzl++++ foreign_cc/private/configure_script.bzl@@ -70,7 +70,7 @@).lstrip())@@ -43,3 +43,253 @@feature_configuration = feature_configuration,action_name = ACTION_NAMES.cpp_link_static_library,+diff --git a/toolchains/built_toolchains.bzl b/toolchains/built_toolchains.bzl+index 5e59e79..ddf63a5 100644+--- toolchains/built_toolchains.bzl++++ toolchains/built_toolchains.bzl+@@ -28,6 +28,7 @@ _CMAKE_SRCS = {+ "3.22.4": [["https://github.com/Kitware/CMake/releases/download/v3.22.4/cmake-3.22.4.tar.gz"], "cmake-3.22.4", "5c55d0b0bc4c191549e3502b8f99a4fe892077611df22b4178cc020626e22a47"],+ "3.23.1": [["https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"], "cmake-3.23.1", "33fd10a8ec687a4d0d5b42473f10459bb92b3ae7def2b745dc10b192760869f3"],+ "3.23.2": [["https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz"], "cmake-3.23.2", "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa"],++ "3.25.0": [["https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0.tar.gz"], "cmake-3.25.0", "306463f541555da0942e6f5a0736560f70c487178b9d94a5ae7f34d0538cdd48"],+ }++ # buildifier: disable=unnamed-macro+@@ -438,6 +439,18 @@ def _ninja_toolchain(version, register_toolchains):+ native.register_toolchains(+ "@rules_foreign_cc//toolchains:built_ninja_toolchain",+ )++ if version == "1.11.1":++ maybe(++ http_archive,++ name = "ninja_build_src",++ build_file_content = _ALL_CONTENT,++ sha256 = "31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea",++ strip_prefix = "ninja-1.11.1",++ urls = [++ "https://github.com/ninja-build/ninja/archive/v1.11.1.tar.gz",++ ],++ )++ return+ if version == "1.11.0":+ maybe(+ http_archive,+diff --git a/toolchains/prebuilt_toolchains.bzl b/toolchains/prebuilt_toolchains.bzl+index dabfb95..d9c38b4 100644+--- toolchains/prebuilt_toolchains.bzl++++ toolchains/prebuilt_toolchains.bzl+@@ -67,6 +67,115 @@ def prebuilt_toolchains(cmake_version, ninja_version, register_toolchains):+ _make_toolchains(register_toolchains)++ def _cmake_toolchains(version, register_toolchains):++ if "3.25.0" == version:++ maybe(++ http_archive,++ name = "cmake-3.25.0-linux-aarch64",++ urls = [++ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-aarch64.tar.gz",++ ],++ sha256 = "27da36d6debe9b30f5c498554ae40cd621a55736f5f2ae2618ed95722a59965a",++ strip_prefix = "cmake-3.25.0-linux-aarch64",++ build_file_content = _CMAKE_BUILD_FILE.format(++ bin = "cmake",++ env = "{}",++ ),++ )++++ maybe(++ http_archive,++ name = "cmake-3.25.0-linux-x86_64",++ urls = [++ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-x86_64.tar.gz",++ ],++ sha256 = "ac634d6f0a81d7089adc7be5acff66a6bee3b08615f9a947858ce92a9ef59c8b",++ strip_prefix = "cmake-3.25.0-linux-x86_64",++ build_file_content = _CMAKE_BUILD_FILE.format(++ bin = "cmake",++ env = "{}",++ ),++ )++++ maybe(++ http_archive,++ name = "cmake-3.25.0-macos-universal",++ urls = [++ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-macos-universal.tar.gz",++ ],++ sha256 = "c088e761534a2078cd9d0581d39f02d3f9ed05302e33135b55c6d619b263b4c3",++ strip_prefix = "cmake-3.25.0-macos-universal/CMake.app/Contents",++ build_file_content = _CMAKE_BUILD_FILE.format(++ bin = "cmake",++ env = "{}",++ ),++ )++++ maybe(++ http_archive,++ name = "cmake-3.25.0-windows-i386",++ urls = [++ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-i386.zip",++ ],++ sha256 = "ddd115257a19ff3dd18fc63f32a00ae742f8b62d2e39bc354629903512f99783",++ strip_prefix = "cmake-3.25.0-windows-i386",++ build_file_content = _CMAKE_BUILD_FILE.format(++ bin = "cmake.exe",++ env = "{}",++ ),++ )++++ maybe(++ http_archive,++ name = "cmake-3.25.0-windows-x86_64",++ urls = [++ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-x86_64.zip",++ ],++ sha256 = "b46030c10cab1170355952f9ac59f7e6dabc248070fc53f15dff11d4ed2910f8",++ strip_prefix = "cmake-3.25.0-windows-x86_64",++ build_file_content = _CMAKE_BUILD_FILE.format(++ bin = "cmake.exe",++ env = "{}",++ ),++ )++++ # buildifier: leave-alone++ maybe(++ prebuilt_toolchains_repository,++ name = "cmake_3.25.0_toolchains",++ repos = {++ "cmake-3.25.0-linux-aarch64": [++ "@platforms//cpu:aarch64",++ "@platforms//os:linux",++ ],++ "cmake-3.25.0-linux-x86_64": [++ "@platforms//cpu:x86_64",++ "@platforms//os:linux",++ ],++ "cmake-3.25.0-macos-universal": [++ "@platforms//os:macos",++ ],++ "cmake-3.25.0-windows-i386": [++ "@platforms//cpu:x86_32",++ "@platforms//os:windows",++ ],++ "cmake-3.25.0-windows-x86_64": [++ "@platforms//cpu:x86_64",++ "@platforms//os:windows",++ ],++ },++ tool = "cmake",++ )++++ if register_toolchains:++ native.register_toolchains(++ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-aarch64_toolchain",++ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-x86_64_toolchain",++ "@cmake_3.25.0_toolchains//:cmake-3.25.0-macos-universal_toolchain",++ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-i386_toolchain",++ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-x86_64_toolchain",++ )++++ return+ if "3.23.2" == version:+ maybe(+ http_archive,+@@ -4196,6 +4305,78 @@ def _cmake_toolchains(version, register_toolchains):+ fail("Unsupported version: " + str(version))++ def _ninja_toolchains(version, register_toolchains):++ if "1.11.1" == version:++ maybe(++ http_archive,++ name = "ninja_1.11.1_linux",++ urls = [++ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip",++ ],++ sha256 = "b901ba96e486dce377f9a070ed4ef3f79deb45f4ffe2938f8e7ddc69cfb3df77",++ strip_prefix = "",++ build_file_content = _NINJA_BUILD_FILE.format(++ bin = "ninja",++ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",++ ),++ )++++ maybe(++ http_archive,++ name = "ninja_1.11.1_mac",++ urls = [++ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip",++ ],++ sha256 = "482ecb23c59ae3d4f158029112de172dd96bb0e97549c4b1ca32d8fad11f873e",++ strip_prefix = "",++ build_file_content = _NINJA_BUILD_FILE.format(++ bin = "ninja",++ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",++ ),++ )++++ maybe(++ http_archive,++ name = "ninja_1.11.1_win",++ urls = [++ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip",++ ],++ sha256 = "524b344a1a9a55005eaf868d991e090ab8ce07fa109f1820d40e74642e289abc",++ strip_prefix = "",++ build_file_content = _NINJA_BUILD_FILE.format(++ bin = "ninja.exe",++ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",++ ),++ )++++ # buildifier: leave-alone++ maybe(++ prebuilt_toolchains_repository,++ name = "ninja_1.11.1_toolchains",++ repos = {++ "ninja_1.11.1_linux": [++ "@platforms//cpu:x86_64",++ "@platforms//os:linux",++ ],++ "ninja_1.11.1_mac": [++ "@platforms//cpu:x86_64",++ "@platforms//os:macos",++ ],++ "ninja_1.11.1_win": [++ "@platforms//cpu:x86_64",++ "@platforms//os:windows",++ ],++ },++ tool = "ninja",++ )++++ if register_toolchains:++ native.register_toolchains(++ "@ninja_1.11.1_toolchains//:ninja_1.11.1_linux_toolchain",++ "@ninja_1.11.1_toolchains//:ninja_1.11.1_mac_toolchain",++ "@ninja_1.11.1_toolchains//:ninja_1.11.1_win_toolchain",++ )++++ return+ if "1.11.0" == version:+ maybe(+ http_archive,+diff --git a/toolchains/prebuilt_toolchains.py b/toolchains/prebuilt_toolchains.py+index 5288b27..a193021 100755+--- toolchains/prebuilt_toolchains.py++++ toolchains/prebuilt_toolchains.py+@@ -10,6 +10,7 @@ CMAKE_SHA256_URL_TEMPLATE = "https://cmake.org/files/v{minor}/cmake-{full}-SHA-2+ CMAKE_URL_TEMPLATE = "https://github.com/Kitware/CMake/releases/download/v{full}/{file}"++ CMAKE_VERSIONS = [++ "3.25.0",+ "3.23.2",+ "3.23.1",+ "3.22.4",+@@ -116,6 +117,7 @@ NINJA_TARGETS = {+ }++ NINJA_VERSIONS = (++ "1.11.1",+ "1.10.2",+ "1.10.1",+ "1.10.0",diff --git a/bazel/foreign_cc_version_compiler.patch b/bazel/foreign_cc_version_compiler.patchdeleted file mode 100644index d668a1db..00000000--- a/bazel/foreign_cc_version_compiler.patch+++ /dev/null@@ -1,283 +0,0 @@-diff --git a/foreign_cc/private/cc_toolchain_util.bzl b/foreign_cc/private/cc_toolchain_util.bzl-index fd7fa4d..188dc5f 100644---- a/foreign_cc/private/cc_toolchain_util.bzl-+++ b/foreign_cc/private/cc_toolchain_util.bzl-@@ -265,15 +265,24 @@ def get_tools_info(ctx):- cc_toolchain = cc_toolchain,- )--+ cxx = cc_common.get_tool_for_action(-+ feature_configuration = feature_configuration,-+ action_name = ACTION_NAMES.cpp_compile,-+ )-+ cxx_splitted = cxx.split("/")-+ if(cxx_splitted[-1] == "gcc"):-+ cxx_splitted[-1] = "g++"-+ cxx = "/".join(cxx_splitted)-+ if(cxx_splitted[-1] == "clang"):-+ cxx_splitted = "clang++"-+ cxx = "/".join(cxx_splitted)-+- return CxxToolsInfo(- cc = cc_common.get_tool_for_action(- feature_configuration = feature_configuration,- action_name = ACTION_NAMES.c_compile,- ),-- cxx = cc_common.get_tool_for_action(-- feature_configuration = feature_configuration,-- action_name = ACTION_NAMES.cpp_compile,-- ),-+ cxx = cxx,- cxx_linker_static = cc_common.get_tool_for_action(- feature_configuration = feature_configuration,- action_name = ACTION_NAMES.cpp_link_static_library,-diff --git a/toolchains/built_toolchains.bzl b/toolchains/built_toolchains.bzl-index 5e59e79..ddf63a5 100644---- a/toolchains/built_toolchains.bzl-+++ b/toolchains/built_toolchains.bzl-@@ -28,6 +28,7 @@ _CMAKE_SRCS = {- "3.22.4": [["https://github.com/Kitware/CMake/releases/download/v3.22.4/cmake-3.22.4.tar.gz"], "cmake-3.22.4", "5c55d0b0bc4c191549e3502b8f99a4fe892077611df22b4178cc020626e22a47"],- "3.23.1": [["https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"], "cmake-3.23.1", "33fd10a8ec687a4d0d5b42473f10459bb92b3ae7def2b745dc10b192760869f3"],- "3.23.2": [["https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz"], "cmake-3.23.2", "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa"],-+ "3.25.0": [["https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0.tar.gz"], "cmake-3.25.0", "306463f541555da0942e6f5a0736560f70c487178b9d94a5ae7f34d0538cdd48"],- }-- # buildifier: disable=unnamed-macro-@@ -438,6 +439,18 @@ def _ninja_toolchain(version, register_toolchains):- native.register_toolchains(- "@rules_foreign_cc//toolchains:built_ninja_toolchain",- )-+ if version == "1.11.1":-+ maybe(-+ http_archive,-+ name = "ninja_build_src",-+ build_file_content = _ALL_CONTENT,-+ sha256 = "31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea",-+ strip_prefix = "ninja-1.11.1",-+ urls = [-+ "https://github.com/ninja-build/ninja/archive/v1.11.1.tar.gz",-+ ],-+ )-+ return- if version == "1.11.0":- maybe(- http_archive,-diff --git a/toolchains/prebuilt_toolchains.bzl b/toolchains/prebuilt_toolchains.bzl-index dabfb95..d9c38b4 100644---- a/toolchains/prebuilt_toolchains.bzl-+++ b/toolchains/prebuilt_toolchains.bzl-@@ -67,6 +67,115 @@ def prebuilt_toolchains(cmake_version, ninja_version, register_toolchains):- _make_toolchains(register_toolchains)-- def _cmake_toolchains(version, register_toolchains):-+ if "3.25.0" == version:-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-linux-aarch64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-aarch64.tar.gz",-+ ],-+ sha256 = "27da36d6debe9b30f5c498554ae40cd621a55736f5f2ae2618ed95722a59965a",-+ strip_prefix = "cmake-3.25.0-linux-aarch64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-linux-x86_64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-x86_64.tar.gz",-+ ],-+ sha256 = "ac634d6f0a81d7089adc7be5acff66a6bee3b08615f9a947858ce92a9ef59c8b",-+ strip_prefix = "cmake-3.25.0-linux-x86_64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-macos-universal",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-macos-universal.tar.gz",-+ ],-+ sha256 = "c088e761534a2078cd9d0581d39f02d3f9ed05302e33135b55c6d619b263b4c3",-+ strip_prefix = "cmake-3.25.0-macos-universal/CMake.app/Contents",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-windows-i386",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-i386.zip",-+ ],-+ sha256 = "ddd115257a19ff3dd18fc63f32a00ae742f8b62d2e39bc354629903512f99783",-+ strip_prefix = "cmake-3.25.0-windows-i386",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake.exe",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-windows-x86_64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-x86_64.zip",-+ ],-+ sha256 = "b46030c10cab1170355952f9ac59f7e6dabc248070fc53f15dff11d4ed2910f8",-+ strip_prefix = "cmake-3.25.0-windows-x86_64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake.exe",-+ env = "{}",-+ ),-+ )-+-+ # buildifier: leave-alone-+ maybe(-+ prebuilt_toolchains_repository,-+ name = "cmake_3.25.0_toolchains",-+ repos = {-+ "cmake-3.25.0-linux-aarch64": [-+ "@platforms//cpu:aarch64",-+ "@platforms//os:linux",-+ ],-+ "cmake-3.25.0-linux-x86_64": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:linux",-+ ],-+ "cmake-3.25.0-macos-universal": [-+ "@platforms//os:macos",-+ ],-+ "cmake-3.25.0-windows-i386": [-+ "@platforms//cpu:x86_32",-+ "@platforms//os:windows",-+ ],-+ "cmake-3.25.0-windows-x86_64": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:windows",-+ ],-+ },-+ tool = "cmake",-+ )-+-+ if register_toolchains:-+ native.register_toolchains(-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-aarch64_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-x86_64_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-macos-universal_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-i386_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-x86_64_toolchain",-+ )-+-+ return- if "3.23.2" == version:- maybe(- http_archive,-@@ -4196,6 +4305,78 @@ def _cmake_toolchains(version, register_toolchains):- fail("Unsupported version: " + str(version))-- def _ninja_toolchains(version, register_toolchains):-+ if "1.11.1" == version:-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_linux",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip",-+ ],-+ sha256 = "b901ba96e486dce377f9a070ed4ef3f79deb45f4ffe2938f8e7ddc69cfb3df77",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_mac",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip",-+ ],-+ sha256 = "482ecb23c59ae3d4f158029112de172dd96bb0e97549c4b1ca32d8fad11f873e",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_win",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip",-+ ],-+ sha256 = "524b344a1a9a55005eaf868d991e090ab8ce07fa109f1820d40e74642e289abc",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja.exe",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ # buildifier: leave-alone-+ maybe(-+ prebuilt_toolchains_repository,-+ name = "ninja_1.11.1_toolchains",-+ repos = {-+ "ninja_1.11.1_linux": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:linux",-+ ],-+ "ninja_1.11.1_mac": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:macos",-+ ],-+ "ninja_1.11.1_win": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:windows",-+ ],-+ },-+ tool = "ninja",-+ )-+-+ if register_toolchains:-+ native.register_toolchains(-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_linux_toolchain",-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_mac_toolchain",-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_win_toolchain",-+ )-+-+ return- if "1.11.0" == version:- maybe(- http_archive,-diff --git a/toolchains/prebuilt_toolchains.py b/toolchains/prebuilt_toolchains.py-index 5288b27..a193021 100755---- a/toolchains/prebuilt_toolchains.py-+++ b/toolchains/prebuilt_toolchains.py-@@ -10,6 +10,7 @@ CMAKE_SHA256_URL_TEMPLATE = "https://cmake.org/files/v{minor}/cmake-{full}-SHA-2- CMAKE_URL_TEMPLATE = "https://github.com/Kitware/CMake/releases/download/v{full}/{file}"-- CMAKE_VERSIONS = [-+ "3.25.0",- "3.23.2",- "3.23.1",- "3.22.4",-@@ -116,6 +117,7 @@ NINJA_TARGETS = {- }-- NINJA_VERSIONS = (-+ "1.11.1",- "1.10.2",- "1.10.1",- "1.10.0",diff --git a/bazel/onnx.patch b/bazel/onnx.patchnew file mode 100644index 00000000..1afbddb2--- /dev/null+++ b/bazel/onnx.patch@@ -0,0 +1,12 @@+diff --git a/cmake/external/helper_functions.cmake b/cmake/external/helper_functions.cmake+index 88b46890b7..d090499971 100644+--- cmake/external/helper_functions.cmake++++ cmake/external/helper_functions.cmake+@@ -117,7 +117,6 @@ macro(onnxruntime_fetchcontent_makeavailable)+ ${__cmake_contentName}+ ${__cmake_contentNameLower}+ )+- find_package(${__cmake_contentName} ${__cmake_fpArgs})+ list(POP_BACK __cmake_fcCurrentNameStack+ __cmake_contentNameLower+ __cmake_contentNamediff --git a/bazel/onnx_ext.patch b/bazel/onnx_ext.patchnew file mode 100644index 00000000..f69648ef--- /dev/null+++ b/bazel/onnx_ext.patch@@ -0,0 +1,389 @@+diff --git a/operators/string_tensor.cc b/operators/string_tensor.cc+index 3d49e64..84975f6 100644+--- operators/string_tensor.cc++++ operators/string_tensor.cc+@@ -1,6 +1,6 @@+ // Copyright (c) Microsoft Corporation. All rights reserved.+ // Licensed under the MIT License.+-#include "string_utils.h"++#include "string_utils_onnx.h"+ #include "string_tensor.h"++ void GetTensorMutableDataString(const OrtApi& api, OrtW::CustomOpApi& ort, OrtKernelContext* context,+diff --git a/operators/string_utils.cc b/operators/string_utils_onnx.cc+similarity index 99%+rename from operators/string_utils.cc+rename to operators/string_utils_onnx.cc+index ecb6713..91dbe76 100644+--- operators/string_utils.cc++++ operators/string_utils_onnx.cc+@@ -2,7 +2,7 @@+ #include "farmhash.h"+ #endif++-#include "string_utils.h"++#include "string_utils_onnx.h"++ std::vector<std::string_view> SplitString(const std::string_view& str, const std::string_view& seps, bool remove_empty_entries) {+ std::vector<std::string_view> result;+diff --git a/operators/string_utils.h b/operators/string_utils_onnx.h+similarity index 89%+rename from operators/string_utils.h+rename to operators/string_utils_onnx.h+index 5653fbd..6556666 100644+--- operators/string_utils.h++++ operators/string_utils_onnx.h+@@ -4,7 +4,6 @@+ #include <iostream>+ #include <sstream>+ #include <vector>+-#include "ocos.h"++ template <typename T>+ inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept {+@@ -23,11 +22,6 @@ inline void MakeStringInternal(std::ostringstream& ss, const std::vector<int64_t+ ss << "]";+ }++-template <>+-inline void MakeStringInternal(std::ostringstream& ss, const OrtTensorDimensions& t) noexcept {+- MakeStringInternal(ss, static_cast<const std::vector<int64_t>&>(t));+-}+-+ template <>+ inline void MakeStringInternal(std::ostringstream& ss, const std::vector<std::string>& t) noexcept {+ ss << "[";+diff --git a/operators/tokenizer/basic_tokenizer.cc b/operators/tokenizer/basic_tokenizer.cc+index 324a774..00eac2b 100644+--- operators/tokenizer/basic_tokenizer.cc++++ operators/tokenizer/basic_tokenizer.cc+@@ -1,9 +1,8 @@+ // Copyright (c) Microsoft Corporation. All rights reserved.+ // Licensed under the MIT License.++-#include "string_utils.h"++#include "string_utils_onnx.h"+ #include "basic_tokenizer.hpp"+-#include "string_tensor.h"+ #include <vector>+ #include <locale>+ #include <codecvt>+@@ -81,52 +80,3 @@ std::vector<ustring> BasicTokenizer::Tokenize(ustring text) {+ push_current_token_and_clear();+ return result;+ }+-+-KernelBasicTokenizer::KernelBasicTokenizer(const OrtApi& api, const OrtKernelInfo* info) : BaseKernel(api, info) {+- bool do_lower_case = TryToGetAttributeWithDefault("do_lower_case", true);+- bool tokenize_chinese_chars = TryToGetAttributeWithDefault("tokenize_chinese_chars", true);+- bool strip_accents = TryToGetAttributeWithDefault("strip_accents", false);+- bool tokenize_punctuation = TryToGetAttributeWithDefault("tokenize_punctuation", false);+- bool remove_control_chars = TryToGetAttributeWithDefault("remove_control_chars", true);+-+- tokenizer_ = std::make_shared<BasicTokenizer>(do_lower_case, tokenize_chinese_chars, strip_accents, tokenize_punctuation, remove_control_chars);+-}+-+-void KernelBasicTokenizer::Compute(OrtKernelContext* context) {+- // Setup inputs+- const OrtValue* input = ort_.KernelContext_GetInput(context, 0);+- std::vector<std::string> input_data;+- GetTensorMutableDataString(api_, ort_, context, input, input_data);+-+- OrtTensorDimensions dimensions(ort_, input);+- if (dimensions.size() != 1 && dimensions[0] != 1) {+- ORTX_CXX_API_THROW("[BasicTokenizer]: only support string scalar.", ORT_INVALID_GRAPH);+- }+-+- OrtValue* output = ort_.KernelContext_GetOutput(context, 0, dimensions.data(), dimensions.size());+- std::vector<ustring> result = tokenizer_->Tokenize(ustring(input_data[0]));+-+- FillTensorDataString(api_, ort_, context, result, output);+-}+-+-void* CustomOpBasicTokenizer::CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const {+- return CreateKernelImpl(api, info);+-};+-+-const char* CustomOpBasicTokenizer::GetName() const { return "BasicTokenizer"; };+-+-size_t CustomOpBasicTokenizer::GetInputTypeCount() const {+- return 1;+-};+-+-ONNXTensorElementDataType CustomOpBasicTokenizer::GetInputType(size_t /*index*/) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING;+-};+-+-size_t CustomOpBasicTokenizer::GetOutputTypeCount() const {+- return 1;+-};+-+-ONNXTensorElementDataType CustomOpBasicTokenizer::GetOutputType(size_t /*index*/) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING;+-};+diff --git a/operators/tokenizer/basic_tokenizer.hpp b/operators/tokenizer/basic_tokenizer.hpp+index 046499e..9fd6f1a 100644+--- operators/tokenizer/basic_tokenizer.hpp++++ operators/tokenizer/basic_tokenizer.hpp+@@ -3,8 +3,7 @@++ #pragma once++-#include "ocos.h"+-#include "string_utils.h"++#include "string_utils_onnx.h"+ #include "ustring.h"++ class BasicTokenizer {+@@ -19,19 +18,3 @@ class BasicTokenizer {+ bool tokenize_punctuation_;+ bool remove_control_chars_;+ };+-+-struct KernelBasicTokenizer : BaseKernel {+- KernelBasicTokenizer(const OrtApi& api, const OrtKernelInfo* info);+- void Compute(OrtKernelContext* context);+- private:+- std::shared_ptr<BasicTokenizer> tokenizer_;+-};+-+-struct CustomOpBasicTokenizer : OrtW::CustomOpBase<CustomOpBasicTokenizer, KernelBasicTokenizer> {+- void* CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const;+- const char* GetName() const;+- size_t GetInputTypeCount() const;+- ONNXTensorElementDataType GetInputType(size_t index) const;+- size_t GetOutputTypeCount() const;+- ONNXTensorElementDataType GetOutputType(size_t index) const;+-};+diff --git a/operators/tokenizer/bert_tokenizer.cc b/operators/tokenizer/bert_tokenizer.cc+index b860ba6..9f43c5e 100644+--- operators/tokenizer/bert_tokenizer.cc++++ operators/tokenizer/bert_tokenizer.cc+@@ -33,7 +33,8 @@ int32_t BertTokenizerVocab::FindTokenId(const ustring& token) {++ auto it = vocab_.find(utf8_token);+ if (it == vocab_.end()) {+- ORTX_CXX_API_THROW("[BertTokenizerVocab]: can not find tokens: " + std::string(token), ORT_RUNTIME_EXCEPTION);++ std::cout << "[BertTokenizerVocab]: can not find tokens: " + std::string(token);++ return -1;+ }++ return it->second;+@@ -276,138 +277,3 @@ TruncateStrategy::TruncateStrategy(std::string_view strategy_name) : strategy_(T+ }+ }++-KernelBertTokenizer::KernelBertTokenizer(const OrtApi& api, const OrtKernelInfo* info) : BaseKernel(api, info) {+- std::string vocab = ort_.KernelInfoGetAttribute<std::string>(info, "vocab_file");+- bool do_lower_case = TryToGetAttributeWithDefault("do_lower_case", true);+- bool do_basic_tokenize = TryToGetAttributeWithDefault("do_basic_tokenize", true);+- std::string unk_token = TryToGetAttributeWithDefault("unk_token", std::string("[UNK]"));+- std::string sep_token = TryToGetAttributeWithDefault("sep_token", std::string("[SEP]"));+- std::string pad_token = TryToGetAttributeWithDefault("pad_token", std::string("[PAD]"));+- std::string cls_token = TryToGetAttributeWithDefault("cls_token", std::string("[CLS]"));+- std::string mask_token = TryToGetAttributeWithDefault("mask_token", std::string("[MASK]"));+- bool tokenize_chinese_chars = TryToGetAttributeWithDefault("tokenize_chinese_chars", true);+- bool strip_accents = TryToGetAttributeWithDefault("strip_accents", false);+- std::string suffix_indicator = TryToGetAttributeWithDefault("suffix_indicator", std::string("##"));+- std::string truncation_strategy_name = TryToGetAttributeWithDefault("truncation_strategy_name", std::string("longest_first"));+- int32_t max_len = static_cast<int32_t>(TryToGetAttributeWithDefault("max_length", int64_t(-1)));+-+- tokenizer_ = std::make_unique<BertTokenizer>(+- vocab, do_lower_case, do_basic_tokenize, ustring(unk_token),+- ustring(sep_token), ustring(pad_token), ustring(cls_token),+- ustring(mask_token), tokenize_chinese_chars, strip_accents,+- ustring(suffix_indicator), max_len, truncation_strategy_name);+-}+-+-void KernelBertTokenizer::Compute(OrtKernelContext* context) {+- // Setup inputs+- const OrtValue* input = ort_.KernelContext_GetInput(context, 0);+- std::vector<std::string> input_data;+- GetTensorMutableDataString(api_, ort_, context, input, input_data);+-+- if (input_data.size() != 1 && input_data.size() != 2) {+- ORTX_CXX_API_THROW("[BertTokenizer]: only support one or two query.", ORT_INVALID_GRAPH);+- }+- std::vector<int64_t> input_ids;+- std::vector<int64_t> token_type_ids;+-+- if (input_data.size() == 1) {+- std::vector<ustring> tokens = tokenizer_->Tokenize(ustring(input_data[0]));+- std::vector<int64_t> encoded = tokenizer_->Encode(tokens);+- tokenizer_->Truncate(encoded);+- input_ids = tokenizer_->AddSpecialToken(encoded);+- token_type_ids = tokenizer_->GenerateTypeId(encoded);+- } else {+- std::vector<ustring> tokens1 = tokenizer_->Tokenize(ustring(input_data[0]));+- std::vector<ustring> tokens2 = tokenizer_->Tokenize(ustring(input_data[1]));+- std::vector<int64_t> encoded1 = tokenizer_->Encode(tokens1);+- std::vector<int64_t> encoded2 = tokenizer_->Encode(tokens2);+- input_ids = tokenizer_->AddSpecialToken(encoded1, encoded2);+- token_type_ids = tokenizer_->GenerateTypeId(encoded1, encoded2);+- }+-+- std::vector<int64_t> attention_mask(input_ids.size(), 1);+-+- std::vector<int64_t> output_dim{static_cast<int64_t>(input_ids.size())};+-+- SetOutput(context, 0, output_dim, input_ids);+- SetOutput(context, 1, output_dim, token_type_ids);+- SetOutput(context, 2, output_dim, attention_mask);+-}+-+-void* CustomOpBertTokenizer::CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const {+- return CreateKernelImpl(api, info);+-}+-+-const char* CustomOpBertTokenizer::GetName() const { return "BertTokenizer"; }+-+-size_t CustomOpBertTokenizer::GetInputTypeCount() const {+- return 1;+-}+-+-ONNXTensorElementDataType CustomOpBertTokenizer::GetInputType(size_t /* index */) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING;+-}+-+-size_t CustomOpBertTokenizer::GetOutputTypeCount() const {+- return 3;+-}+-+-ONNXTensorElementDataType CustomOpBertTokenizer::GetOutputType(size_t /* index */) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64;+-}+-+-KernelHfBertTokenizer::KernelHfBertTokenizer(const OrtApi& api, const OrtKernelInfo* info) : KernelBertTokenizer(api, info) {}+-+-void KernelHfBertTokenizer::Compute(OrtKernelContext* context) {+- // Setup inputs+- const OrtValue *const input = ort_.KernelContext_GetInput(context, 0);+- std::vector<std::string> input_data;+- GetTensorMutableDataString(api_, ort_, context, input, input_data);+-+- if (input_data.size() != 2) {+- ORTX_CXX_API_THROW("[HfBertTokenizer]: Support only two input strings.", ORT_INVALID_GRAPH);+- }+-+- std::vector<ustring> tokens1 = tokenizer_->Tokenize(ustring(input_data[0]));+- std::vector<ustring> tokens2 = tokenizer_->Tokenize(ustring(input_data[1]));+- std::vector<int64_t> encoded1 = tokenizer_->Encode(tokens1);+- std::vector<int64_t> encoded2 = tokenizer_->Encode(tokens2);+- std::vector<int64_t> input_ids = tokenizer_->AddSpecialToken(encoded1, encoded2);+- std::vector<int64_t> token_type_ids = tokenizer_->GenerateTypeId(encoded1, encoded2);+- std::vector<int64_t> attention_mask(input_ids.size(), 1LL);+-+- const std::vector<int64_t> outer_dims{1LL, static_cast<int64_t>(input_ids.size())};+- const std::vector<int64_t> inner_dims{1LL};+- for (int32_t i = 0; i < 3; ++i) {+- OrtValue* const value = ort_.KernelContext_GetOutput(context, i, outer_dims.data(), outer_dims.size());+- OrtTensorTypeAndShapeInfo *const info = ort_.GetTensorTypeAndShape(value);+- ort_.SetDimensions(info, inner_dims.data(), inner_dims.size());+- ort_.ReleaseTensorTypeAndShapeInfo(info);+- }+-+- SetOutput(context, 0, outer_dims, input_ids);+- SetOutput(context, 1, outer_dims, attention_mask);+- SetOutput(context, 2, outer_dims, token_type_ids);+-}+-+-void* CustomOpHfBertTokenizer::CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const {+- return CreateKernelImpl(api, info);+-}+-+-const char* CustomOpHfBertTokenizer::GetName() const { return "HfBertTokenizer"; }+-+-size_t CustomOpHfBertTokenizer::GetInputTypeCount() const {+- return 1;+-}+-+-ONNXTensorElementDataType CustomOpHfBertTokenizer::GetInputType(size_t /* index */) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING;+-}+-+-size_t CustomOpHfBertTokenizer::GetOutputTypeCount() const {+- return 3;+-}+-+-ONNXTensorElementDataType CustomOpHfBertTokenizer::GetOutputType(size_t /* index */) const {+- return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64;+-}+diff --git a/operators/tokenizer/bert_tokenizer.hpp b/operators/tokenizer/bert_tokenizer.hpp+index 6dfcd84..10565e4 100644+--- operators/tokenizer/bert_tokenizer.hpp++++ operators/tokenizer/bert_tokenizer.hpp+@@ -3,12 +3,11 @@++ #pragma once+++#include <memory>+ #include <unordered_map>+ #include <vector>+-#include "ocos.h"+ #include "ustring.h"+-#include "string_utils.h"+-#include "string_tensor.h"++#include "string_utils_onnx.h"+ #include "basic_tokenizer.hpp"++ class BertTokenizerVocab final {+@@ -89,33 +88,4 @@ class BertTokenizer final {+ std::unique_ptr<WordpieceTokenizer> wordpiece_tokenizer_;+ };++-struct KernelBertTokenizer : BaseKernel {+- KernelBertTokenizer(const OrtApi& api, const OrtKernelInfo* info);+- void Compute(OrtKernelContext* context);++- protected:+- std::unique_ptr<BertTokenizer> tokenizer_;+-};+-+-struct CustomOpBertTokenizer : OrtW::CustomOpBase<CustomOpBertTokenizer, KernelBertTokenizer> {+- void* CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const;+- const char* GetName() const;+- size_t GetInputTypeCount() const;+- ONNXTensorElementDataType GetInputType(size_t index) const;+- size_t GetOutputTypeCount() const;+- ONNXTensorElementDataType GetOutputType(size_t index) const;+-};+-+-struct KernelHfBertTokenizer : KernelBertTokenizer {+- KernelHfBertTokenizer(const OrtApi& api, const OrtKernelInfo* info);+- void Compute(OrtKernelContext* context);+-};+-+-struct CustomOpHfBertTokenizer : OrtW::CustomOpBase<CustomOpHfBertTokenizer, KernelHfBertTokenizer> {+- void* CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const;+- const char* GetName() const;+- size_t GetInputTypeCount() const;+- ONNXTensorElementDataType GetInputType(size_t index) const;+- size_t GetOutputTypeCount() const;+- ONNXTensorElementDataType GetOutputType(size_t index) const;+-};+diff --git a/operators/tokenizer/bert_tokenizer_decoder.hpp b/operators/tokenizer/bert_tokenizer_decoder.hpp+index 5365fb0..cc3e106 100644+--- operators/tokenizer/bert_tokenizer_decoder.hpp++++ operators/tokenizer/bert_tokenizer_decoder.hpp+@@ -7,7 +7,7 @@+ #include <vector>+ #include "ocos.h"+ #include "ustring.h"+-#include "string_utils.h"++#include "string_utils_onnx.h"+ #include "string_tensor.h"+++diff --git a/operators/tokenizer/blingfire_sentencebreaker.hpp b/operators/tokenizer/blingfire_sentencebreaker.hpp+index 623f72b..d11b7b4 100644+--- operators/tokenizer/blingfire_sentencebreaker.hpp++++ operators/tokenizer/blingfire_sentencebreaker.hpp+@@ -4,7 +4,7 @@+ #pragma once++ #include "ocos.h"+-#include "string_utils.h"++#include "string_utils_onnx.h"++ extern "C" const int TextToSentencesWithOffsetsWithModel(+ const char* pInUtf8Str, int InUtf8StrByteCount,diff --git a/bazel/onnxruntime.BUILD b/bazel/onnxruntime.BUILDnew file mode 100644index 00000000..3cef44e7--- /dev/null+++ b/bazel/onnxruntime.BUILD@@ -0,0 +1,12 @@+filegroup(+ name = "all_srcs",+ srcs = glob(["**"]),+ visibility = ["//visibility:public"],+)++cc_library(+ name = "hdrs",+ hdrs = glob(["include/onnxruntime/**/*.h"]),+ strip_include_prefix = "include/onnxruntime",+ visibility = ["//visibility:public"],+)\ No newline at end of filediff --git a/bazel/onnxruntime_extensions.BUILD b/bazel/onnxruntime_extensions.BUILDnew file mode 100644index 00000000..a693d4aa--- /dev/null+++ b/bazel/onnxruntime_extensions.BUILD@@ -0,0 +1,25 @@+cc_library(+ name = "headers",+ hdrs = glob(["includes/**"]),+ strip_include_prefix = "includes",+ visibility = ["//visibility:public"],+ deps = [+ "@github_nlohmann_json//:json",+ ],+)++cc_library(+ name = "operators",+ hdrs = glob(["operators/base64.h",+ "operators/string_utils*",+ "operators/ustring.h",+ "operators/tokenizer/bert_tokenizer.hpp",+ "operators/tokenizer/basic_tokenizer.hpp"]),+ srcs = glob(["operators/base64.cc",+ "operators/string_utils*",+ "operators/ustring.cc",+ "operators/tokenizer/bert_tokenizer.cc",+ "operators/tokenizer/basic_tokenizer.cc"]),+ strip_include_prefix = "operators",+ visibility = ["//visibility:public"],+)diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmakeindex a27be6f7..23a64a5b 100644--- a/cmake/onnxruntime.cmake+++ b/cmake/onnxruntime.cmake@@ -10,9 +10,10 @@ endif()ExternalProject_Add(onnxruntimeGIT_REPOSITORY https://github.com/microsoft/onnxruntime- GIT_TAG origin/main+ GIT_TAG origin/rel-1.14.0SOURCE_DIR ${DEP_ROOT_DIR}/${ONNX_NAME}+ PATCH_COMMAND cd ${DEP_ROOT_DIR}/${ONNX_NAME} && sudo bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch.sh ${CMAKE_CURRENT_SOURCE_DIR}/cmake/onnx.patchBINARY_DIR ${DEP_ROOT_DIR}/${ONNX_NAME}-build- CONFIGURE_COMMAND ${CMAKE_COMMAND} ${DEP_ROOT_DIR}/${ONNX_NAME}/cmake -B${DEP_ROOT_DIR}/${ONNX_NAME}-build -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_GENERATE_TEST_REPORTS=ON -Donnxruntime_USE_MIMALLOC=OFF -Donnxruntime_ENABLE_PYTHON=OFF -Donnxruntime_BUILD_CSHARP=OFF -Donnxruntime_BUILD_JAVA=OFF -Donnxruntime_BUILD_NODEJS=OFF -Donnxruntime_BUILD_OBJC=OFF -Donnxruntime_BUILD_SHARED_LIB=OFF -Donnxruntime_BUILD_APPLE_FRAMEWORK=OFF -Donnxruntime_USE_DNNL=OFF -Donnxruntime_USE_NNAPI_BUILTIN=OFF -Donnxruntime_USE_RKNPU=OFF -Donnxruntime_USE_LLVM=OFF -Donnxruntime_ENABLE_MICROSOFT_INTERNAL=OFF -Donnxruntime_USE_VITISAI=OFF -Donnxruntime_USE_TENSORRT=OFF -Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=ON -Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=OFF -Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=OFF -Donnxruntime_USE_TVM=OFF -Donnxruntime_TVM_CUDA_RUNTIME=OFF -Donnxruntime_TVM_USE_HASH=OFF -Donnxruntime_USE_MIGRAPHX=OFF -Donnxruntime_CROSS_COMPILING=OFF -Donnxruntime_DISABLE_CONTRIB_OPS=OFF -Donnxruntime_DISABLE_ML_OPS=OFF -Donnxruntime_DISABLE_RTTI=OFF -Donnxruntime_DISABLE_EXCEPTIONS=OFF -Donnxruntime_MINIMAL_BUILD=OFF -Donnxruntime_EXTENDED_MINIMAL_BUILD=OFF -Donnxruntime_MINIMAL_BUILD_CUSTOM_OPS=OFF -Donnxruntime_REDUCED_OPS_BUILD=OFF -Donnxruntime_ENABLE_LANGUAGE_INTEROP_OPS=OFF -Donnxruntime_USE_DML=OFF -Donnxruntime_USE_WINML=OFF -Donnxruntime_BUILD_MS_EXPERIMENTAL_OPS=OFF -Donnxruntime_USE_TELEMETRY=OFF -Donnxruntime_ENABLE_LTO=OFF -Donnxruntime_USE_ACL=OFF -Donnxruntime_USE_ACL_1902=OFF -Donnxruntime_USE_ACL_1905=OFF -Donnxruntime_USE_ACL_1908=OFF -Donnxruntime_USE_ACL_2002=OFF -Donnxruntime_USE_ARMNN=OFF -Donnxruntime_ARMNN_RELU_USE_CPU=ON -Donnxruntime_ARMNN_BN_USE_CPU=ON -Donnxruntime_ENABLE_NVTX_PROFILE=OFF -Donnxruntime_ENABLE_TRAINING=OFF -Donnxruntime_ENABLE_TRAINING_OPS=OFF -Donnxruntime_ENABLE_TRAINING_APIS=OFF -Donnxruntime_ENABLE_CPU_FP16_OPS=OFF -Donnxruntime_USE_NCCL=OFF -Donnxruntime_BUILD_BENCHMARKS=OFF -Donnxruntime_USE_ROCM=OFF -Donnxruntime_GCOV_COVERAGE=OFF -Donnxruntime_USE_MPI=ON -Donnxruntime_ENABLE_MEMORY_PROFILE=OFF -Donnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO=OFF -Donnxruntime_BUILD_WEBASSEMBLY=OFF -Donnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING=ON -Donnxruntime_ENABLE_WEBASSEMBLY_API_EXCEPTION_CATCHING=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING=ON -Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_PROFILING=OFF -Donnxruntime_ENABLE_EAGER_MODE=OFF -Donnxruntime_ENABLE_LAZY_TENSOR=OFF -Donnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS=OFF -Donnxruntime_ENABLE_CUDA_PROFILING=OFF -Donnxruntime_ENABLE_ROCM_PROFILING=OFF -Donnxruntime_USE_XNNPACK=OFF -Donnxruntime_USE_CANN=OFF -DCMAKE_TLS_VERIFY=ON -DFETCHCONTENT_QUIET=OFF -Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF -Donnxruntime_ENABLE_MEMLEAK_CHECKER=OFF -DCMAKE_BUILD_TYPE=Release+ CONFIGURE_COMMAND ${CMAKE_COMMAND} ${DEP_ROOT_DIR}/${ONNX_NAME}/cmake -B${DEP_ROOT_DIR}/${ONNX_NAME}-build -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_GENERATE_TEST_REPORTS=ON -Donnxruntime_USE_MIMALLOC=OFF -Donnxruntime_ENABLE_PYTHON=OFF -Donnxruntime_BUILD_CSHARP=OFF -Donnxruntime_BUILD_JAVA=OFF -Donnxruntime_BUILD_NODEJS=OFF -Donnxruntime_BUILD_OBJC=OFF -Donnxruntime_BUILD_SHARED_LIB=OFF -Donnxruntime_BUILD_APPLE_FRAMEWORK=OFF -Donnxruntime_USE_DNNL=OFF -Donnxruntime_USE_NNAPI_BUILTIN=OFF -Donnxruntime_USE_RKNPU=OFF -Donnxruntime_USE_LLVM=OFF -Donnxruntime_ENABLE_MICROSOFT_INTERNAL=OFF -Donnxruntime_USE_VITISAI=OFF -Donnxruntime_USE_TENSORRT=OFF -Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=ON -Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=OFF -Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=OFF -Donnxruntime_USE_TVM=OFF -Donnxruntime_TVM_CUDA_RUNTIME=OFF -Donnxruntime_TVM_USE_HASH=OFF -Donnxruntime_USE_MIGRAPHX=OFF -Donnxruntime_CROSS_COMPILING=OFF -Donnxruntime_DISABLE_CONTRIB_OPS=OFF -Donnxruntime_DISABLE_ML_OPS=OFF -Donnxruntime_DISABLE_RTTI=OFF -Donnxruntime_DISABLE_EXCEPTIONS=OFF -Donnxruntime_MINIMAL_BUILD=OFF -Donnxruntime_EXTENDED_MINIMAL_BUILD=OFF -Donnxruntime_MINIMAL_BUILD_CUSTOM_OPS=OFF -Donnxruntime_REDUCED_OPS_BUILD=OFF -Donnxruntime_ENABLE_LANGUAGE_INTEROP_OPS=OFF -Donnxruntime_USE_DML=OFF -Donnxruntime_USE_WINML=OFF -Donnxruntime_BUILD_MS_EXPERIMENTAL_OPS=OFF -Donnxruntime_USE_TELEMETRY=OFF -Donnxruntime_ENABLE_LTO=OFF -Donnxruntime_USE_ACL=OFF -Donnxruntime_USE_ACL_1902=OFF -Donnxruntime_USE_ACL_1905=OFF -Donnxruntime_USE_ACL_1908=OFF -Donnxruntime_USE_ACL_2002=OFF -Donnxruntime_USE_ARMNN=OFF -Donnxruntime_ARMNN_RELU_USE_CPU=ON -Donnxruntime_ARMNN_BN_USE_CPU=ON -Donnxruntime_ENABLE_NVTX_PROFILE=OFF -Donnxruntime_ENABLE_TRAINING=OFF -Donnxruntime_ENABLE_TRAINING_OPS=OFF -Donnxruntime_ENABLE_TRAINING_APIS=OFF -Donnxruntime_ENABLE_CPU_FP16_OPS=OFF -Donnxruntime_USE_NCCL=OFF -Donnxruntime_BUILD_BENCHMARKS=OFF -Donnxruntime_USE_ROCM=OFF -Donnxruntime_GCOV_COVERAGE=OFF -Donnxruntime_USE_MPI=ON -Donnxruntime_ENABLE_MEMORY_PROFILE=OFF -Donnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO=OFF -Donnxruntime_BUILD_WEBASSEMBLY=OFF -Donnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING=ON -Donnxruntime_ENABLE_WEBASSEMBLY_API_EXCEPTION_CATCHING=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING=ON -Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO=OFF -Donnxruntime_ENABLE_WEBASSEMBLY_PROFILING=OFF -Donnxruntime_ENABLE_EAGER_MODE=OFF -Donnxruntime_ENABLE_LAZY_TENSOR=OFF -Donnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS=OFF -Donnxruntime_ENABLE_CUDA_PROFILING=OFF -Donnxruntime_ENABLE_ROCM_PROFILING=OFF -Donnxruntime_USE_XNNPACK=OFF -Donnxruntime_USE_CANN=OFF -DCMAKE_TLS_VERIFY=ON -DFETCHCONTENT_QUIET=OFF -Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF -Donnxruntime_ENABLE_MEMLEAK_CHECKER=OFF -Donnxruntime_BUILD_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=ReleaseBUILD_COMMAND ${CMAKE_COMMAND} --build ${DEP_ROOT_DIR}/${ONNX_NAME}-build --config Release -- -j8)diff --git a/cmake/onnxruntime_ext.cmake b/cmake/onnxruntime_ext.cmakenew file mode 100644index 00000000..03085433--- /dev/null+++ b/cmake/onnxruntime_ext.cmake@@ -0,0 +1,21 @@+project(onnxruntme_ext)+set(ONNX_EXT_NAME onnxruntime_ext)+include(ExternalProject)++if(NOT EXISTS ${DEP_ROOT_DIR}/${ONNX_EXT_NAME})+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${ONNX_EXT_NAME})+else()+ file(REMOVE_RECURSE ${DEP_ROOT_DIR}/${ONNX_EXT_NAME})+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${ONNX_EXT_NAME})+endif()++ExternalProject_Add(+ onnxruntime_ext+ GIT_REPOSITORY https://github.com/microsoft/onnxruntime-extensions+ GIT_TAG 81e7799c69044c745239202085eb0a98f102937b+ SOURCE_DIR ${DEP_ROOT_DIR}/${ONNX_EXT_NAME}+ PATCH_COMMAND cd ${DEP_ROOT_DIR}/${ONNX_EXT_NAME} && sudo bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch.sh ${CMAKE_CURRENT_SOURCE_DIR}/cmake/onnx_ext.patch && cd operators && mkdir -p src_dir && mkdir -p src_dir/tokenizer && cp base64.h base64.cc string_utils_onnx.h string_utils_onnx.cc ustring.h ustring.cc src_dir && cp tokenizer/bert_tokenizer.hpp tokenizer/bert_tokenizer.cc tokenizer/basic_tokenizer.hpp tokenizer/basic_tokenizer.cc src_dir/tokenizer+ CONFIGURE_COMMAND ""+ BUILD_COMMAND ""+ INSTALL_COMMAND ""+)diff --git a/cmake/patch.sh b/cmake/patch.shnew file mode 100644index 00000000..410c1254--- /dev/null+++ b/cmake/patch.sh@@ -0,0 +1,18 @@+#! /bin/sh++set +x+set -euo pipefail+++patch="$1"; shift++# ignore the error if the patch is already applied+if ! out=$(patch -p1 -N -r "rejects.bin" < "$patch")+then+ echo "$out" | grep -q "Reversed (or previously applied) patch detected! Skipping patch."+ test -s "rejects.bin" # Make sure we have rejects.+else+ test -f "rejects.bin" && ! test -s "rejects.bin" # Make sure we have no rejects.+fi++rm -f "rejects.bin"\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex f638b392..621d5387 100644--- a/include/collection.h+++ b/include/collection.h@@ -265,6 +265,8 @@ private:const spp::sparse_hash_set<std::string>& exclude_fields,tsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;++public:@@ -345,6 +347,8 @@ public:const DIRTY_VALUES dirty_values,const std::string& id="");+ Option<bool> embed_fields(nlohmann::json& document);+static uint32_t get_seq_id_from_key(const std::string & key);Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document, bool raw_doc = false) const;@@ -402,7 +406,7 @@ public:tsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;- Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,+ Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,size_t per_page = 10, size_t page = 1,diff --git a/include/field.h b/include/field.hindex af609fa1..6894dcee 100644--- a/include/field.h+++ b/include/field.h@@ -48,6 +48,8 @@ namespace fields {static const std::string num_dim = "num_dim";static const std::string vec_dist = "vec_dist";static const std::string reference = "reference";+ static const std::string create_from = "create_from";+ static const std::string model_path = "model_path";}enum vector_distance_type_t {@@ -73,6 +75,8 @@ struct field {int nested_array;size_t num_dim;+ std::vector<std::string> create_from;+ std::string model_path;vector_distance_type_t vec_dist;static constexpr int VAL_UNKNOWN = 2;@@ -83,9 +87,9 @@ struct field {field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "") :+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &create_from = {}, const std::string& model_path = "") :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference) {+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), create_from(create_from), model_path(model_path) {set_computed_defaults(sort, infix);}@@ -314,6 +318,12 @@ struct field {field_val[fields::reference] = field.reference;}+ if(field.create_from.size() > 0) {+ field_val[fields::create_from] = field.create_from;+ if(field.model_path.size() > 0) {+ field_val[fields::model_path] = field.model_path;+ }+ }fields_json.push_back(field_val);if(!field.has_valid_type()) {@@ -399,7 +409,7 @@ struct field {static Option<bool> json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,std::vector<field>& the_fields,- string& fallback_field_type, size_t& num_auto_detect_fields);+ string& fallback_field_type, size_t& num_auto_detect_fields,const nlohmann::json& all_fields_json = nlohmann::json());static Option<bool> json_fields_to_fields(bool enable_nested_fields,nlohmann::json& fields_json,@@ -410,7 +420,7 @@ struct field {for(nlohmann::json & field_json: fields_json) {auto op = json_field_to_field(enable_nested_fields,- field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ field_json, the_fields, fallback_field_type, num_auto_detect_fields, fields_json);if(!op.ok()) {return op;}diff --git a/include/http_client.h b/include/http_client.hindex a52aae5e..9d07f391 100644--- a/include/http_client.h+++ b/include/http_client.h@@ -24,6 +24,8 @@ private:static size_t curl_write_async_done(void* context, curl_socket_t item);+ static size_t curl_write_download(void *ptr, size_t size, size_t nmemb, FILE *stream);+static CURL* init_curl(const std::string& url, std::string& response);static CURL* init_curl_async(const std::string& url, deferred_req_res_t* req_res, curl_slist*& chunk);@@ -43,6 +45,8 @@ public:void init(const std::string & api_key);+ static long download_file(const std::string& url, const std::string& file_path);+static long get_response(const std::string& url, std::string& response,std::map<std::string, std::string>& res_headers, long timeout_ms=4000);diff --git a/include/text_embedder.h b/include/text_embedder.hnew file mode 100644index 00000000..d65b7345--- /dev/null+++ b/include/text_embedder.h@@ -0,0 +1,29 @@+#pragma once++#include <core/session/onnxruntime_cxx_api.h>+#include <tokenizer/bert_tokenizer.hpp>+#include <vector>++struct encoded_input_t {+ std::vector<int64_t> input_ids;+ std::vector<int64_t> token_type_ids;+ std::vector<int64_t> attention_mask;+};+++class TextEmbedder {+ public:+ TextEmbedder(const std::string& model_path);+ ~TextEmbedder();+ std::vector<float> Embed(const std::string& text);++ static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);+ private:+ Ort::Session* session_;+ Ort::Env env_;+ encoded_input_t Encode(const std::string& text);+ BertTokenizer* tokenizer_;+ static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);+ static std::string get_absolute_model_path(const std::string& model_path);+ std::string output_tensor_name;+};diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hnew file mode 100644index 00000000..4dd033e5--- /dev/null+++ b/include/text_embedder_manager.h@@ -0,0 +1,66 @@+#pragma once+++#include <unordered_map>++#include "text_embedder.h"++// singleton class+class TextEmbedderManager {+public:+ static TextEmbedderManager& get_instance() {+ static TextEmbedderManager instance;+ return instance;+ }++ TextEmbedderManager(TextEmbedderManager&&) = delete;+ TextEmbedderManager& operator=(TextEmbedderManager&&) = delete;+ TextEmbedderManager(const TextEmbedderManager&) = delete;+ TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;++ TextEmbedder* get_text_embedder(const std::string& model_path) {+ if (text_embedders.find(model_path) == text_embedders.end()) {+ text_embedders[model_path] = new TextEmbedder(model_path);+ }+ return text_embedders[model_path];+ }++ void delete_text_embedder(const std::string& model_path) {+ if (text_embedders.find(model_path) != text_embedders.end()) {+ delete text_embedders[model_path];+ text_embedders.erase(model_path);+ }+ }++ void delete_all_text_embedders() {+ for (auto& text_embedder : text_embedders) {+ delete text_embedder.second;+ }+ text_embedders.clear();+ }++ static void set_model_dir(const std::string& dir) {+ model_dir = dir;+ }++ static const std::string& get_model_dir() {+ return model_dir;+ }++ ~TextEmbedderManager() {+ delete_all_text_embedders();+ }++ static constexpr char* DEFAULT_MODEL_URL = "https://huggingface.co/ozanarmagan/e5-small-onnx/resolve/main/model.onnx";+ static constexpr char* DEFAULT_MODEL_NAME = "model.onnx";+ static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/ozanarmagan/e5-small-onnx/resolve/main/vocab.txt";+ static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";+ inline static std::string model_dir = "";+private:+ TextEmbedderManager() = default;+ std::unordered_map<std::string, TextEmbedder*> text_embedders;+};++++diff --git a/include/tsconfig.h b/include/tsconfig.hindex 59095649..0c02a188 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -11,6 +11,7 @@ class Config {private:std::string data_dir;std::string log_dir;+ std::string model_dir;std::string api_key;@@ -108,6 +109,10 @@ public:this->log_dir = log_dir;}+ void set_model_dir(const std::string & model_dir) {+ this->model_dir = model_dir;+ }+void set_api_key(const std::string & api_key) {this->api_key = api_key;}@@ -171,6 +176,10 @@ public:return this->log_dir;}+ std::string get_model_dir() const {+ return this->model_dir;+ }+std::string get_api_key() const {return this->api_key;}@@ -565,6 +574,10 @@ public:auto skip_writes_str = reader.Get("server", "skip-writes", "false");this->skip_writes = (skip_writes_str == "true");}++ if(reader.Exists("server", "model-dir")) {+ this->model_dir = reader.Get("server", "model-dir", "");+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -580,6 +593,11 @@ public:this->api_key = options.get<std::string>("api-key");}+ if(options.exist("model-dir")) {+ LOG(INFO) << "model-dir found";+ this->model_dir = options.get<std::string>("model-dir");+ }+// @deprecatedif(options.exist("search-only-api-key")) {this->search_only_api_key = options.get<std::string>("search-only-api-key");diff --git a/src/collection.cpp b/src/collection.cppindex 85965e6b..4c70bb39 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -16,6 +16,7 @@#include "logger.h"#include "thread_local_vars.h"#include "vector_query_ops.h"+#include "text_embedder_manager.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@ -71,6 +72,10 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::const std::string& id) {try {document = nlohmann::json::parse(json_str);+ auto embed_res = embed_fields(document);+ if (!embed_res.ok()) {+ return Option<doc_seq_id_t>(400, embed_res.error());+ }} catch(const std::exception& e) {LOG(ERROR) << "JSON error: " << e.what();return Option<doc_seq_id_t>(400, std::string("Bad JSON: ") + e.what());@@ -224,7 +229,15 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::sort] = coll_field.sort;field_json[fields::infix] = coll_field.infix;field_json[fields::locale] = coll_field.locale;++ if(coll_field.create_from.size() > 0) {+ field_json[fields::create_from] = coll_field.create_from;+ }+ if(coll_field.model_path.size() > 0) {+ field_json[fields::model_path] = coll_field.model_path;+ }+if(coll_field.num_dim > 0) {field_json[fields::num_dim] = coll_field.num_dim;}@@ -233,6 +246,7 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::reference] = coll_field.reference;}+fields_arr.push_back(field_json);}@@ -281,6 +295,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaconst std::string & json_line = json_lines[i];Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, dirty_values, id);+const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;index_record record(i, seq_id, document, operation, dirty_values);@@ -963,8 +978,9 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {bool exact_key_match = (kv.key().size() == field_name.size());bool exact_primitive_match = exact_key_match && !kv.value().is_object();+ bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().create_from.size() > 0;- if(extract_only_string_fields && !kv.value().is_string()) {+ if(extract_only_string_fields && !kv.value().is_string() && !text_embedding) {if(exact_primitive_match && !is_wildcard) {// upstream needs to be returned an errorreturn Option<bool>(400, "Field `" + field_name + "` should be a string or a string array.");@@ -973,7 +989,7 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,continue;}- if (exact_primitive_match || is_wildcard ||+ if (exact_primitive_match || is_wildcard || text_embedding ||// field_name prefix must be followed by a "." to indicate an object search(enable_nested_fields && kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.')) {processed_search_fields.push_back(kv.key());@@ -992,7 +1008,7 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,return Option<bool>(true);}-Option<nlohmann::json> Collection::search(const std::string & raw_query,+Option<nlohmann::json> Collection::search(std::string raw_query,const std::vector<std::string>& raw_search_fields,const std::string & filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,@@ -1113,10 +1129,12 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}++// validate search fieldsstd::vector<std::string> processed_search_fields;std::vector<uint32_t> query_by_weights;-+ bool has_embedding_query = false;for(size_t i = 0; i < raw_search_fields.size(); i++) {const std::string& field_name = raw_search_fields[i];if(field_name == "id") {@@ -1132,6 +1150,30 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}for(const auto& expanded_search_field: expanded_search_fields) {+ auto search_field = search_schema.at(expanded_search_field);++ if(search_field.num_dim > 0) {+ if(has_embedding_query) {+ std::string error = "Only one embedding field is allowed in the query.";+ return Option<nlohmann::json>(400, error);+ }++ if(TextEmbedderManager::model_dir.empty()) {+ std::string error = "Text embedding is not enabled. Please set `model-dir` at startup.";+ return Option<nlohmann::json>(400, error);+ }++ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();+ auto embedder = embedder_manager.get_text_embedder(search_field.model_path.size() > 0 ? search_field.model_path : TextEmbedderManager::DEFAULT_MODEL_NAME);++ std::vector<float> embedding = embedder->Embed(raw_query);+ vector_query._reset();+ vector_query.values = embedding;+ vector_query.field_name = field_name;+ has_embedding_query = true;+ continue;+ }+processed_search_fields.push_back(expanded_search_field);if(!raw_query_by_weights.empty()) {query_by_weights.push_back(raw_query_by_weights[i]);@@ -1139,14 +1181,19 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}+ std::string real_raw_query = raw_query;+ if(has_embedding_query && processed_search_fields.size() == 0) {+ raw_query = "*";+ }+if(!query_by_weights.empty() && processed_search_fields.size() != query_by_weights.size()) {std::string error = "Error, query_by_weights.size != query_by.size.";return Option<nlohmann::json>(400, error);}+for(const std::string & field_name: processed_search_fields) {field search_field = search_schema.at(field_name);-if(!search_field.index) {std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";return Option<nlohmann::json>(400, error);@@ -1593,7 +1640,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}nlohmann::json result = nlohmann::json::object();-result["found"] = total_found;if(exclude_fields.count("out_of") == 0) {@@ -1799,7 +1845,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,wrapper_doc["geo_distance_meters"] = geo_distances;}- if(!vector_query.field_name.empty()) {+ if(!vector_query.field_name.empty() && query == "*") {wrapper_doc["vector_distance"] = Index::int64_t_to_float(-field_order_kv->scores[0]);}@@ -1983,7 +2029,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,result["request_params"] = nlohmann::json::object();result["request_params"]["collection_name"] = name;result["request_params"]["per_page"] = per_page;- result["request_params"]["q"] = query;+ result["request_params"]["q"] = real_raw_query;//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";@@ -4586,3 +4632,33 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_hastd::shared_lock lock(mutex);return populate_include_exclude_fields(include_fields, exclude_fields, include_fields_full, exclude_fields_full);}+++Option<bool> Collection::embed_fields(nlohmann::json& document) {+ for(const auto& field : fields) {+ if(field.create_from.size() > 0) {+ if(TextEmbedderManager::model_dir.empty()) {+ return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");+ }+ std::string text_to_embed;+ for(const auto& field_name : field.create_from) {+ auto field_it = document.find(field_name);+ if(field_it != document.end()) {+ if(field_it->is_string()) {+ text_to_embed += field_it->get<std::string>() + " ";+ } else {+ return Option<bool>(400, "Field `" + field_name + "` is not a string.");+ }+ } else {+ return Option<bool>(400, "Field `" + field_name + "` not found in document.");+ }+ }++ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();+ auto embedder = embedder_manager.get_text_embedder(field.model_path.size() > 0 ? field.model_path : "model.onnx");+ std::vector<float> embedding = embedder->Embed(text_to_embed);+ document[field.name] = embedding;+ }+ }+ return Option<bool>(true);+}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 97a7954b..59ee80ff 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -58,6 +58,14 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::reference] = "";}+ if(field_obj.count(fields::create_from) == 0) {+ field_obj[fields::create_from] = std::vector<std::string>();+ }++ if(field_obj.count(fields::model_path) == 0) {+ field_obj[fields::model_path] = "";+ }+vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;if(field_obj.count(fields::vec_dist) != 0) {@@ -70,7 +78,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],- field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference]);+ field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::create_from],+ field_obj[fields::model_path]);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {@@ -200,7 +209,6 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sfor(size_t coll_index = 0; coll_index < num_collections; coll_index++) {const auto& collection_meta_json = collection_meta_jsons[coll_index];nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json, nullptr, false);-if(collection_meta.is_discarded()) {LOG(ERROR) << "Error while parsing collection meta, json: " << collection_meta_json;return Option<bool>(500, "Error while parsing collection meta.");diff --git a/src/field.cpp b/src/field.cppindex a23d0824..e73ee5bd 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -1,6 +1,7 @@#include <store.h>#include "field.h"#include "magic_enum.hpp"+#include "text_embedder_manager.h"#include <stack>#include <collection_manager.h>@@ -502,7 +503,7 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,std::vector<field>& the_fields,- string& fallback_field_type, size_t& num_auto_detect_fields) {+ string& fallback_field_type, size_t& num_auto_detect_fields, const nlohmann::json& all_fields_json) {if(field_json["name"] == "id") {// No field should exist with the name "id" as it is reserved for internal use@@ -662,6 +663,90 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}+ if(!field_json[fields::model_path].is_null() && field_json[fields::create_from].is_null()) {+ return Option<bool>(400, "Property `" + fields::model_path + "` can only be used with `" + fields::create_from + "`.");+ }++ if(field_json.count(fields::create_from) != 0) {+ if(TextEmbedderManager::model_dir.size() == 0) {+ return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");+ }++ if(!field_json[fields::create_from].is_array()) {+ return Option<bool>(400, "Property `" + fields::create_from + "` must be an array.");+ }++ if(field_json[fields::create_from].size() == 0) {+ return Option<bool>(400, "Property `" + fields::create_from + "` must have at least one element.");+ }++ for(auto& create_from_field : field_json[fields::create_from]) {+ if(!create_from_field.is_string()) {+ return Option<bool>(400, "Property `" + fields::create_from + "` must be an array of strings.");+ }+ }++ if(field_json[fields::type] != field_types::FLOAT_ARRAY) {+ return Option<bool>(400, "Property `" + fields::create_from + "` is only allowed on a float array field.");+ }+++ std::unordered_set<std::string> found_fields;+ for(auto& create_from_field : field_json[fields::create_from]) {+ bool flag = false;+ for(const auto& field : all_fields_json) {+ if(field[fields::name] == create_from_field) {+ if(field[fields::type] != field_types::STRING) {+ return Option<bool>(400, "Property `" + fields::create_from + "` can only be used with array of string fields.");+ }+ found_fields.insert(create_from_field.get<std::string>());+ flag = true;+ break;+ }+ }+ if(!flag) {+ for(const auto& field : the_fields) {+ if(field.name == create_from_field) {+ if(field.type != field_types::STRING) {+ return Option<bool>(400, "Property `" + fields::create_from + "` can only be used with array of string fields.");+ }+ found_fields.insert(create_from_field.get<std::string>());+ break;+ }+ }+ }+ }+ if(found_fields.size() != field_json[fields::create_from].size()) {+ return Option<bool>(400, "Property `" + fields::create_from + "` must be an array of existing string fields.");+ }++ // If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)+ field_json[fields::num_dim] = static_cast<unsigned int>(384);+ if(!field_json[fields::model_path].is_null()) {+ unsigned int num_dim = 0;+ if(!field_json[fields::model_path].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_path + "` must be a string.");+ }+ if(field_json[fields::model_path].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::model_path + "` must be a non-empty string.");+ }++ if(TextEmbedder::is_model_valid(field_json[fields::model_path].get<std::string>(), num_dim)) {+ field_json[fields::num_dim] = num_dim;+ } else {+ return Option<bool>(400, "Property `" + fields::model_path + "` must be a valid model path.");+ }+ }+ } else {+ field_json[fields::create_from] = std::vector<std::string>();+ }+++ if(field_json[fields::model_path].is_null()) {+ field_json[fields::model_path] = "";+ }++auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);if(field_json.count(fields::num_dim) == 0) {@@ -698,6 +783,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}+if(field_json.count(fields::optional) == 0) {// dynamic type fields are always optionalbool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);@@ -741,7 +827,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::optional], field_json[fields::index], field_json[fields::locale],field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,- field_json[fields::reference])+ field_json[fields::reference], field_json[fields::create_from].get<std::vector<std::string>>(),+ field_json[fields::model_path]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/http_client.cpp b/src/http_client.cppindex afc99e60..4bf9f397 100644--- a/src/http_client.cpp+++ b/src/http_client.cpp@@ -352,3 +352,52 @@ size_t HttpClient::curl_write(char *contents, size_t size, size_t nmemb, std::sts->append(contents, size*nmemb);return size*nmemb;}++size_t HttpClient::curl_write_download(void *ptr, size_t size, size_t nmemb, FILE *stream) {+ size_t written = fwrite(ptr, size, nmemb, stream);+ return written;+}++long HttpClient::download_file(const std::string& url, const std::string& file_path) {+ CURL *curl = curl_easy_init();+++ if(curl == nullptr) {+ return -1;+ }++ FILE *fp = fopen(file_path.c_str(), "wb");++ if(fp == nullptr) {+ LOG(ERROR) << "Unable to open file for writing: " << file_path;+ return -1;+ }++ curl_easy_setopt(curl, CURLOPT_URL, url.c_str());+ curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 4000);+ curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);++ // to allow self-signed certs+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);++ curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_download);+ // follow redirects+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);++ CURLcode res_code = curl_easy_perform(curl);++ if(res_code != CURLE_OK) {+ LOG(ERROR) << "Unable to download file: " << url << " to " << file_path << " - " << curl_easy_strerror(res_code);+ return -1;+ }+ long http_code = 0;+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);++ curl_easy_cleanup(curl);+ fclose(fp);++ return http_code;+}+diff --git a/src/index.cpp b/src/index.cppindex 0975f05e..e734b8b3 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3010,9 +3010,44 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conssort_order, field_values, geopoint_indices,curated_ids_sorted, all_result_ids, all_result_ids_len, groups_processed);+ LOG(INFO) << "all_result_ids_len: " << all_result_ids_len;+ if(!vector_query.field_name.empty()) {+ VectorFilterFunctor filterFunctor(seq_ids->uncompress(), seq_ids->num_ids());+ LOG(INFO) << "vector_query.field_name: " << vector_query.field_name;+ auto& field_vector_index = vector_index.at(vector_query.field_name);+ LOG(INFO) << "field_vector_index->vecdex->size():";+ std::vector<std::pair<float, size_t>> dist_labels;+ auto k = std::max<size_t>(vector_query.k, per_page * page);++ if(field_vector_index->distance_type == cosine) {+ std::vector<float> normalized_q(vector_query.values.size());+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);+ } else {+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);+ }++ for (const auto& dist_label : dist_labels) {+ uint32 seq_id = dist_label.second;++ auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :+ dist_label.first;++ auto score = (1.0 - vec_dist_score) * 100000000000.0;++ auto found = topster->kv_map.find(seq_id);++ if (found != topster->kv_map.end()) {+ found->second->scores[0] += score;+ }+ }++ }+/*auto timeMillis0 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin0).count();LOG(INFO) << "Time taken for multi-field aggregation: " << timeMillis0 << "ms";*/+}//LOG(INFO) << "topster size: " << topster->size;diff --git a/src/text_embedder.cpp b/src/text_embedder.cppnew file mode 100644index 00000000..976da0f5--- /dev/null+++ b/src/text_embedder.cpp@@ -0,0 +1,179 @@+#include "text_embedder.h"+#include "text_embedder_manager.h"+#include "logger.h"+#include <string>+#include <fstream>+#include <sstream>+#include <filesystem>++TextEmbedder::TextEmbedder(const std::string& model_path) {++ // create environment+ Ort::SessionOptions session_options;+ std::string abs_path = get_absolute_model_path(model_path);+ LOG(INFO) << "Loading model from: " << abs_path;+ session_ = new Ort::Session(env_, abs_path.c_str(), session_options);+ std::ifstream stream("vocab.txt");+ std::stringstream ss;+ ss << stream.rdbuf();+ auto vocab_ = ss.str();+ tokenizer_ = new BertTokenizer(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),+ ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));++ auto output_tensor_count = session_->GetOutputCount();+ bool found_output_tensor = false;+ for (int i = 0; i < output_tensor_count; i++) {+ auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {+ found_output_tensor = true;+ Ort::AllocatorWithDefaultOptions allocator;+ output_tensor_name = std::string(session_->GetOutputNameAllocated(i, allocator).get());+ break;+ }+ }+}++encoded_input_t TextEmbedder::Encode(const std::string& text) {++ auto encoded = tokenizer_->Encode(tokenizer_->Tokenize(ustring(text)));+ auto input_ids = tokenizer_->AddSpecialToken(encoded);+ auto token_type_ids = tokenizer_->GenerateTypeId(encoded);+ auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);+ return {input_ids, token_type_ids, attention_mask};+}+++std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<float>>& inputs) {++ std::vector<float> pooled_output;+ for (int i = 0; i < inputs[0].size(); i++) {+ float sum = 0;+ for (int j = 0; j < inputs.size(); j++) {+ sum += inputs[j][i];+ }+ pooled_output.push_back(sum / inputs.size());+ }+ return pooled_output;+}++std::vector<float> TextEmbedder::Embed(const std::string& text) {++ LOG(INFO) << "Embedding text: " << text;+ auto encoded_input = Encode(text);+ // create input tensor object from data values+ Ort::AllocatorWithDefaultOptions allocator;+ Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);+ std::vector<Ort::Value> input_tensors;+ std::vector<std::vector<int64_t>> input_shapes;+ std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));++ //LOG(INFO) << "Running model";+ // create output tensor object+ std::vector<const char*> output_node_names = {output_tensor_name.c_str()};+ std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size+ auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());+ std::vector<std::vector<float>> output;+ float* data = output_tensor[0].GetTensorMutableData<float>();+ // print output tensor shape+ auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();++ //LOG(INFO) << "Output tensor size: " << shape[0] << " x " << shape[1] << " x " << shape[2];+ for (int i = 0; i < shape[1]; i++) {+ std::vector<float> temp;+ for (int j = 0; j < shape[2]; j++) {+ temp.push_back(data[i * shape[2] + j]);+ }+ output.push_back(temp);+ }+ //LOG(INFO) << "Mean pooling";+ auto pooled_output = mean_pooling(output);++++ return pooled_output;+}++TextEmbedder::~TextEmbedder() {+ delete tokenizer_;+ delete session_;+}+++bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& num_dims) {+ LOG(INFO) << "Loading model: " << model_path;+ Ort::SessionOptions session_options;+ Ort::Env env;+ std::string abs_path = get_absolute_model_path(model_path);++ if(!std::filesystem::exists(abs_path)) {+ LOG(ERROR) << "Model file not found: " << abs_path;+ return false;+ }++ Ort::Session session(env, abs_path.c_str(), session_options);+ if(session.GetInputCount() != 3) {+ LOG(ERROR) << "Invalid model: input count is not 3";+ return false;+ }+ Ort::AllocatorWithDefaultOptions allocator;+ auto input_ids_name = session.GetInputNameAllocated(0, allocator);+ if (std::strcmp(input_ids_name.get(), "input_ids") != 0) {+ LOG(ERROR) << "Invalid model: input_ids tensor not found";+ return false;+ }++ auto attention_mask_name = session.GetInputNameAllocated(1, allocator);+ if (std::strcmp(attention_mask_name.get(), "attention_mask") != 0) {+ LOG(ERROR) << "Invalid model: attention_mask tensor not found";+ return false;+ }++ auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);+ if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {+ LOG(ERROR) << "Invalid model: token_type_ids tensor not found";+ return false;+ }++ auto output_tensor_count = session.GetOutputCount();+ bool found_output_tensor = false;+ for (int i = 0; i < output_tensor_count; i++) {+ auto shape = session.GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {+ num_dims = shape[2];+ found_output_tensor = true;+ break;+ }+ }++ if (!found_output_tensor) {+ LOG(ERROR) << "Invalid model: Output tensor not found";+ return false;+ }++ return true;+}++std::string TextEmbedder::get_absolute_model_path(const std::string & model_path) {+ const std::string model_dir = TextEmbedderManager::model_dir;++ if(model_dir.back() != '/') {+ if(model_path.front() != '/') {+ return model_dir + "/" + model_path;+ } else {+ return model_dir + model_path;+ }+ } else {+ if(model_path.front() != '/') {+ return model_dir + model_path;+ } else {+ return model_dir + "/" + model_path;+ }+ }+}+diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 57a79db2..5acc2220 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -16,6 +16,7 @@#include "core_api.h"#include "ratelimit_manager.h"+#include "text_embedder_manager.h"#include "typesense_server_utils.h"#include "file_utils.h"#include "threadpool.h"@@ -66,6 +67,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.set_program_name("./typesense-server");options.add<std::string>("data-dir", 'd', "Directory where data will be stored.", true);+ options.add<std::string>("model-dir", '\0', "Directory where text embedding models will be stored.", false, "");options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);@@ -452,6 +454,23 @@ int run_server(const Config & config, const std::string & version, void (*masterLOG(INFO) << "Failed to initialize rate limit manager: " << rate_limit_manager_init.error();}++ LOG(INFO) << "MODEL DIR: " << config.get_model_dir();+ if(config.get_model_dir().size() > 0) {+ LOG(INFO) << "Loading text embedding models from " << config.get_model_dir();+ TextEmbedderManager::get_instance().set_model_dir(config.get_model_dir());++ LOG(INFO) << "Downloading default model and vocab";+ long res = httpClient.download_file(TextEmbedderManager::DEFAULT_MODEL_URL, config.get_model_dir() + "/" + TextEmbedderManager::DEFAULT_MODEL_NAME);+ if(res != 200) {+ LOG(INFO) << "Failed to download default model: " << res;+ }++ res = httpClient.download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, config.get_model_dir() + "/" + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ if(res != 200) {+ LOG(INFO) << "Failed to download default vocab: " << res;+ }+ }// first we start the peering serviceReplicationState replication_state(server, batch_indexer, &store,diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex e3f7f961..8bc2a656 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -5,6 +5,8 @@#include <algorithm>#include <collection_manager.h>#include "collection.h"+#include "text_embedder_manager.h"+#include "http_client.h"class CollectionAllFieldsTest : public ::testing::Test {protected:@@ -27,6 +29,7 @@ protected:virtual void SetUp() {setupCollection();+ system("mkdir -p models");}virtual void TearDown() {@@ -59,6 +62,8 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {while (std::getline(infile, json_line)) {nlohmann::json document = nlohmann::json::parse(json_line);Option<nlohmann::json> add_op = coll1->add(document.dump());++ LOG(INFO) << "Add op: " << add_op.error();ASSERT_TRUE(add_op.ok());}@@ -1586,3 +1591,100 @@ TEST_F(CollectionAllFieldsTest, FieldNameMatchingRegexpShouldNotBeIndexedInNonAuASSERT_EQ(1, results["hits"].size());}+TEST_F(CollectionAllFieldsTest, CreateFromFieldJSONInvalidField) {+ TextEmbedderManager::model_dir = "./models";+ nlohmann::json field_json;+ field_json["name"] = "embedding";+ field_json["type"] = "float[]";+ field_json["create_from"] = {"name"};++ std::vector<field> fields;+ std::string fallback_field_type;+ size_t num_auto_detect_fields;++ auto field_op = field::json_field_to_field(false, field_json, fields, fallback_field_type, num_auto_detect_fields);++ ASSERT_FALSE(field_op.ok());+ ASSERT_EQ("Property `create_from` must be an array of existing string fields.", field_op.error());+}++TEST_F(CollectionAllFieldsTest, CreateFromFieldNoModelDir) {+ nlohmann::json field_json;+ field_json["name"] = "embedding";+ field_json["type"] = "float[]";+ field_json["create_from"] = {"name"};++ std::vector<field> fields;+ std::string fallback_field_type;+ size_t num_auto_detect_fields;++ auto field_op = field::json_field_to_field(false, field_json, fields, fallback_field_type, num_auto_detect_fields);++ ASSERT_FALSE(field_op.ok());+ ASSERT_EQ("Property `create_from` must be an array of existing string fields.", field_op.error());+}++TEST_F(CollectionAllFieldsTest, CreateFromNotArray) {+ TextEmbedderManager::model_dir = "./models";+ nlohmann::json field_json;+ field_json["name"] = "embedding";+ field_json["type"] = "float[]";+ field_json["create_from"] = "name";++ std::vector<field> fields;+ std::string fallback_field_type;+ size_t num_auto_detect_fields;++ auto field_op = field::json_field_to_field(false, field_json, fields, fallback_field_type, num_auto_detect_fields);++ ASSERT_FALSE(field_op.ok());+ ASSERT_EQ("Property `create_from` must be an array.", field_op.error());+}++TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {+ TextEmbedderManager::model_dir = "./models";+ nlohmann::json field_json;+ field_json["name"] = "embedding";+ field_json["type"] = "float[]";+ field_json["model_path"] = "model.onnx";++ std::vector<field> fields;+ std::string fallback_field_type;+ size_t num_auto_detect_fields;++ auto field_op = field::json_field_to_field(false, field_json, fields, fallback_field_type, num_auto_detect_fields);++ ASSERT_FALSE(field_op.ok());+ ASSERT_EQ("Property `model_path` can only be used with `create_from`.", field_op.error());+}+++TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);++ field embedding = field("embedding", field_types::FLOAT_ARRAY, false);+ embedding.create_from.push_back("name");+ std::vector<field> fields = {field("name", field_types::STRING, false),+ embedding};+ auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);++ ASSERT_TRUE(obj_coll_op.ok());+ Collection* obj_coll = obj_coll_op.get();++ nlohmann::json doc1;+ doc1["name"] = "One Two Three";++ auto add_res = obj_coll->add(doc1.dump());++ ASSERT_TRUE(add_res.ok());+ ASSERT_TRUE(add_res.get()["name"].is_string());+ ASSERT_TRUE(add_res.get()["embedding"].is_array());+ ASSERT_EQ(384, add_res.get()["embedding"].size());++ // delete models folder+ system("rm -rf ./models");+}+diff --git a/test/collection_test.cpp b/test/collection_test.cppindex f241e135..58ba40bf 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -5,6 +5,8 @@#include <algorithm>#include <collection_manager.h>#include "collection.h"+#include "text_embedder_manager.h"+#include "http_client.h"class CollectionTest : public ::testing::Test {protected:@@ -62,6 +64,7 @@ protected:virtual void SetUp() {setupCollection();+ system("mkdir -p models");}virtual void TearDown() {@@ -4605,4 +4608,153 @@ TEST_F(CollectionTest, WildcardHighlightFullFields) {ASSERT_EQ(0, result["hits"][0]["highlight"]["user"]["bio"].count("value"));ASSERT_EQ(0, result["hits"][0]["highlight"]["user_name"].count("value"));+}+++TEST_F(CollectionTest, SemanticSearchTest) {++ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();+ nlohmann::json object;+ object["name"] = "apple";+ auto add_op = coll->add(object.dump());+ ASSERT_TRUE(add_op.ok());++ ASSERT_EQ("apple", add_op.get()["name"]);+ ASSERT_EQ(384, add_op.get()["embedding"].size());++ spp::sparse_hash_set<std::string> dummy_include_exclude;++ auto search_res_op = coll->search("apple", {"embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");++ ASSERT_TRUE(search_res_op.ok());+ auto search_res = search_res_op.get();+ ASSERT_EQ(1, search_res["found"].get<size_t>());+ ASSERT_EQ(1, search_res["hits"].size());+ ASSERT_EQ("apple", search_res["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ(384, search_res["hits"][0]["document"]["embedding"].size());++ // delete models folder+ system("rm -rf ./models");+}++TEST_F(CollectionTest, InvalidSemanticSearch) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);++ auto op = collectionManager.create_collection(schema);+ LOG(INFO) << "op.error(): " << op.error();+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();+ nlohmann::json object;+ object["name"] = "apple";+ auto add_op = coll->add(object.dump());+ ASSERT_TRUE(add_op.ok());+ LOG(INFO) << "add_op.get(): " << add_op.get().dump();+ ASSERT_EQ("apple", add_op.get()["name"]);+ ASSERT_EQ(384, add_op.get()["embedding"].size());++ spp::sparse_hash_set<std::string> dummy_include_exclude;++ auto search_res_op = coll->search("apple", {"embedding", "embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");++ ASSERT_FALSE(search_res_op.ok());++ // delete models folder+ system("rm -rf ./models");+}++TEST_F(CollectionTest, HybridSearch) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);++ auto op = collectionManager.create_collection(schema);+ LOG(INFO) << collectionManager.get_collection("objects")->get_summary_json();+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();+ nlohmann::json object;+ object["name"] = "apple";+ auto add_op = coll->add(object.dump());+ LOG(INFO) << "hybrid search";+ ASSERT_TRUE(add_op.ok());++ ASSERT_EQ("apple", add_op.get()["name"]);+ ASSERT_EQ(384, add_op.get()["embedding"].size());++ spp::sparse_hash_set<std::string> dummy_include_exclude;+ LOG(INFO) << "hybrid search 2";+ auto search_res_op = coll->search("apple", {"name","embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");+ LOG(INFO) << "hybrid search 3";+ ASSERT_TRUE(search_res_op.ok());+ auto search_res = search_res_op.get();+ ASSERT_EQ(1, search_res["found"].get<size_t>());+ ASSERT_EQ(1, search_res["hits"].size());+ ASSERT_EQ("apple", search_res["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ(384, search_res["hits"][0]["document"]["embedding"].size());++ // delete models folder+ system("rm -rf ./models");+}++TEST_F(CollectionTest, EmbedFielsTest) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ nlohmann::json object = R"({+ "name": "apple"+ })"_json;++ auto embed_op = coll->embed_fields(object);++ ASSERT_TRUE(embed_op.ok());++ ASSERT_EQ("apple", object["name"]);+ ASSERT_EQ(384, object["embedding"].get<std::vector<float>>().size());++ // delete models folder+ system("rm -rf ./models");}\ No newline at end of file