Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(cpp): ONNX Runtime backend laya_onnx and the onnx lane

Adeshina Lasisi4e9628def302
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index a9fd5a7b..9737d665 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -43,6 +43,8 @@ env:
# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus
# compile_flags). Raise it with every test added.
MIN_TESTS: "423"
+ # The onnx job registers the same tests plus the laya_onnx_tests cases.
+ MIN_TESTS_ONNX: "466"
GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints
@@ -120,6 +122,110 @@ jobs:
if: ${{ !cancelled() }}
run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest.xml" --min-tests "$MIN_TESTS"
+ # The ONNX Runtime backend, mirroring go.yml's `onnx` job: the same graph cache keys as java.yml
+ # (so a graph exported there is reused), real graphs for both shipped checkpoints, goldens
+ # recorded by Python's ONNXAgent on those graphs, and a floor on the tests that must run. The
+ # runtime comes from the release archive pinned by sha256 in laya-cpp/cmake/onnxruntime.cmake:
+ # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`.
+ onnx:
+ name: onnx (${{ matrix.compiler }}, ${{ matrix.preset }})
+ runs-on: ubuntu-latest
+ needs: security
+ continue-on-error: true # advisory: see the header
+ timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ matrix:
+ compiler: [g++]
+ preset: [debug]
+ include:
+ - compiler: clang++
+ preset: asan-ubsan
+ - compiler: clang++
+ preset: tsan
+ env:
+ ONNX_ROOT: ${{ github.workspace }}/laya-java/.work/onnx
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt
+ - name: Install a comma-decimal locale
+ run: sudo locale-gen de_DE.UTF-8
+ - name: Install the Python toolchain the goldens are recorded with
+ run: |
+ python -m pip install --upgrade pip
+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+ pip install -r laya-dotnet/tools/requirements-regen.txt
+
+ # Keys are byte-identical to the predict cells in java.yml, so both share the entries.
+ - name: Cache the multilingual graph
+ id: graph-multilingual
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: laya-java/.work/onnx
+ key: laya-java-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
+ - name: Cache the english graph
+ id: graph-english
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: laya-java/.work/onnx
+ key: laya-java-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
+ - name: Prepare both checkpoints and their graphs
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ HIT_MULTILINGUAL: ${{ steps.graph-multilingual.outputs.cache-hit }}
+ HIT_ENGLISH: ${{ steps.graph-english.outputs.cache-hit }}
+ run: |
+ for c in multilingual english; do
+ hit=HIT_${c^^}
+ if [ "${!hit}" = "true" ]; then
+ python laya-java/scripts/prepare_checkpoint.py --checkpoint "$c" --no-graph
+ else
+ python laya-java/scripts/prepare_checkpoint.py --checkpoint "$c"
+ fi
+ for f in laya.onnx laya.onnx.data laya.onnx.laya-revision; do
+ test -s "$ONNX_ROOT/$c/$f" || { echo "::error::missing $c/$f"; exit 1; }
+ done
+ done
+
+ - name: Regenerate the goldens from Python
+ env:
+ LAYA_FIXTURE_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints
+ PYTHONPATH: ${{ github.workspace }}
+ run: |
+ python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json --only sequences.json --only window_plan.json --only decode.json
+ python laya-go/scripts/gen_go_fixtures.py --out "$GOLDENS"
+ # predict.json is recorded per checkpoint from that checkpoint's own graph.
+ for c in multilingual english; do
+ LAYA_ONNX_GRAPH="$ONNX_ROOT/$c/laya.onnx" LAYA_PREDICT_MODEL="$c" \
+ python laya-java/scripts/gen_fixtures.py --strict --out "$RUNNER_TEMP/predict-$c" --only predict.json
+ done
+ cp "$RUNNER_TEMP/predict-multilingual/predict.json" "$GOLDENS/predict.json"
+ cp "$RUNNER_TEMP/predict-english/predict.json" "$GOLDENS/predict-english.json"
+ - name: Compiled C++ tables match CPython
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ run: python laya-go/scripts/gen_tables.py --lang cpp --check
+
+ - name: Configure (fetches the pinned ONNX Runtime and verifies its sha256)
+ working-directory: laya-cpp
+ run: cmake --preset "${{ matrix.preset }}" -DLAYA_ONNX=ON -DCMAKE_CXX_COMPILER="${{ matrix.compiler }}"
+ - name: Build
+ working-directory: laya-cpp
+ run: cmake --build --preset "${{ matrix.preset }}"
+ - name: Test
+ working-directory: laya-cpp
+ env:
+ LAYA_ONNX_GRAPH: ${{ env.ONNX_ROOT }}/multilingual/laya.onnx
+ LAYA_ONNX_GRAPH_ENGLISH: ${{ env.ONNX_ROOT }}/english/laya.onnx
+ run: ctest --preset "${{ matrix.preset }}" --output-junit "$RUNNER_TEMP/ctest-onnx.xml"
+ - name: Count tests (fail on skip, enforce the onnx floor)
+ if: ${{ !cancelled() }}
+ run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest-onnx.xml" --min-tests "$MIN_TESTS_ONNX"
+
security:
name: cpp security scan
runs-on: ubuntu-latest
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index 3f8bfe42..fe87d4b6 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.20)
project(laya_cpp VERSION 0.0.0 LANGUAGES CXX)
option(LAYA_BUILD_TESTS "Build the GoogleTest suite" ON)
-option(LAYA_ONNX "Build the ONNX Runtime backend (not implemented yet)" OFF)
+option(LAYA_ONNX "Build laya_onnx, the ONNX Runtime backend (fetches the pinned runtime unless LAYA_ONNXRUNTIME_ROOT is set)" OFF)
+set(LAYA_ONNXRUNTIME_ROOT "" CACHE PATH "An unpacked onnxruntime release (include/ and lib/) to use instead of the download")
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -31,7 +32,11 @@ target_include_directories(laya PUBLIC include)
laya_target_flags(laya)
if(LAYA_ONNX)
- message(FATAL_ERROR "LAYA_ONNX is not implemented yet")
+ include(cmake/onnxruntime.cmake)
+ add_library(laya_onnx src/onnx_graph.cpp src/onnx.cpp)
+ target_include_directories(laya_onnx PUBLIC include)
+ target_link_libraries(laya_onnx PUBLIC laya PRIVATE laya_ort)
+ laya_target_flags(laya_onnx)
endif()
if(LAYA_BUILD_TESTS)
@@ -63,10 +68,22 @@ if(LAYA_BUILD_TESTS)
target_link_libraries(laya_c4_cli PRIVATE laya)
laya_target_flags(laya_c4_cli)
+ if(LAYA_ONNX)
+ add_executable(laya_onnx_tests tests/onnx_test.cpp)
+ target_link_libraries(laya_onnx_tests PRIVATE laya_onnx laya_ort GTest::gtest_main)
+ laya_target_flags(laya_onnx_tests)
+ laya_ort_runtime_path(laya_onnx_tests)
+ target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}")
+ endif()
+
include(GoogleTest)
# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.
gtest_discover_tests(laya_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)
+ if(LAYA_ONNX)
+ gtest_discover_tests(laya_onnx_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)
+ endif()
+
# Listing must stay cheap (it runs before every ctest): bound it at 3 s with the goldens loaded.
add_test(NAME discovery_fast
COMMAND ${CMAKE_COMMAND} -DEXE=$<TARGET_FILE:laya_tests> -DLIMIT=3
diff --git a/laya-cpp/cmake/onnxruntime.cmake b/laya-cpp/cmake/onnxruntime.cmake
new file mode 100644
index 00000000..6d8178eb
--- /dev/null
+++ b/laya-cpp/cmake/onnxruntime.cmake
@@ -0,0 +1,55 @@
+# The ONNX Runtime used by laya_onnx: LAYA_ONNXRUNTIME_ROOT (an unpacked release) or the pinned
+# release archive, verified by sha256 at the moment of download. Defines the imported target laya_ort.
+set(LAYA_ORT_VERSION "1.30.0")
+include(FetchContent)
+
+if(LAYA_ONNXRUNTIME_ROOT)
+ set(_ort_root "${LAYA_ONNXRUNTIME_ROOT}")
+else()
+ set(_ort_base "https://github.com/microsoft/onnxruntime/releases/download/v${LAYA_ORT_VERSION}")
+ string(TOLOWER "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}" _ort_platform)
+ if(_ort_platform STREQUAL "linux-x86_64")
+ set(_ort_asset "onnxruntime-linux-x64-${LAYA_ORT_VERSION}.tgz")
+ set(_ort_sha256 "a5ed5a3cac51fbb2e90da632ae43d19212faaa20e76484e62bcb7c23ddb3b3fd")
+ elseif(_ort_platform STREQUAL "darwin-arm64")
+ set(_ort_asset "onnxruntime-osx-arm64-${LAYA_ORT_VERSION}.tgz")
+ set(_ort_sha256 "6ebb5062a934537c352937821f9fe9718e7de1a2db1122a93dd363ffd53a7012")
+ else()
+ message(FATAL_ERROR "No pinned ONNX Runtime for ${_ort_platform}; unpack a ${LAYA_ORT_VERSION} release and pass -DLAYA_ONNXRUNTIME_ROOT=<dir>")
+ endif()
+ # URL_HASH makes CMake verify the archive before it is extracted; a mismatch is a configure error.
+ FetchContent_Declare(onnxruntime
+ URL "${_ort_base}/${_ort_asset}"
+ URL_HASH SHA256=${_ort_sha256}
+ DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
+ FetchContent_MakeAvailable(onnxruntime)
+ set(_ort_root "${onnxruntime_SOURCE_DIR}")
+endif()
+
+if(NOT EXISTS "${_ort_root}/include/onnxruntime_cxx_api.h")
+ message(FATAL_ERROR "${_ort_root}/include/onnxruntime_cxx_api.h not found: not an ONNX Runtime release directory")
+endif()
+if(APPLE)
+ set(_ort_lib "${_ort_root}/lib/libonnxruntime.${LAYA_ORT_VERSION}.dylib")
+ if(NOT EXISTS "${_ort_lib}")
+ file(GLOB _ort_lib "${_ort_root}/lib/libonnxruntime*.dylib")
+ list(GET _ort_lib 0 _ort_lib)
+ endif()
+elseif(WIN32)
+ set(_ort_lib "${_ort_root}/lib/onnxruntime.lib")
+else()
+ set(_ort_lib "${_ort_root}/lib/libonnxruntime.so")
+endif()
+if(NOT EXISTS "${_ort_lib}")
+ message(FATAL_ERROR "ONNX Runtime library not found: ${_ort_lib}")
+endif()
+message(STATUS "ONNX Runtime ${LAYA_ORT_VERSION}: ${_ort_lib}")
+
+add_library(laya_ort SHARED IMPORTED)
+set_target_properties(laya_ort PROPERTIES IMPORTED_LOCATION "${_ort_lib}" INTERFACE_INCLUDE_DIRECTORIES "${_ort_root}/include")
+set(LAYA_ORT_LIBDIR "${_ort_root}/lib")
+
+# Build-tree executables find the shared library through an rpath to its directory.
+function(laya_ort_runtime_path target)
+ set_property(TARGET ${target} APPEND PROPERTY BUILD_RPATH "${LAYA_ORT_LIBDIR}")
+endfunction()
diff --git a/laya-cpp/include/laya/onnx.hpp b/laya-cpp/include/laya/onnx.hpp
new file mode 100644
index 00000000..f391721b
--- /dev/null
+++ b/laya-cpp/include/laya/onnx.hpp
@@ -0,0 +1,36 @@
+#pragma once
+// The ONNX Runtime backend (laya-go/onnx). Linking laya_onnx pulls in ONNX Runtime; the core `laya`
+// library never does, and this header includes no ONNX Runtime header.
+#include <cstdint>
+#include <memory>
+#include <string>
+#include "laya/agent.hpp"
+#include "laya/infer.hpp"
+#include "laya/result.hpp"
+
+namespace laya {
+namespace onnx {
+
+struct Options {
+ int intra_op_threads = 0; // 0 leaves ONNX Runtime's default; at most kMaxThreads
+ int inter_op_threads = 0;
+};
+constexpr int kMaxThreads = 4096;
+
+// The version of the ONNX Runtime library loaded, e.g. "1.30.0".
+std::string RuntimeVersion();
+
+// A Session over one fused laya.onnx (+ laya.onnx.data beside it). Checks the graph's inputs and
+// outputs at once. Run is thread-safe; Close is idempotent and waits for runs in flight. No
+// exception leaves this API.
+Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options = {});
+
+} // namespace onnx
+
+namespace agent {
+// checkpoint_dir holds rl_agent_config.json and the tokenizer; graph_path is the fused laya.onnx.
+Result<std::unique_ptr<Agent>> OpenOnnx(const std::string& checkpoint_dir, const std::string& graph_path,
+ const onnx::Options& options = {});
+} // namespace agent
+
+} // namespace laya
diff --git a/laya-cpp/include/laya/onnx_graph.hpp b/laya-cpp/include/laya/onnx_graph.hpp
new file mode 100644
index 00000000..0a5eeed7
--- /dev/null
+++ b/laya-cpp/include/laya/onnx_graph.hpp
@@ -0,0 +1,39 @@
+#pragma once
+// The ONNX backend's checks that need no ONNX Runtime (laya-go/onnx/graph.go): what a graph must
+// declare, where a graph path may point, and how a run's outputs become an infer::Output.
+#include <cstdint>
+#include <string>
+#include <vector>
+#include "laya/infer.hpp"
+#include "laya/result.hpp"
+
+namespace laya {
+namespace onnx {
+
+struct TensorInfo {
+ std::string name;
+ std::string dtype; // "int64", "bool", "float32", or whatever else the graph declares
+ std::vector<int64_t> dims; // -1 for a symbolic or unknown dimension
+};
+
+// "" when the graph's own metadata is what the agent feeds: five inputs (int64, int64, int64, bool,
+// int64) and two float32 outputs, matching by name, no extras, dynamic batch dimension, and a fixed
+// 2 columns for act_logits. Else the first problem.
+std::string CheckGraph(const std::vector<TensorInfo>& inputs, const std::vector<TensorInfo>& outputs);
+
+// "" when path names a file that may be opened as the fused graph; refuses the split
+// encoder.onnx + head.onnx layout with a message that says how to re-export.
+std::string CheckPath(const std::string& path);
+
+struct OutTensor {
+ std::vector<int64_t> shape;
+ const float* data = nullptr;
+ int64_t count = 0; // elements at data
+};
+
+// Copies the two outputs after checking their shapes against b, then Output::Check. NaN anywhere is
+// an error: a NaN would decode to a silent first-max answer.
+Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logits, const OutTensor& act_logits);
+
+} // namespace onnx
+} // namespace laya
diff --git a/laya-cpp/src/onnx.cpp b/laya-cpp/src/onnx.cpp
new file mode 100644
index 00000000..878e6f07
--- /dev/null
+++ b/laya-cpp/src/onnx.cpp
@@ -0,0 +1,191 @@
+#include "laya/onnx.hpp"
+
+#include <onnxruntime_cxx_api.h>
+
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+
+#include "laya/onnx_graph.hpp"
+
+namespace laya {
+namespace onnx {
+namespace {
+
+// One Ort::Env per process, shared by every session and freed with the last one, so there is no
+// static destructor racing a session's teardown. The mutex orders creation against that release.
+std::shared_ptr<Ort::Env> SharedEnv() {
+ static std::mutex mu;
+ static std::weak_ptr<Ort::Env> weak;
+ std::lock_guard<std::mutex> lock(mu);
+ std::shared_ptr<Ort::Env> env = weak.lock();
+ if (!env) {
+ env = std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_ERROR, "laya");
+ weak = env;
+ }
+ return env;
+}
+
+std::string Dtype(ONNXTensorElementDataType t) {
+ switch (t) {
+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return "int64";
+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return "bool";
+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return "float32";
+ default: return "type " + std::to_string(int(t));
+ }
+}
+
+// Inputs by name, in the order the five tensors are bound below.
+const char* const kInputNames[5] = {"input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"};
+const char* const kOutputNames[2] = {"logits", "act_logits"};
+
+class OrtSession final : public infer::Session {
+ public:
+ OrtSession(std::shared_ptr<Ort::Env> env, std::unique_ptr<Ort::Session> s) : env_(std::move(env)), session_(std::move(s)) {}
+ ~OrtSession() override { Close(); }
+
+ Result<infer::Output> Run(const infer::Batch& b, const CancelToken& cancel) override {
+ try {
+ return RunImpl(b, cancel);
+ } catch (const Ort::Exception& e) {
+ return Result<infer::Output>::Fail(std::string("onnx: run: ") + e.what());
+ } catch (const std::exception& e) {
+ return Result<infer::Output>::Fail(std::string("onnx: run: ") + e.what());
+ } catch (...) {
+ return Result<infer::Output>::Fail("onnx: run: unknown exception");
+ }
+ }
+
+ std::string Close() override {
+ try {
+ std::unique_lock<std::shared_mutex> lock(mu_); // waits for runs in flight
+ session_.reset();
+ return "";
+ } catch (...) {
+ return "onnx: close failed";
+ }
+ }
+
+ private:
+ Result<infer::Output> RunImpl(const infer::Batch& b, const CancelToken& cancel) {
+ using R = Result<infer::Output>;
+ if (cancel.cancelled()) return R::Fail(infer::kCancelled);
+ const std::string bad = b.Validate();
+ if (!bad.empty()) return R::Fail("onnx: " + bad);
+ for (uint8_t m : b.marker_mask) {
+ if (m > 1) return R::Fail("onnx: marker_mask holds a value other than 0 or 1");
+ }
+ std::shared_lock<std::shared_mutex> lock(mu_);
+ if (!session_) return R::Fail("onnx: session is closed");
+
+ const Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+ const int64_t shape2[2] = {b.rows, b.length};
+ const int64_t shape_m[2] = {b.rows, b.markers};
+ const int64_t shape1[1] = {b.rows};
+ // ORT takes non-const pointers but never writes to an input.
+ auto i64 = [&](const std::vector<int64_t>& v, const int64_t* shape, size_t rank) {
+ return Ort::Value::CreateTensor<int64_t>(mem, const_cast<int64_t*>(v.data()), v.size(), shape, rank);
+ };
+ std::vector<Ort::Value> in;
+ in.reserve(5);
+ in.push_back(i64(b.input_ids, shape2, 2));
+ in.push_back(i64(b.attention_mask, shape2, 2));
+ in.push_back(i64(b.marker_pos, shape_m, 2));
+ in.push_back(Ort::Value::CreateTensor(mem, const_cast<uint8_t*>(b.marker_mask.data()), b.marker_mask.size() * sizeof(uint8_t),
+ shape_m, 2, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL));
+ in.push_back(i64(b.qtype, shape1, 1));
+
+ Ort::RunOptions ro;
+ std::vector<Ort::Value> out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);
+ if (out.size() != 2) return R::Fail("onnx: run returned " + std::to_string(out.size()) + " outputs, want 2");
+ if (cancel.cancelled()) return R::Fail(infer::kCancelled);
+
+ OutTensor t[2];
+ for (size_t i = 0; i < 2; i++) {
+ if (!out[i].IsTensor()) return R::Fail(std::string("onnx: output ") + kOutputNames[i] + " is not a tensor");
+ const Ort::TensorTypeAndShapeInfo info = out[i].GetTensorTypeAndShapeInfo();
+ if (info.GetElementType() != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
+ return R::Fail(std::string("onnx: output ") + kOutputNames[i] + " is " + Dtype(info.GetElementType()) + ", want float32");
+ }
+ t[i].shape = info.GetShape();
+ t[i].count = int64_t(info.GetElementCount());
+ t[i].data = out[i].GetTensorData<float>();
+ }
+ return CollectOutput(b, t[0], t[1]);
+ }
+
+ std::shared_ptr<Ort::Env> env_; // keeps the environment alive past session_
+ std::shared_mutex mu_;
+ std::unique_ptr<Ort::Session> session_;
+};
+
+} // namespace
+
+std::string RuntimeVersion() {
+ try {
+ return Ort::GetVersionString();
+ } catch (...) {
+ return "";
+ }
+}
+
+Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options) {
+ using R = Result<std::unique_ptr<infer::Session>>;
+ if (options.intra_op_threads < 0 || options.inter_op_threads < 0 || options.intra_op_threads > kMaxThreads ||
+ options.inter_op_threads > kMaxThreads) {
+ return R::Fail("onnx: thread counts must be between 0 and " + std::to_string(kMaxThreads));
+ }
+ const std::string path_problem = CheckPath(graph_path);
+ if (!path_problem.empty()) return R::Fail("onnx: " + path_problem);
+ try {
+ std::shared_ptr<Ort::Env> env = SharedEnv();
+ Ort::SessionOptions so;
+ if (options.intra_op_threads > 0) so.SetIntraOpNumThreads(options.intra_op_threads);
+ if (options.inter_op_threads > 0) so.SetInterOpNumThreads(options.inter_op_threads);
+ const std::filesystem::path p(graph_path);
+ auto session = std::make_unique<Ort::Session>(*env, p.c_str(), so);
+
+ Ort::AllocatorWithDefaultOptions alloc;
+ auto describe = [&](bool inputs) {
+ std::vector<TensorInfo> v;
+ const size_t n = inputs ? session->GetInputCount() : session->GetOutputCount();
+ for (size_t i = 0; i < n; i++) {
+ TensorInfo t;
+ Ort::AllocatedStringPtr name = inputs ? session->GetInputNameAllocated(i, alloc) : session->GetOutputNameAllocated(i, alloc);
+ t.name = name.get();
+ Ort::TypeInfo ti = inputs ? session->GetInputTypeInfo(i) : session->GetOutputTypeInfo(i);
+ if (ti.GetONNXType() != ONNX_TYPE_TENSOR) {
+ t.dtype = "not a tensor";
+ } else {
+ const Ort::ConstTensorTypeAndShapeInfo info = ti.GetTensorTypeAndShapeInfo();
+ t.dtype = Dtype(info.GetElementType());
+ t.dims = info.GetShape();
+ }
+ v.push_back(std::move(t));
+ }
+ return v;
+ };
+ const std::string bad = CheckGraph(describe(true), describe(false));
+ if (!bad.empty()) return R::Fail("onnx: graph " + graph_path + ": " + bad);
+ return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(env), std::move(session))));
+ } catch (const Ort::Exception& e) {
+ return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());
+ } catch (const std::exception& e) {
+ return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());
+ } catch (...) {
+ return R::Fail("onnx: cannot open " + graph_path + ": unknown exception");
+ }
+}
+
+} // namespace onnx
+
+namespace agent {
+Result<std::unique_ptr<Agent>> OpenOnnx(const std::string& checkpoint_dir, const std::string& graph_path, const onnx::Options& options) {
+ auto s = onnx::Open(graph_path, options);
+ if (!s.ok()) return Result<std::unique_ptr<Agent>>::Fail(s.error());
+ return Agent::Open(checkpoint_dir, std::move(s).value()); // closes the session on failure
+}
+} // namespace agent
+
+} // namespace laya
diff --git a/laya-cpp/src/onnx_graph.cpp b/laya-cpp/src/onnx_graph.cpp
new file mode 100644
index 00000000..808c0b1f
--- /dev/null
+++ b/laya-cpp/src/onnx_graph.cpp
@@ -0,0 +1,133 @@
+#include "laya/onnx_graph.hpp"
+
+#include <cmath>
+#include <filesystem>
+#include <set>
+
+namespace laya {
+namespace onnx {
+namespace {
+
+const char* const kReexport = "re-export with scripts/export_onnx.py";
+const std::string kSplitProblem = std::string("the split encoder.onnx + head.onnx layout is not supported; ") + kReexport +
+ " to write one fused laya.onnx";
+
+struct Spec {
+ const char* name;
+ const char* dtype;
+ size_t rank;
+ int64_t fixed_cols; // 0: every dimension after the batch is dynamic
+};
+
+const Spec kInputs[] = {{"input_ids", "int64", 2, 0},
+ {"attention_mask", "int64", 2, 0},
+ {"marker_pos", "int64", 2, 0},
+ {"marker_mask", "bool", 2, 0},
+ {"qtype", "int64", 1, 0}};
+const Spec kOutputs[] = {{"logits", "float32", 2, 0}, {"act_logits", "float32", 2, 2}};
+
+template <size_t N>
+std::string CheckSide(const char* kind, const std::vector<TensorInfo>& got, const Spec (&want)[N]) {
+ for (const Spec& w : want) {
+ const TensorInfo* g = nullptr;
+ for (const TensorInfo& t : got) {
+ if (t.name == w.name) {
+ g = &t;
+ break;
+ }
+ }
+ const std::string q = std::string(kind) + " \"" + w.name + "\"";
+ if (g == nullptr) return std::string("missing ") + q + "; " + kReexport;
+ if (g->dtype != w.dtype) return q + " is " + g->dtype + ", want " + w.dtype + "; " + kReexport;
+ if (g->dims.size() != w.rank) {
+ return q + " has rank " + std::to_string(g->dims.size()) + ", want " + std::to_string(w.rank) + "; " + kReexport;
+ }
+ for (size_t i = 0; i < g->dims.size(); i++) {
+ const int64_t d = g->dims[i];
+ if (d == -1 || (i > 0 && w.fixed_cols > 0 && d == w.fixed_cols)) continue;
+ return q + " declares " + (i == 0 ? "a fixed batch dimension " : "a fixed size ") + std::to_string(d) + "; " + kReexport;
+ }
+ }
+ std::set<std::string> known;
+ for (const Spec& w : want) known.insert(w.name);
+ for (const TensorInfo& t : got) {
+ if (known.count(t.name) == 0) {
+ return std::string("unexpected ") + kind + " \"" + t.name + "\"; the agent feeds only " + std::to_string(sizeof(kInputs) / sizeof(kInputs[0])) +
+ " inputs and " + std::to_string(sizeof(kOutputs) / sizeof(kOutputs[0])) + " outputs; " + kReexport;
+ }
+ }
+ if (got.size() != N) return std::string("duplicate ") + kind + " name; " + kReexport;
+ return "";
+}
+
+bool IsSplit(const std::filesystem::path& dir) {
+ std::error_code ec;
+ for (const char* n : {"encoder.onnx", "head.onnx"}) {
+ if (!std::filesystem::is_regular_file(dir / n, ec)) return false;
+ }
+ return true;
+}
+
+std::string Shape(const std::vector<int64_t>& s) {
+ std::string out = "[";
+ for (size_t i = 0; i < s.size(); i++) out += (i ? " " : "") + std::to_string(s[i]);
+ return out + "]";
+}
+
+} // namespace
+
+std::string CheckGraph(const std::vector<TensorInfo>& inputs, const std::vector<TensorInfo>& outputs) {
+ std::string p = CheckSide("input", inputs, kInputs);
+ if (p.empty()) p = CheckSide("output", outputs, kOutputs);
+ return p;
+}
+
+std::string CheckPath(const std::string& path) {
+ namespace fs = std::filesystem;
+ if (path.empty()) return "graph path is empty";
+ std::error_code ec;
+ const fs::path p(path);
+ const fs::file_status st = fs::status(p, ec);
+ if (!ec && fs::is_directory(st)) {
+ return IsSplit(p) ? kSplitProblem : "graph " + path + " is a directory; pass the path of laya.onnx";
+ }
+ if (!ec && fs::is_regular_file(st)) {
+ const std::string base = p.filename().string();
+ if ((base == "encoder.onnx" || base == "head.onnx") && IsSplit(p.parent_path())) return kSplitProblem;
+ return "";
+ }
+ const fs::path dir = p.has_parent_path() ? p.parent_path() : fs::path(".");
+ if (st.type() == fs::file_type::not_found && IsSplit(dir)) return "graph " + path + ": no such file; " + kSplitProblem;
+ if (st.type() == fs::file_type::not_found) return "graph " + path + ": no such file";
+ return "graph " + path + ": " + (ec ? ec.message() : std::string("not a regular file"));
+}
+
+Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logits, const OutTensor& act) {
+ struct Col {
+ const char* name;
+ const OutTensor& t;
+ int64_t cols;
+ };
+ for (const Col& c : {Col{"logits", logits, b.markers}, Col{"act_logits", act, 2}}) {
+ if (c.t.shape.size() != 2 || c.t.shape[0] != b.rows || c.t.shape[1] != c.cols) {
+ return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " has shape " + Shape(c.t.shape) + ", want [" +
+ std::to_string(b.rows) + " " + std::to_string(c.cols) + "]");
+ }
+ // rows and cols are bounded by Batch::Validate, so this product cannot overflow.
+ if (c.t.count != b.rows * c.cols || (c.t.count > 0 && c.t.data == nullptr)) {
+ return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " holds " + std::to_string(c.t.count) + " values");
+ }
+ for (int64_t i = 0; i < c.t.count; i++) {
+ if (std::isnan(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN");
+ }
+ }
+ infer::Output o;
+ o.logits.assign(logits.data, logits.data + logits.count);
+ o.act_logits.assign(act.data, act.data + act.count);
+ const std::string bad = o.Check(b);
+ if (!bad.empty()) return Result<infer::Output>::Fail("onnx: " + bad);
+ return Result<infer::Output>::Ok(std::move(o));
+}
+
+} // namespace onnx
+} // namespace laya
diff --git a/laya-cpp/tests/onnx_test.cpp b/laya-cpp/tests/onnx_test.cpp
new file mode 100644
index 00000000..72e33341
--- /dev/null
+++ b/laya-cpp/tests/onnx_test.cpp
@@ -0,0 +1,662 @@
+// Tests of the ONNX backend. The pure checks run anywhere; the tests that load a graph need
+// LAYA_ONNX_GRAPH (multilingual), LAYA_ONNX_GRAPH_ENGLISH and LAYA_CHECKPOINTS, and with
+// LAYA_REQUIRE_GOLDENS=1 (the onnx lane) a missing one FAILS instead of skipping.
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <cmath>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <map>
+#include <mutex>
+#include <thread>
+
+#include <unistd.h>
+
+#include "golden.hpp"
+#include "laya/onnx.hpp"
+#include "laya/onnx_graph.hpp"
+
+namespace {
+
+namespace fs = std::filesystem;
+using laya::Result;
+using laya::infer::Batch;
+using laya::onnx::OutTensor;
+using laya::onnx::TensorInfo;
+using laya::pyvalue::Value;
+using Kind = Value::Kind;
+
+// ---------------------------------------------------------------- graph checks (no runtime needed)
+
+std::vector<TensorInfo> GoodInputs() {
+ return {{"input_ids", "int64", {-1, -1}},
+ {"attention_mask", "int64", {-1, -1}},
+ {"marker_pos", "int64", {-1, -1}},
+ {"marker_mask", "bool", {-1, -1}},
+ {"qtype", "int64", {-1}}};
+}
+std::vector<TensorInfo> GoodOutputs() { return {{"logits", "float32", {-1, -1}}, {"act_logits", "float32", {-1, 2}}}; }
+
+TEST(OnnxGraph, TheExportedShapeIsAccepted) { EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), GoodOutputs()), ""); }
+
+TEST(OnnxGraph, InputOrderInTheGraphDoesNotMatter) {
+ auto in = GoodInputs();
+ std::swap(in[0], in[4]);
+ std::swap(in[1], in[3]);
+ EXPECT_EQ(laya::onnx::CheckGraph(in, GoodOutputs()), "");
+}
+
+TEST(OnnxGraph, EveryInputIsHeldToItsName) {
+ for (size_t i = 0; i < 5; i++) {
+ auto in = GoodInputs();
+ in[i].name = "renamed";
+ const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());
+ EXPECT_NE(e.find("missing input"), std::string::npos) << i << ": " << e;
+ }
+}
+
+TEST(OnnxGraph, EveryInputIsHeldToItsDtype) {
+ for (size_t i = 0; i < 5; i++) {
+ auto in = GoodInputs();
+ in[i].dtype = in[i].dtype == "int64" ? "float32" : "int64";
+ const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());
+ EXPECT_NE(e.find(in[i].name), std::string::npos) << e;
+ EXPECT_NE(e.find("want"), std::string::npos) << e;
+ }
+}
+
+TEST(OnnxGraph, TheMaskMustBeBoolNotInt64) {
+ auto in = GoodInputs();
+ in[3].dtype = "int64";
+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("is int64, want bool"), std::string::npos);
+}
+
+TEST(OnnxGraph, RankIsChecked) {
+ auto in = GoodInputs();
+ in[4].dims = {-1, -1};
+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 2, want 1"), std::string::npos);
+ in = GoodInputs();
+ in[0].dims = {-1};
+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 1, want 2"), std::string::npos);
+}
+
+TEST(OnnxGraph, AFixedBatchDimensionIsRefused) {
+ auto out = GoodOutputs();
+ out[1].dims = {1, 2}; // the layout of onnx/full-multilingual: act_logits traced at batch 1
+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed batch dimension 1"), std::string::npos);
+ auto in = GoodInputs();
+ in[0].dims = {4, -1};
+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("fixed batch dimension 4"), std::string::npos);
+}
+
+TEST(OnnxGraph, ActLogitsHasExactlyTwoColumns) {
+ auto out = GoodOutputs();
+ out[1].dims = {-1, 3};
+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed size 3"), std::string::npos);
+ out[1].dims = {-1, -1};
+ EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), out), "");
+}
+
+TEST(OnnxGraph, ExtrasAndDuplicatesAreRefused) {
+ auto in = GoodInputs();
+ in.push_back({"extra", "int64", {-1}});
+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("unexpected input \"extra\""), std::string::npos);
+ auto out = GoodOutputs();
+ out.push_back({"logits", "float32", {-1, -1}});
+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("duplicate output"), std::string::npos);
+ EXPECT_NE(laya::onnx::CheckGraph({}, GoodOutputs()).find("missing input"), std::string::npos);
+}
+
+TEST(OnnxGraph, AnOutputDtypeIsChecked) {
+ auto out = GoodOutputs();
+ out[0].dtype = "int64";
+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("output \"logits\" is int64"), std::string::npos);
+}
+
+// ---------------------------------------------------------------- paths
+
+class TempDir {
+ public:
+ TempDir() {
+ static std::atomic<int> n{0};
+ path_ = fs::temp_directory_path() / ("laya_onnx_test_" + std::to_string(::getpid()) + "_" + std::to_string(n++));
+ fs::create_directories(path_);
+ }
+ ~TempDir() {
+ std::error_code ec;
+ fs::remove_all(path_, ec);
+ }
+ TempDir(const TempDir&) = delete;
+ TempDir& operator=(const TempDir&) = delete;
+ std::string File(const std::string& name, const std::string& body = "x") const {
+ std::ofstream(path_ / name, std::ios::binary) << body;
+ return (path_ / name).string();
+ }
+ std::string str() const { return path_.string(); }
+
+ private:
+ fs::path path_;
+};
+
+TEST(OnnxPath, MissingAndDirectoryAndEmptyAreErrors) {
+ TempDir d;
+ EXPECT_NE(laya::onnx::CheckPath(d.str() + "/nope.onnx").find("no such file"), std::string::npos);
+ EXPECT_NE(laya::onnx::CheckPath(d.str()).find("is a directory"), std::string::npos);
+ EXPECT_NE(laya::onnx::CheckPath("").find("empty"), std::string::npos);
+ EXPECT_EQ(laya::onnx::CheckPath(d.File("laya.onnx")), "");
+}
+
+TEST(OnnxPath, TheSplitLayoutIsRefusedWhicheverPathIsGiven) {
+ TempDir d;
+ const std::string enc = d.File("encoder.onnx");
+ const std::string head = d.File("head.onnx");
+ for (const std::string& p : {enc, head, d.str(), d.str() + "/laya.onnx"}) {
+ const std::string e = laya::onnx::CheckPath(p);
+ EXPECT_NE(e.find("split encoder.onnx + head.onnx layout is not supported"), std::string::npos) << p << ": " << e;
+ EXPECT_NE(e.find("export_onnx.py"), std::string::npos) << e;
+ }
+}
+
+TEST(OnnxPath, OneHalfAloneIsNotTheSplitLayout) {
+ TempDir d;
+ const std::string enc = d.File("encoder.onnx");
+ EXPECT_EQ(laya::onnx::CheckPath(enc), "");
+}
+
+// ---------------------------------------------------------------- collecting outputs
+
+Batch Tiny(int64_t rows, int64_t markers) {
+ Batch b;
+ b.rows = rows;
+ b.length = 3;
+ b.markers = markers;
+ b.input_ids.assign(size_t(rows * 3), 1);
+ b.attention_mask.assign(size_t(rows * 3), 1);
+ b.marker_pos.assign(size_t(rows * markers), 1);
+ b.marker_mask.assign(size_t(rows * markers), 1);
+ b.qtype.assign(size_t(rows), 0);
+ return b;
+}
+
+OutTensor Tensor(const std::vector<float>& v, std::vector<int64_t> shape) {
+ OutTensor t;
+ t.shape = std::move(shape);
+ t.data = v.data();
+ t.count = int64_t(v.size());
+ return t;
+}
+
+TEST(OnnxCollect, ACorrectPairIsCopied) {
+ const Batch b = Tiny(2, 3);
+ const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};
+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));
+ ASSERT_TRUE(r.ok()) << r.error();
+ EXPECT_EQ(r.value().logits, l);
+ EXPECT_EQ(r.value().act_logits, a);
+}
+
+TEST(OnnxCollect, ASlicedOutputIsTheOutputOfTheRightRows) {
+ const Batch b = Tiny(2, 3);
+ const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};
+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));
+ ASSERT_TRUE(r.ok());
+ EXPECT_EQ(r.value().logits[3], 4.0f); // row 1, column 0
+ EXPECT_EQ(r.value().act_logits[2], 9.0f);
+}
+
+TEST(OnnxCollect, ANaNIsAnErrorNotAnAnswer) {
+ const Batch b = Tiny(1, 3);
+ std::vector<float> l = {1, std::numeric_limits<float>::quiet_NaN(), 3}, a = {1, 2};
+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("NaN"), std::string::npos);
+ l[1] = 0;
+ a[0] = std::numeric_limits<float>::quiet_NaN();
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());
+}
+
+TEST(OnnxCollect, WrongShapesAreErrors) {
+ const Batch b = Tiny(2, 3);
+ const std::vector<float> l(6, 1.f), a(4, 1.f), small(2, 1.f);
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {3, 2}), Tensor(a, {2, 2})).ok());
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {4, 1})).ok());
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {6}), Tensor(a, {2, 2})).ok());
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3, 1}), Tensor(a, {2, 2})).ok());
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(small, {2, 3}), Tensor(a, {2, 2})).ok()); // shape says 6, holds 2
+}
+
+// ---------------------------------------------------------------- options
+
+TEST(OnnxOpen, ThreadCountsAreRange_Checked) {
+ laya::onnx::Options o;
+ o.intra_op_threads = -1;
+ EXPECT_FALSE(laya::onnx::Open("/nonexistent/laya.onnx", o).ok());
+ o.intra_op_threads = 0;
+ o.inter_op_threads = laya::onnx::kMaxThreads + 1;
+ const auto r = laya::onnx::Open("/nonexistent/laya.onnx", o);
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("thread counts"), std::string::npos);
+}
+
+TEST(OnnxOpen, AMissingGraphIsAnError) {
+ const auto r = laya::onnx::Open("/nonexistent/dir/laya.onnx");
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("no such file"), std::string::npos);
+}
+
+TEST(OnnxOpen, AGarbageFileIsAnErrorNotAnException) {
+ TempDir d;
+ const auto r = laya::onnx::Open(d.File("laya.onnx", "this is not a protobuf \x01\x02\xff"));
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
+}
+
+TEST(OnnxOpen, AnEmptyFileIsAnError) {
+ TempDir d;
+ EXPECT_FALSE(laya::onnx::Open(d.File("laya.onnx", "")).ok());
+}
+
+TEST(OnnxOpen, TheSplitLayoutIsRefusedByOpen) {
+ TempDir d;
+ d.File("encoder.onnx");
+ d.File("head.onnx");
+ const auto r = laya::onnx::Open(d.str() + "/encoder.onnx");
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("split encoder.onnx + head.onnx layout"), std::string::npos) << r.error();
+}
+
+TEST(OnnxRuntime, TheLoadedLibraryIsThePinnedVersion) { EXPECT_EQ(laya::onnx::RuntimeVersion(), LAYA_ORT_VERSION); }
+
+// ---------------------------------------------------------------- real graphs
+
+struct Rig {
+ const char* checkpoint; // directory name under LAYA_CHECKPOINTS
+ const char* graph_env; // names the exported laya.onnx
+ const char* golden; // predict golden recorded from that graph
+};
+const Rig kRigs[] = {{"multilingual", "LAYA_ONNX_GRAPH", "predict.json"},
+ {"english", "LAYA_ONNX_GRAPH_ENGLISH", "predict-english.json"}};
+
+// "" and a recorded failure (or skip) when the rig is not there.
+std::string Need(const char* var) {
+ const char* v = std::getenv(var);
+ if (v != nullptr && *v != 0) return v;
+ if (laya_test::RequireGoldens()) ADD_FAILURE() << var << " is unset but LAYA_REQUIRE_GOLDENS=1";
+ return "";
+}
+
+#define NEED_RIG(rig, graph, dir) \
+ const std::string graph = Need(rig.graph_env); \
+ const std::string root = Need("LAYA_CHECKPOINTS"); \
+ if (graph.empty() || root.empty()) { \
+ if (!HasFailure()) GTEST_SKIP() << "no ONNX rig (local run)"; \
+ return; \
+ } \
+ const std::string dir = root + "/" + rig.checkpoint
+
+std::mutex g_agents_mu;
+std::map<std::string, std::unique_ptr<laya::agent::Agent>> g_agents;
+
+// The agents hold ONNX Runtime, which must be released before static destruction begins (its own
+// statics may already be gone by then), so the test environment closes them.
+class CloseAgents : public ::testing::Environment {
+ public:
+ void TearDown() override {
+ std::lock_guard<std::mutex> lock(g_agents_mu);
+ for (auto& kv : g_agents) EXPECT_EQ(kv.second->Close(), "");
+ g_agents.clear();
+ }
+};
+const ::testing::Environment* const g_close_agents = ::testing::AddGlobalTestEnvironment(new CloseAgents);
+
+// One agent per checkpoint for the whole process; the tests only read it.
+laya::agent::Agent* Shared(const Rig& rig, const std::string& graph, const std::string& dir) {
+ auto& agents = g_agents;
+ std::lock_guard<std::mutex> lock(g_agents_mu);
+ auto it = agents.find(rig.checkpoint);
+ if (it != agents.end()) return it->second.get();
+ auto a = laya::agent::OpenOnnx(dir, graph);
+ if (!a.ok()) {
+ ADD_FAILURE() << "open " << rig.checkpoint << ": " << a.error();
+ return nullptr;
+ }
+ return (agents[rig.checkpoint] = std::move(a).value()).get();
+}
+
+constexpr double kTolerance = 2e-4; // probabilities, confidences, act_probability, noul
+constexpr double kScoreTolerance = 8e-4; // score, the expectation over the probabilities
+constexpr double kDecisiveGap = 1e-3; // the chosen label is compared when the top two differ by more
+constexpr double kMaxDiffering = 0.05; // of all compared numbers may differ at all
+
+struct Tally {
+ int numbers = 0, differing = 0, labels = 0, ties = 0, label_diffs = 0, failures = 0;
+ double worst = 0;
+};
+
+void Compare(const std::string& path, const std::string& key, const Value& got, const Value& want, Tally& t) {
+ if (want.kind() == Kind::Object) {
+ if (got.kind() != Kind::Object || got.size() != want.size()) {
+ t.failures++;
+ ADD_FAILURE() << path << ": object shape differs";
+ return;
+ }
+ for (size_t i = 0; i < want.size(); i++) {
+ const Value* g = got.Find(want.key(i));
+ if (g == nullptr || got.key(i) != want.key(i)) {
+ t.failures++;
+ ADD_FAILURE() << path << ": key " << want.key(i) << " missing or out of order";
+ continue;
+ }
+ Compare(path + "." + want.key(i), want.key(i), *g, want.value(i), t);
+ }
+ } else if (want.kind() == Kind::Array) {
+ if (got.kind() != Kind::Array || got.size() != want.items().size()) {
+ t.failures++;
+ ADD_FAILURE() << path << ": array shape differs";
+ return;
+ }
+ for (size_t i = 0; i < want.items().size(); i++) Compare(path + "[]", key, got.items()[i], want.items()[i], t);
+ } else if (want.kind() == Kind::Int || want.kind() == Kind::Float) {
+ if (got.kind() != Kind::Int && got.kind() != Kind::Float) {
+ t.failures++;
+ ADD_FAILURE() << path << ": not a number";
+ return;
+ }
+ const double g = laya_test::Num(got), w = laya_test::Num(want), d = std::fabs(g - w);
+ t.numbers++;
+ if (d > 0) t.differing++;
+ t.worst = std::max(t.worst, d);
+ if (d > (key == "score" ? kScoreTolerance : kTolerance)) {
+ t.failures++;
+ ADD_FAILURE() << path << ": got " << g << " want " << w << " (off by " << d << ")";
+ }
+ } else if (got.kind() != want.kind() || laya::pyvalue::Repr(got) != laya::pyvalue::Repr(want)) {
+ t.failures++;
+ ADD_FAILURE() << path << ": " << laya::pyvalue::Repr(got) << " want " << laya::pyvalue::Repr(want);
+ }
+}
+
+Value Without(const Value& obj, const std::string& drop) {
+ Value out = Value::Object();
+ for (size_t i = 0; i < obj.size(); i++) {
+ if (obj.key(i) != drop) out.Set(obj.key(i), obj.value(i));
+ }
+ return out;
+}
+
+void CompareAnswer(const std::string& path, const Value& got, const Value& want, Tally& t) {
+ const Value* type = want.Find("type");
+ if (type != nullptr && type->as_string() == "choice") {
+ std::vector<double> ps;
+ for (const Value& p : want.Find("probabilities")->items()) ps.push_back(laya_test::Num(p));
+ std::sort(ps.rbegin(), ps.rend());
+ if (ps.size() > 1 && ps[0] - ps[1] <= kDecisiveGap) {
+ t.ties++;
+ Compare(path, "", Without(got, "choice"), Without(want, "choice"), t);
+ return;
+ }
+ t.labels++;
+ const Value* g = got.Find("choice");
+ if (g == nullptr || laya::pyvalue::Repr(*g) != laya::pyvalue::Repr(*want.Find("choice"))) t.label_diffs++;
+ }
+ Compare(path, "", got, want, t);
+}
+
+void ComparePrediction(const std::string& path, const laya::schema::Prediction& p, const Value& want, Tally& t) {
+ auto json = p.ToJson();
+ ASSERT_TRUE(json.ok()) << json.error();
+ auto parsed = laya::pyvalue::Parse(json.value());
+ ASSERT_TRUE(parsed.ok()) << parsed.error();
+ const Value& got = parsed.value();
+ Compare(path + ".model", "model", *got.Find("model"), *want.Find("model"), t);
+ Compare(path + ".usage", "usage", *got.Find("usage"), *want.Find("usage"), t);
+ const Value& ga = *got.Find("answers");
+ const Value& wa = *want.Find("answers");
+ ASSERT_EQ(ga.size(), wa.size()) << path;
+ for (size_t i = 0; i < wa.size(); i++) {
+ ASSERT_EQ(ga.key(i), wa.key(i)) << path;
+ CompareAnswer(path + "." + wa.key(i), ga.value(i), wa.value(i), t);
+ }
+}
+
+laya::schema::Questions QuestionsOf(const Value& v) {
+ auto text = laya::pyvalue::Dumps(v);
+ EXPECT_TRUE(text.ok());
+ auto qs = laya::schema::Questions::Parse(text.value());
+ EXPECT_TRUE(qs.ok()) << qs.error();
+ return qs.ok() ? qs.value() : laya::schema::Questions::Of({}).value();
+}
+
+void Finish(const char* name, const Tally& t, int min_numbers) {
+ std::printf("[ tally ] %s: %d numbers, %d differ (worst %.2e), %d decisive labels (%d differ), %d near-ties\n", name, t.numbers,
+ t.differing, t.worst, t.labels, t.label_diffs, t.ties);
+ EXPECT_GE(t.numbers, min_numbers) << "too few numbers were compared";
+ EXPECT_LE(double(t.differing), kMaxDiffering * t.numbers) << "more of the numbers differ at all than Python's CPU noise explains";
+ EXPECT_EQ(t.label_diffs, 0) << "decisive labels differ from Python";
+}
+
+class OnnxRig : public ::testing::TestWithParam<int> {};
+INSTANTIATE_TEST_SUITE_P(Checkpoints, OnnxRig, ::testing::Values(0, 1), [](const auto& i) { return std::string(kRigs[i.param].checkpoint); });
+
+TEST_P(OnnxRig, SinglePredictionsMatchPython) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");
+ LAYA_EXPECT_ROWS(singles, 6);
+ laya::agent::Agent* a = Shared(rig, graph, dir);
+ ASSERT_NE(a, nullptr);
+ Tally t;
+ for (size_t i = 0; i < singles->size(); i++) {
+ const Value& row = singles->value(i);
+ const std::string name = singles->key(i);
+ auto pred = a->Predict(*row.Find("state"), QuestionsOf(*row.Find("questions")));
+ ASSERT_TRUE(pred.ok()) << name << ": " << pred.error();
+ ComparePrediction(name, pred.value(), row, t);
+ }
+ Finish("single", t, 60);
+}
+
+TEST_P(OnnxRig, BatchPredictionsMatchPythonInInputOrder) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ LAYA_GOLDEN_LIST(batches, rig.golden, "batch");
+ LAYA_EXPECT_ROWS(batches, 3);
+ laya::agent::Agent* a = Shared(rig, graph, dir);
+ ASSERT_NE(a, nullptr);
+ Tally t;
+ for (const Value& rec : batches->items()) {
+ laya::agent::CallOptions opts;
+ if (!rec.Find("batch_size")->is_null()) opts.batch_size = rec.Find("batch_size")->as_int();
+ opts.sort_by_length = rec.Find("sort_by_length")->kind() == Kind::Bool && rec.Find("sort_by_length")->as_bool();
+ const std::vector<Value>& states = rec.Find("states")->items();
+ auto preds = a->PredictBatch(states, QuestionsOf(*rec.Find("questions")), opts);
+ ASSERT_TRUE(preds.ok()) << preds.error();
+ const std::vector<Value>& want = rec.Find("results")->items();
+ ASSERT_EQ(preds.value().size(), want.size());
+ ASSERT_EQ(want.size(), 9u);
+ for (size_t i = 0; i < want.size(); i++) ComparePrediction("batch/state" + std::to_string(i), preds.value()[i], want[i], t);
+ }
+ Finish("batch", t, 240);
+}
+
+TEST_P(OnnxRig, BatchOrderFollowsTheInputWhateverTheSortAndBatchSize) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ LAYA_GOLDEN_LIST(batches, rig.golden, "batch");
+ laya::agent::Agent* a = Shared(rig, graph, dir);
+ ASSERT_NE(a, nullptr);
+ const Value& rec = batches->items()[0];
+ std::vector<Value> states = rec.Find("states")->items();
+ const auto qs = QuestionsOf(*rec.Find("questions"));
+ auto fwd = a->PredictBatch(states, qs);
+ ASSERT_TRUE(fwd.ok()) << fwd.error();
+ std::vector<Value> rev(states.rbegin(), states.rend());
+ laya::agent::CallOptions o;
+ o.batch_size = 2;
+ o.sort_by_length = true;
+ auto back = a->PredictBatch(rev, qs, o);
+ ASSERT_TRUE(back.ok()) << back.error();
+ ASSERT_EQ(fwd.value().size(), back.value().size());
+ const size_t n = states.size();
+ for (size_t i = 0; i < n; i++) {
+ auto x = fwd.value()[i].ToJson(), y = back.value()[n - 1 - i].ToJson();
+ ASSERT_TRUE(x.ok() && y.ok());
+ auto px = laya::pyvalue::Parse(x.value()), py = laya::pyvalue::Parse(y.value());
+ Tally t;
+ Compare("state" + std::to_string(i), "", py.value(), px.value(), t); // within the numeric tolerances
+ EXPECT_EQ(t.failures, 0) << "state " << i << " changed with its position in the batch";
+ }
+}
+
+TEST_P(OnnxRig, ConcurrentPredictAgreesWithASerialCall) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");
+ laya::agent::Agent* a = Shared(rig, graph, dir);
+ ASSERT_NE(a, nullptr);
+ const size_t n = singles->size();
+ std::vector<laya::schema::Questions> qs;
+ std::vector<std::string> serial;
+ for (size_t i = 0; i < n; i++) {
+ qs.push_back(QuestionsOf(*singles->value(i).Find("questions")));
+ auto p = a->Predict(*singles->value(i).Find("state"), qs.back());
+ ASSERT_TRUE(p.ok()) << p.error();
+ serial.push_back(p.value().ToJson().value());
+ }
+ std::atomic<int> bad{0};
+ std::vector<std::thread> threads;
+ for (int th = 0; th < 8; th++) {
+ threads.emplace_back([&, th] {
+ for (int round = 0; round < 3; round++) {
+ for (size_t k = 0; k < n; k++) {
+ const size_t i = (k + size_t(th)) % n;
+ auto p = a->Predict(*singles->value(i).Find("state"), qs[i]);
+ if (!p.ok() || p.value().ToJson().value() != serial[i]) bad++;
+ }
+ }
+ });
+ }
+ for (auto& t : threads) t.join();
+ EXPECT_EQ(bad.load(), 0);
+}
+
+TEST_P(OnnxRig, ACancelledCallStopsBeforeTheRuntime) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");
+ laya::agent::Agent* a = Shared(rig, graph, dir);
+ ASSERT_NE(a, nullptr);
+ laya::agent::CallOptions o;
+ o.cancel.Cancel();
+ auto p = a->Predict(*singles->value(0).Find("state"), QuestionsOf(*singles->value(0).Find("questions")), o);
+ ASSERT_FALSE(p.ok());
+ EXPECT_TRUE(laya::agent::IsCancelled(p.error())) << p.error();
+}
+
+TEST_P(OnnxRig, SessionRunIsGuardedAgainstHostileBatches) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ laya::infer::Session& session = *s.value();
+ const laya::CancelToken live;
+
+ auto ok = session.Run(Tiny(2, 3), live);
+ ASSERT_TRUE(ok.ok()) << ok.error();
+ EXPECT_EQ(ok.value().logits.size(), 6u);
+ EXPECT_EQ(ok.value().act_logits.size(), 4u);
+
+ Batch wrong_rank = Tiny(2, 3);
+ wrong_rank.input_ids.pop_back(); // a flattened slice that no longer matches rows*length
+ EXPECT_FALSE(session.Run(wrong_rank, live).ok());
+
+ Batch huge = Tiny(1, 3);
+ huge.rows = int64_t(1) << 40; // far past the cap: refused before any allocation
+ EXPECT_FALSE(session.Run(huge, live).ok());
+ huge = Tiny(1, 3);
+ huge.length = int64_t(1) << 40;
+ EXPECT_FALSE(session.Run(huge, live).ok());
+ Batch neg = Tiny(1, 3);
+ neg.length = -1;
+ EXPECT_FALSE(session.Run(neg, live).ok());
+
+ Batch one_marker = Tiny(1, 1); // the exported head takes topk(2)
+ EXPECT_FALSE(session.Run(one_marker, live).ok());
+ Batch oob = Tiny(1, 3);
+ oob.marker_pos[0] = 99;
+ EXPECT_FALSE(session.Run(oob, live).ok());
+ Batch mask2 = Tiny(1, 3);
+ mask2.marker_mask[1] = 2; // not a bool
+ EXPECT_FALSE(session.Run(mask2, live).ok());
+ Batch bad_q = Tiny(1, 3);
+ bad_q.qtype[0] = 7;
+ EXPECT_FALSE(session.Run(bad_q, live).ok());
+ Batch empty;
+ EXPECT_FALSE(session.Run(empty, live).ok());
+
+ laya::CancelToken fired;
+ fired.Cancel();
+ auto c = session.Run(Tiny(1, 3), fired);
+ ASSERT_FALSE(c.ok());
+ EXPECT_TRUE(laya::infer::IsCancelled(c.error()));
+
+ // The session still works after every refusal.
+ EXPECT_TRUE(session.Run(Tiny(1, 3), live).ok());
+}
+
+TEST_P(OnnxRig, CloseIsIdempotentAndRunAfterCloseIsAnError) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ EXPECT_EQ(s.value()->Close(), "");
+ EXPECT_EQ(s.value()->Close(), "");
+ auto r = s.value()->Run(Tiny(1, 3), laya::CancelToken());
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("closed"), std::string::npos) << r.error();
+}
+
+TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto first = laya::onnx::Open(graph);
+ ASSERT_TRUE(first.ok()) << first.error();
+ std::atomic<int> bad{0};
+ std::thread worker([&] {
+ for (int i = 0; i < 20; i++) {
+ if (!first.value()->Run(Tiny(2, 3), laya::CancelToken()).ok()) bad++;
+ }
+ });
+ for (int i = 0; i < 3; i++) {
+ laya::onnx::Options o;
+ o.intra_op_threads = 1;
+ o.inter_op_threads = 1;
+ auto other = laya::onnx::Open(graph, o); // a second session, dropped without Close
+ if (!other.ok() || !other.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()) bad++;
+ }
+ worker.join();
+ EXPECT_EQ(bad.load(), 0);
+}
+
+TEST_P(OnnxRig, OpenOnnxWithAWrongCheckpointDirIsAnError) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ TempDir empty;
+ const auto a = laya::agent::OpenOnnx(empty.str(), graph);
+ EXPECT_FALSE(a.ok());
+}
+
+TEST_P(OnnxRig, TheGraphWithoutItsDataFileIsAnError) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ TempDir d;
+ fs::copy_file(graph, d.str() + "/laya.onnx"); // the .data file stays behind
+ const auto r = laya::onnx::Open(d.str() + "/laya.onnx");
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
+}
+
+} // namespace