Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(cpp): ONNX Runtime backend laya_onnx and the onnx lane
Adeshina Lasisi4e9628def302
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex a9fd5a7b..9737d665 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -43,6 +43,8 @@ env:# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus# compile_flags). Raise it with every test added.MIN_TESTS: "423"+ # The onnx job registers the same tests plus the laya_onnx_tests cases.+ MIN_TESTS_ONNX: "466"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints@@ -120,6 +122,110 @@ jobs:if: ${{ !cancelled() }}run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest.xml" --min-tests "$MIN_TESTS"+ # The ONNX Runtime backend, mirroring go.yml's `onnx` job: the same graph cache keys as java.yml+ # (so a graph exported there is reused), real graphs for both shipped checkpoints, goldens+ # recorded by Python's ONNXAgent on those graphs, and a floor on the tests that must run. The+ # runtime comes from the release archive pinned by sha256 in laya-cpp/cmake/onnxruntime.cmake:+ # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`.+ onnx:+ name: onnx (${{ matrix.compiler }}, ${{ matrix.preset }})+ runs-on: ubuntu-latest+ needs: security+ continue-on-error: true # advisory: see the header+ timeout-minutes: 90+ strategy:+ fail-fast: false+ matrix:+ compiler: [g++]+ preset: [debug]+ include:+ - compiler: clang+++ preset: asan-ubsan+ - compiler: clang+++ preset: tsan+ env:+ ONNX_ROOT: ${{ github.workspace }}/laya-java/.work/onnx+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.12"+ cache: pip+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt+ - name: Install a comma-decimal locale+ run: sudo locale-gen de_DE.UTF-8+ - name: Install the Python toolchain the goldens are recorded with+ run: |+ python -m pip install --upgrade pip+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu+ pip install -r laya-dotnet/tools/requirements-regen.txt++ # Keys are byte-identical to the predict cells in java.yml, so both share the entries.+ - name: Cache the multilingual graph+ id: graph-multilingual+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: laya-java/.work/onnx+ key: laya-java-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}+ - name: Cache the english graph+ id: graph-english+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: laya-java/.work/onnx+ key: laya-java-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}+ - name: Prepare both checkpoints and their graphs+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN }}+ HIT_MULTILINGUAL: ${{ steps.graph-multilingual.outputs.cache-hit }}+ HIT_ENGLISH: ${{ steps.graph-english.outputs.cache-hit }}+ run: |+ for c in multilingual english; do+ hit=HIT_${c^^}+ if [ "${!hit}" = "true" ]; then+ python laya-java/scripts/prepare_checkpoint.py --checkpoint "$c" --no-graph+ else+ python laya-java/scripts/prepare_checkpoint.py --checkpoint "$c"+ fi+ for f in laya.onnx laya.onnx.data laya.onnx.laya-revision; do+ test -s "$ONNX_ROOT/$c/$f" || { echo "::error::missing $c/$f"; exit 1; }+ done+ done++ - name: Regenerate the goldens from Python+ env:+ LAYA_FIXTURE_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints+ PYTHONPATH: ${{ github.workspace }}+ run: |+ python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json --only sequences.json --only window_plan.json --only decode.json+ python laya-go/scripts/gen_go_fixtures.py --out "$GOLDENS"+ # predict.json is recorded per checkpoint from that checkpoint's own graph.+ for c in multilingual english; do+ LAYA_ONNX_GRAPH="$ONNX_ROOT/$c/laya.onnx" LAYA_PREDICT_MODEL="$c" \+ python laya-java/scripts/gen_fixtures.py --strict --out "$RUNNER_TEMP/predict-$c" --only predict.json+ done+ cp "$RUNNER_TEMP/predict-multilingual/predict.json" "$GOLDENS/predict.json"+ cp "$RUNNER_TEMP/predict-english/predict.json" "$GOLDENS/predict-english.json"+ - name: Compiled C++ tables match CPython+ env:+ PYTHONPATH: ${{ github.workspace }}+ run: python laya-go/scripts/gen_tables.py --lang cpp --check++ - name: Configure (fetches the pinned ONNX Runtime and verifies its sha256)+ working-directory: laya-cpp+ run: cmake --preset "${{ matrix.preset }}" -DLAYA_ONNX=ON -DCMAKE_CXX_COMPILER="${{ matrix.compiler }}"+ - name: Build+ working-directory: laya-cpp+ run: cmake --build --preset "${{ matrix.preset }}"+ - name: Test+ working-directory: laya-cpp+ env:+ LAYA_ONNX_GRAPH: ${{ env.ONNX_ROOT }}/multilingual/laya.onnx+ LAYA_ONNX_GRAPH_ENGLISH: ${{ env.ONNX_ROOT }}/english/laya.onnx+ run: ctest --preset "${{ matrix.preset }}" --output-junit "$RUNNER_TEMP/ctest-onnx.xml"+ - name: Count tests (fail on skip, enforce the onnx floor)+ if: ${{ !cancelled() }}+ run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest-onnx.xml" --min-tests "$MIN_TESTS_ONNX"+security:name: cpp security scanruns-on: ubuntu-latestdiff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex 3f8bfe42..fe87d4b6 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.20)project(laya_cpp VERSION 0.0.0 LANGUAGES CXX)option(LAYA_BUILD_TESTS "Build the GoogleTest suite" ON)-option(LAYA_ONNX "Build the ONNX Runtime backend (not implemented yet)" OFF)+option(LAYA_ONNX "Build laya_onnx, the ONNX Runtime backend (fetches the pinned runtime unless LAYA_ONNXRUNTIME_ROOT is set)" OFF)+set(LAYA_ONNXRUNTIME_ROOT "" CACHE PATH "An unpacked onnxruntime release (include/ and lib/) to use instead of the download")set(CMAKE_CXX_STANDARD 17)set(CMAKE_CXX_STANDARD_REQUIRED ON)@@ -31,7 +32,11 @@ target_include_directories(laya PUBLIC include)laya_target_flags(laya)if(LAYA_ONNX)- message(FATAL_ERROR "LAYA_ONNX is not implemented yet")+ include(cmake/onnxruntime.cmake)+ add_library(laya_onnx src/onnx_graph.cpp src/onnx.cpp)+ target_include_directories(laya_onnx PUBLIC include)+ target_link_libraries(laya_onnx PUBLIC laya PRIVATE laya_ort)+ laya_target_flags(laya_onnx)endif()if(LAYA_BUILD_TESTS)@@ -63,10 +68,22 @@ if(LAYA_BUILD_TESTS)target_link_libraries(laya_c4_cli PRIVATE laya)laya_target_flags(laya_c4_cli)+ if(LAYA_ONNX)+ add_executable(laya_onnx_tests tests/onnx_test.cpp)+ target_link_libraries(laya_onnx_tests PRIVATE laya_onnx laya_ort GTest::gtest_main)+ laya_target_flags(laya_onnx_tests)+ laya_ort_runtime_path(laya_onnx_tests)+ target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}")+ endif()+include(GoogleTest)# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.gtest_discover_tests(laya_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)+ if(LAYA_ONNX)+ gtest_discover_tests(laya_onnx_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)+ endif()+# Listing must stay cheap (it runs before every ctest): bound it at 3 s with the goldens loaded.add_test(NAME discovery_fastCOMMAND ${CMAKE_COMMAND} -DEXE=$<TARGET_FILE:laya_tests> -DLIMIT=3diff --git a/laya-cpp/cmake/onnxruntime.cmake b/laya-cpp/cmake/onnxruntime.cmakenew file mode 100644index 00000000..6d8178eb--- /dev/null+++ b/laya-cpp/cmake/onnxruntime.cmake@@ -0,0 +1,55 @@+# The ONNX Runtime used by laya_onnx: LAYA_ONNXRUNTIME_ROOT (an unpacked release) or the pinned+# release archive, verified by sha256 at the moment of download. Defines the imported target laya_ort.+set(LAYA_ORT_VERSION "1.30.0")+include(FetchContent)++if(LAYA_ONNXRUNTIME_ROOT)+ set(_ort_root "${LAYA_ONNXRUNTIME_ROOT}")+else()+ set(_ort_base "https://github.com/microsoft/onnxruntime/releases/download/v${LAYA_ORT_VERSION}")+ string(TOLOWER "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}" _ort_platform)+ if(_ort_platform STREQUAL "linux-x86_64")+ set(_ort_asset "onnxruntime-linux-x64-${LAYA_ORT_VERSION}.tgz")+ set(_ort_sha256 "a5ed5a3cac51fbb2e90da632ae43d19212faaa20e76484e62bcb7c23ddb3b3fd")+ elseif(_ort_platform STREQUAL "darwin-arm64")+ set(_ort_asset "onnxruntime-osx-arm64-${LAYA_ORT_VERSION}.tgz")+ set(_ort_sha256 "6ebb5062a934537c352937821f9fe9718e7de1a2db1122a93dd363ffd53a7012")+ else()+ message(FATAL_ERROR "No pinned ONNX Runtime for ${_ort_platform}; unpack a ${LAYA_ORT_VERSION} release and pass -DLAYA_ONNXRUNTIME_ROOT=<dir>")+ endif()+ # URL_HASH makes CMake verify the archive before it is extracted; a mismatch is a configure error.+ FetchContent_Declare(onnxruntime+ URL "${_ort_base}/${_ort_asset}"+ URL_HASH SHA256=${_ort_sha256}+ DOWNLOAD_EXTRACT_TIMESTAMP TRUE)+ FetchContent_MakeAvailable(onnxruntime)+ set(_ort_root "${onnxruntime_SOURCE_DIR}")+endif()++if(NOT EXISTS "${_ort_root}/include/onnxruntime_cxx_api.h")+ message(FATAL_ERROR "${_ort_root}/include/onnxruntime_cxx_api.h not found: not an ONNX Runtime release directory")+endif()+if(APPLE)+ set(_ort_lib "${_ort_root}/lib/libonnxruntime.${LAYA_ORT_VERSION}.dylib")+ if(NOT EXISTS "${_ort_lib}")+ file(GLOB _ort_lib "${_ort_root}/lib/libonnxruntime*.dylib")+ list(GET _ort_lib 0 _ort_lib)+ endif()+elseif(WIN32)+ set(_ort_lib "${_ort_root}/lib/onnxruntime.lib")+else()+ set(_ort_lib "${_ort_root}/lib/libonnxruntime.so")+endif()+if(NOT EXISTS "${_ort_lib}")+ message(FATAL_ERROR "ONNX Runtime library not found: ${_ort_lib}")+endif()+message(STATUS "ONNX Runtime ${LAYA_ORT_VERSION}: ${_ort_lib}")++add_library(laya_ort SHARED IMPORTED)+set_target_properties(laya_ort PROPERTIES IMPORTED_LOCATION "${_ort_lib}" INTERFACE_INCLUDE_DIRECTORIES "${_ort_root}/include")+set(LAYA_ORT_LIBDIR "${_ort_root}/lib")++# Build-tree executables find the shared library through an rpath to its directory.+function(laya_ort_runtime_path target)+ set_property(TARGET ${target} APPEND PROPERTY BUILD_RPATH "${LAYA_ORT_LIBDIR}")+endfunction()diff --git a/laya-cpp/include/laya/onnx.hpp b/laya-cpp/include/laya/onnx.hppnew file mode 100644index 00000000..f391721b--- /dev/null+++ b/laya-cpp/include/laya/onnx.hpp@@ -0,0 +1,36 @@+#pragma once+// The ONNX Runtime backend (laya-go/onnx). Linking laya_onnx pulls in ONNX Runtime; the core `laya`+// library never does, and this header includes no ONNX Runtime header.+#include <cstdint>+#include <memory>+#include <string>+#include "laya/agent.hpp"+#include "laya/infer.hpp"+#include "laya/result.hpp"++namespace laya {+namespace onnx {++struct Options {+ int intra_op_threads = 0; // 0 leaves ONNX Runtime's default; at most kMaxThreads+ int inter_op_threads = 0;+};+constexpr int kMaxThreads = 4096;++// The version of the ONNX Runtime library loaded, e.g. "1.30.0".+std::string RuntimeVersion();++// A Session over one fused laya.onnx (+ laya.onnx.data beside it). Checks the graph's inputs and+// outputs at once. Run is thread-safe; Close is idempotent and waits for runs in flight. No+// exception leaves this API.+Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options = {});++} // namespace onnx++namespace agent {+// checkpoint_dir holds rl_agent_config.json and the tokenizer; graph_path is the fused laya.onnx.+Result<std::unique_ptr<Agent>> OpenOnnx(const std::string& checkpoint_dir, const std::string& graph_path,+ const onnx::Options& options = {});+} // namespace agent++} // namespace layadiff --git a/laya-cpp/include/laya/onnx_graph.hpp b/laya-cpp/include/laya/onnx_graph.hppnew file mode 100644index 00000000..0a5eeed7--- /dev/null+++ b/laya-cpp/include/laya/onnx_graph.hpp@@ -0,0 +1,39 @@+#pragma once+// The ONNX backend's checks that need no ONNX Runtime (laya-go/onnx/graph.go): what a graph must+// declare, where a graph path may point, and how a run's outputs become an infer::Output.+#include <cstdint>+#include <string>+#include <vector>+#include "laya/infer.hpp"+#include "laya/result.hpp"++namespace laya {+namespace onnx {++struct TensorInfo {+ std::string name;+ std::string dtype; // "int64", "bool", "float32", or whatever else the graph declares+ std::vector<int64_t> dims; // -1 for a symbolic or unknown dimension+};++// "" when the graph's own metadata is what the agent feeds: five inputs (int64, int64, int64, bool,+// int64) and two float32 outputs, matching by name, no extras, dynamic batch dimension, and a fixed+// 2 columns for act_logits. Else the first problem.+std::string CheckGraph(const std::vector<TensorInfo>& inputs, const std::vector<TensorInfo>& outputs);++// "" when path names a file that may be opened as the fused graph; refuses the split+// encoder.onnx + head.onnx layout with a message that says how to re-export.+std::string CheckPath(const std::string& path);++struct OutTensor {+ std::vector<int64_t> shape;+ const float* data = nullptr;+ int64_t count = 0; // elements at data+};++// Copies the two outputs after checking their shapes against b, then Output::Check. NaN anywhere is+// an error: a NaN would decode to a silent first-max answer.+Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logits, const OutTensor& act_logits);++} // namespace onnx+} // namespace layadiff --git a/laya-cpp/src/onnx.cpp b/laya-cpp/src/onnx.cppnew file mode 100644index 00000000..878e6f07--- /dev/null+++ b/laya-cpp/src/onnx.cpp@@ -0,0 +1,191 @@+#include "laya/onnx.hpp"++#include <onnxruntime_cxx_api.h>++#include <filesystem>+#include <memory>+#include <mutex>+#include <shared_mutex>++#include "laya/onnx_graph.hpp"++namespace laya {+namespace onnx {+namespace {++// One Ort::Env per process, shared by every session and freed with the last one, so there is no+// static destructor racing a session's teardown. The mutex orders creation against that release.+std::shared_ptr<Ort::Env> SharedEnv() {+ static std::mutex mu;+ static std::weak_ptr<Ort::Env> weak;+ std::lock_guard<std::mutex> lock(mu);+ std::shared_ptr<Ort::Env> env = weak.lock();+ if (!env) {+ env = std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_ERROR, "laya");+ weak = env;+ }+ return env;+}++std::string Dtype(ONNXTensorElementDataType t) {+ switch (t) {+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return "int64";+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return "bool";+ case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return "float32";+ default: return "type " + std::to_string(int(t));+ }+}++// Inputs by name, in the order the five tensors are bound below.+const char* const kInputNames[5] = {"input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"};+const char* const kOutputNames[2] = {"logits", "act_logits"};++class OrtSession final : public infer::Session {+ public:+ OrtSession(std::shared_ptr<Ort::Env> env, std::unique_ptr<Ort::Session> s) : env_(std::move(env)), session_(std::move(s)) {}+ ~OrtSession() override { Close(); }++ Result<infer::Output> Run(const infer::Batch& b, const CancelToken& cancel) override {+ try {+ return RunImpl(b, cancel);+ } catch (const Ort::Exception& e) {+ return Result<infer::Output>::Fail(std::string("onnx: run: ") + e.what());+ } catch (const std::exception& e) {+ return Result<infer::Output>::Fail(std::string("onnx: run: ") + e.what());+ } catch (...) {+ return Result<infer::Output>::Fail("onnx: run: unknown exception");+ }+ }++ std::string Close() override {+ try {+ std::unique_lock<std::shared_mutex> lock(mu_); // waits for runs in flight+ session_.reset();+ return "";+ } catch (...) {+ return "onnx: close failed";+ }+ }++ private:+ Result<infer::Output> RunImpl(const infer::Batch& b, const CancelToken& cancel) {+ using R = Result<infer::Output>;+ if (cancel.cancelled()) return R::Fail(infer::kCancelled);+ const std::string bad = b.Validate();+ if (!bad.empty()) return R::Fail("onnx: " + bad);+ for (uint8_t m : b.marker_mask) {+ if (m > 1) return R::Fail("onnx: marker_mask holds a value other than 0 or 1");+ }+ std::shared_lock<std::shared_mutex> lock(mu_);+ if (!session_) return R::Fail("onnx: session is closed");++ const Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);+ const int64_t shape2[2] = {b.rows, b.length};+ const int64_t shape_m[2] = {b.rows, b.markers};+ const int64_t shape1[1] = {b.rows};+ // ORT takes non-const pointers but never writes to an input.+ auto i64 = [&](const std::vector<int64_t>& v, const int64_t* shape, size_t rank) {+ return Ort::Value::CreateTensor<int64_t>(mem, const_cast<int64_t*>(v.data()), v.size(), shape, rank);+ };+ std::vector<Ort::Value> in;+ in.reserve(5);+ in.push_back(i64(b.input_ids, shape2, 2));+ in.push_back(i64(b.attention_mask, shape2, 2));+ in.push_back(i64(b.marker_pos, shape_m, 2));+ in.push_back(Ort::Value::CreateTensor(mem, const_cast<uint8_t*>(b.marker_mask.data()), b.marker_mask.size() * sizeof(uint8_t),+ shape_m, 2, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL));+ in.push_back(i64(b.qtype, shape1, 1));++ Ort::RunOptions ro;+ std::vector<Ort::Value> out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);+ if (out.size() != 2) return R::Fail("onnx: run returned " + std::to_string(out.size()) + " outputs, want 2");+ if (cancel.cancelled()) return R::Fail(infer::kCancelled);++ OutTensor t[2];+ for (size_t i = 0; i < 2; i++) {+ if (!out[i].IsTensor()) return R::Fail(std::string("onnx: output ") + kOutputNames[i] + " is not a tensor");+ const Ort::TensorTypeAndShapeInfo info = out[i].GetTensorTypeAndShapeInfo();+ if (info.GetElementType() != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {+ return R::Fail(std::string("onnx: output ") + kOutputNames[i] + " is " + Dtype(info.GetElementType()) + ", want float32");+ }+ t[i].shape = info.GetShape();+ t[i].count = int64_t(info.GetElementCount());+ t[i].data = out[i].GetTensorData<float>();+ }+ return CollectOutput(b, t[0], t[1]);+ }++ std::shared_ptr<Ort::Env> env_; // keeps the environment alive past session_+ std::shared_mutex mu_;+ std::unique_ptr<Ort::Session> session_;+};++} // namespace++std::string RuntimeVersion() {+ try {+ return Ort::GetVersionString();+ } catch (...) {+ return "";+ }+}++Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options) {+ using R = Result<std::unique_ptr<infer::Session>>;+ if (options.intra_op_threads < 0 || options.inter_op_threads < 0 || options.intra_op_threads > kMaxThreads ||+ options.inter_op_threads > kMaxThreads) {+ return R::Fail("onnx: thread counts must be between 0 and " + std::to_string(kMaxThreads));+ }+ const std::string path_problem = CheckPath(graph_path);+ if (!path_problem.empty()) return R::Fail("onnx: " + path_problem);+ try {+ std::shared_ptr<Ort::Env> env = SharedEnv();+ Ort::SessionOptions so;+ if (options.intra_op_threads > 0) so.SetIntraOpNumThreads(options.intra_op_threads);+ if (options.inter_op_threads > 0) so.SetInterOpNumThreads(options.inter_op_threads);+ const std::filesystem::path p(graph_path);+ auto session = std::make_unique<Ort::Session>(*env, p.c_str(), so);++ Ort::AllocatorWithDefaultOptions alloc;+ auto describe = [&](bool inputs) {+ std::vector<TensorInfo> v;+ const size_t n = inputs ? session->GetInputCount() : session->GetOutputCount();+ for (size_t i = 0; i < n; i++) {+ TensorInfo t;+ Ort::AllocatedStringPtr name = inputs ? session->GetInputNameAllocated(i, alloc) : session->GetOutputNameAllocated(i, alloc);+ t.name = name.get();+ Ort::TypeInfo ti = inputs ? session->GetInputTypeInfo(i) : session->GetOutputTypeInfo(i);+ if (ti.GetONNXType() != ONNX_TYPE_TENSOR) {+ t.dtype = "not a tensor";+ } else {+ const Ort::ConstTensorTypeAndShapeInfo info = ti.GetTensorTypeAndShapeInfo();+ t.dtype = Dtype(info.GetElementType());+ t.dims = info.GetShape();+ }+ v.push_back(std::move(t));+ }+ return v;+ };+ const std::string bad = CheckGraph(describe(true), describe(false));+ if (!bad.empty()) return R::Fail("onnx: graph " + graph_path + ": " + bad);+ return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(env), std::move(session))));+ } catch (const Ort::Exception& e) {+ return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());+ } catch (const std::exception& e) {+ return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());+ } catch (...) {+ return R::Fail("onnx: cannot open " + graph_path + ": unknown exception");+ }+}++} // namespace onnx++namespace agent {+Result<std::unique_ptr<Agent>> OpenOnnx(const std::string& checkpoint_dir, const std::string& graph_path, const onnx::Options& options) {+ auto s = onnx::Open(graph_path, options);+ if (!s.ok()) return Result<std::unique_ptr<Agent>>::Fail(s.error());+ return Agent::Open(checkpoint_dir, std::move(s).value()); // closes the session on failure+}+} // namespace agent++} // namespace layadiff --git a/laya-cpp/src/onnx_graph.cpp b/laya-cpp/src/onnx_graph.cppnew file mode 100644index 00000000..808c0b1f--- /dev/null+++ b/laya-cpp/src/onnx_graph.cpp@@ -0,0 +1,133 @@+#include "laya/onnx_graph.hpp"++#include <cmath>+#include <filesystem>+#include <set>++namespace laya {+namespace onnx {+namespace {++const char* const kReexport = "re-export with scripts/export_onnx.py";+const std::string kSplitProblem = std::string("the split encoder.onnx + head.onnx layout is not supported; ") + kReexport ++ " to write one fused laya.onnx";++struct Spec {+ const char* name;+ const char* dtype;+ size_t rank;+ int64_t fixed_cols; // 0: every dimension after the batch is dynamic+};++const Spec kInputs[] = {{"input_ids", "int64", 2, 0},+ {"attention_mask", "int64", 2, 0},+ {"marker_pos", "int64", 2, 0},+ {"marker_mask", "bool", 2, 0},+ {"qtype", "int64", 1, 0}};+const Spec kOutputs[] = {{"logits", "float32", 2, 0}, {"act_logits", "float32", 2, 2}};++template <size_t N>+std::string CheckSide(const char* kind, const std::vector<TensorInfo>& got, const Spec (&want)[N]) {+ for (const Spec& w : want) {+ const TensorInfo* g = nullptr;+ for (const TensorInfo& t : got) {+ if (t.name == w.name) {+ g = &t;+ break;+ }+ }+ const std::string q = std::string(kind) + " \"" + w.name + "\"";+ if (g == nullptr) return std::string("missing ") + q + "; " + kReexport;+ if (g->dtype != w.dtype) return q + " is " + g->dtype + ", want " + w.dtype + "; " + kReexport;+ if (g->dims.size() != w.rank) {+ return q + " has rank " + std::to_string(g->dims.size()) + ", want " + std::to_string(w.rank) + "; " + kReexport;+ }+ for (size_t i = 0; i < g->dims.size(); i++) {+ const int64_t d = g->dims[i];+ if (d == -1 || (i > 0 && w.fixed_cols > 0 && d == w.fixed_cols)) continue;+ return q + " declares " + (i == 0 ? "a fixed batch dimension " : "a fixed size ") + std::to_string(d) + "; " + kReexport;+ }+ }+ std::set<std::string> known;+ for (const Spec& w : want) known.insert(w.name);+ for (const TensorInfo& t : got) {+ if (known.count(t.name) == 0) {+ return std::string("unexpected ") + kind + " \"" + t.name + "\"; the agent feeds only " + std::to_string(sizeof(kInputs) / sizeof(kInputs[0])) ++ " inputs and " + std::to_string(sizeof(kOutputs) / sizeof(kOutputs[0])) + " outputs; " + kReexport;+ }+ }+ if (got.size() != N) return std::string("duplicate ") + kind + " name; " + kReexport;+ return "";+}++bool IsSplit(const std::filesystem::path& dir) {+ std::error_code ec;+ for (const char* n : {"encoder.onnx", "head.onnx"}) {+ if (!std::filesystem::is_regular_file(dir / n, ec)) return false;+ }+ return true;+}++std::string Shape(const std::vector<int64_t>& s) {+ std::string out = "[";+ for (size_t i = 0; i < s.size(); i++) out += (i ? " " : "") + std::to_string(s[i]);+ return out + "]";+}++} // namespace++std::string CheckGraph(const std::vector<TensorInfo>& inputs, const std::vector<TensorInfo>& outputs) {+ std::string p = CheckSide("input", inputs, kInputs);+ if (p.empty()) p = CheckSide("output", outputs, kOutputs);+ return p;+}++std::string CheckPath(const std::string& path) {+ namespace fs = std::filesystem;+ if (path.empty()) return "graph path is empty";+ std::error_code ec;+ const fs::path p(path);+ const fs::file_status st = fs::status(p, ec);+ if (!ec && fs::is_directory(st)) {+ return IsSplit(p) ? kSplitProblem : "graph " + path + " is a directory; pass the path of laya.onnx";+ }+ if (!ec && fs::is_regular_file(st)) {+ const std::string base = p.filename().string();+ if ((base == "encoder.onnx" || base == "head.onnx") && IsSplit(p.parent_path())) return kSplitProblem;+ return "";+ }+ const fs::path dir = p.has_parent_path() ? p.parent_path() : fs::path(".");+ if (st.type() == fs::file_type::not_found && IsSplit(dir)) return "graph " + path + ": no such file; " + kSplitProblem;+ if (st.type() == fs::file_type::not_found) return "graph " + path + ": no such file";+ return "graph " + path + ": " + (ec ? ec.message() : std::string("not a regular file"));+}++Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logits, const OutTensor& act) {+ struct Col {+ const char* name;+ const OutTensor& t;+ int64_t cols;+ };+ for (const Col& c : {Col{"logits", logits, b.markers}, Col{"act_logits", act, 2}}) {+ if (c.t.shape.size() != 2 || c.t.shape[0] != b.rows || c.t.shape[1] != c.cols) {+ return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " has shape " + Shape(c.t.shape) + ", want [" ++ std::to_string(b.rows) + " " + std::to_string(c.cols) + "]");+ }+ // rows and cols are bounded by Batch::Validate, so this product cannot overflow.+ if (c.t.count != b.rows * c.cols || (c.t.count > 0 && c.t.data == nullptr)) {+ return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " holds " + std::to_string(c.t.count) + " values");+ }+ for (int64_t i = 0; i < c.t.count; i++) {+ if (std::isnan(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN");+ }+ }+ infer::Output o;+ o.logits.assign(logits.data, logits.data + logits.count);+ o.act_logits.assign(act.data, act.data + act.count);+ const std::string bad = o.Check(b);+ if (!bad.empty()) return Result<infer::Output>::Fail("onnx: " + bad);+ return Result<infer::Output>::Ok(std::move(o));+}++} // namespace onnx+} // namespace layadiff --git a/laya-cpp/tests/onnx_test.cpp b/laya-cpp/tests/onnx_test.cppnew file mode 100644index 00000000..72e33341--- /dev/null+++ b/laya-cpp/tests/onnx_test.cpp@@ -0,0 +1,662 @@+// Tests of the ONNX backend. The pure checks run anywhere; the tests that load a graph need+// LAYA_ONNX_GRAPH (multilingual), LAYA_ONNX_GRAPH_ENGLISH and LAYA_CHECKPOINTS, and with+// LAYA_REQUIRE_GOLDENS=1 (the onnx lane) a missing one FAILS instead of skipping.+#include <gtest/gtest.h>++#include <atomic>+#include <cmath>+#include <cstdio>+#include <filesystem>+#include <fstream>+#include <limits>+#include <map>+#include <mutex>+#include <thread>++#include <unistd.h>++#include "golden.hpp"+#include "laya/onnx.hpp"+#include "laya/onnx_graph.hpp"++namespace {++namespace fs = std::filesystem;+using laya::Result;+using laya::infer::Batch;+using laya::onnx::OutTensor;+using laya::onnx::TensorInfo;+using laya::pyvalue::Value;+using Kind = Value::Kind;++// ---------------------------------------------------------------- graph checks (no runtime needed)++std::vector<TensorInfo> GoodInputs() {+ return {{"input_ids", "int64", {-1, -1}},+ {"attention_mask", "int64", {-1, -1}},+ {"marker_pos", "int64", {-1, -1}},+ {"marker_mask", "bool", {-1, -1}},+ {"qtype", "int64", {-1}}};+}+std::vector<TensorInfo> GoodOutputs() { return {{"logits", "float32", {-1, -1}}, {"act_logits", "float32", {-1, 2}}}; }++TEST(OnnxGraph, TheExportedShapeIsAccepted) { EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), GoodOutputs()), ""); }++TEST(OnnxGraph, InputOrderInTheGraphDoesNotMatter) {+ auto in = GoodInputs();+ std::swap(in[0], in[4]);+ std::swap(in[1], in[3]);+ EXPECT_EQ(laya::onnx::CheckGraph(in, GoodOutputs()), "");+}++TEST(OnnxGraph, EveryInputIsHeldToItsName) {+ for (size_t i = 0; i < 5; i++) {+ auto in = GoodInputs();+ in[i].name = "renamed";+ const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());+ EXPECT_NE(e.find("missing input"), std::string::npos) << i << ": " << e;+ }+}++TEST(OnnxGraph, EveryInputIsHeldToItsDtype) {+ for (size_t i = 0; i < 5; i++) {+ auto in = GoodInputs();+ in[i].dtype = in[i].dtype == "int64" ? "float32" : "int64";+ const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());+ EXPECT_NE(e.find(in[i].name), std::string::npos) << e;+ EXPECT_NE(e.find("want"), std::string::npos) << e;+ }+}++TEST(OnnxGraph, TheMaskMustBeBoolNotInt64) {+ auto in = GoodInputs();+ in[3].dtype = "int64";+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("is int64, want bool"), std::string::npos);+}++TEST(OnnxGraph, RankIsChecked) {+ auto in = GoodInputs();+ in[4].dims = {-1, -1};+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 2, want 1"), std::string::npos);+ in = GoodInputs();+ in[0].dims = {-1};+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 1, want 2"), std::string::npos);+}++TEST(OnnxGraph, AFixedBatchDimensionIsRefused) {+ auto out = GoodOutputs();+ out[1].dims = {1, 2}; // the layout of onnx/full-multilingual: act_logits traced at batch 1+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed batch dimension 1"), std::string::npos);+ auto in = GoodInputs();+ in[0].dims = {4, -1};+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("fixed batch dimension 4"), std::string::npos);+}++TEST(OnnxGraph, ActLogitsHasExactlyTwoColumns) {+ auto out = GoodOutputs();+ out[1].dims = {-1, 3};+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed size 3"), std::string::npos);+ out[1].dims = {-1, -1};+ EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), out), "");+}++TEST(OnnxGraph, ExtrasAndDuplicatesAreRefused) {+ auto in = GoodInputs();+ in.push_back({"extra", "int64", {-1}});+ EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("unexpected input \"extra\""), std::string::npos);+ auto out = GoodOutputs();+ out.push_back({"logits", "float32", {-1, -1}});+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("duplicate output"), std::string::npos);+ EXPECT_NE(laya::onnx::CheckGraph({}, GoodOutputs()).find("missing input"), std::string::npos);+}++TEST(OnnxGraph, AnOutputDtypeIsChecked) {+ auto out = GoodOutputs();+ out[0].dtype = "int64";+ EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("output \"logits\" is int64"), std::string::npos);+}++// ---------------------------------------------------------------- paths++class TempDir {+ public:+ TempDir() {+ static std::atomic<int> n{0};+ path_ = fs::temp_directory_path() / ("laya_onnx_test_" + std::to_string(::getpid()) + "_" + std::to_string(n++));+ fs::create_directories(path_);+ }+ ~TempDir() {+ std::error_code ec;+ fs::remove_all(path_, ec);+ }+ TempDir(const TempDir&) = delete;+ TempDir& operator=(const TempDir&) = delete;+ std::string File(const std::string& name, const std::string& body = "x") const {+ std::ofstream(path_ / name, std::ios::binary) << body;+ return (path_ / name).string();+ }+ std::string str() const { return path_.string(); }++ private:+ fs::path path_;+};++TEST(OnnxPath, MissingAndDirectoryAndEmptyAreErrors) {+ TempDir d;+ EXPECT_NE(laya::onnx::CheckPath(d.str() + "/nope.onnx").find("no such file"), std::string::npos);+ EXPECT_NE(laya::onnx::CheckPath(d.str()).find("is a directory"), std::string::npos);+ EXPECT_NE(laya::onnx::CheckPath("").find("empty"), std::string::npos);+ EXPECT_EQ(laya::onnx::CheckPath(d.File("laya.onnx")), "");+}++TEST(OnnxPath, TheSplitLayoutIsRefusedWhicheverPathIsGiven) {+ TempDir d;+ const std::string enc = d.File("encoder.onnx");+ const std::string head = d.File("head.onnx");+ for (const std::string& p : {enc, head, d.str(), d.str() + "/laya.onnx"}) {+ const std::string e = laya::onnx::CheckPath(p);+ EXPECT_NE(e.find("split encoder.onnx + head.onnx layout is not supported"), std::string::npos) << p << ": " << e;+ EXPECT_NE(e.find("export_onnx.py"), std::string::npos) << e;+ }+}++TEST(OnnxPath, OneHalfAloneIsNotTheSplitLayout) {+ TempDir d;+ const std::string enc = d.File("encoder.onnx");+ EXPECT_EQ(laya::onnx::CheckPath(enc), "");+}++// ---------------------------------------------------------------- collecting outputs++Batch Tiny(int64_t rows, int64_t markers) {+ Batch b;+ b.rows = rows;+ b.length = 3;+ b.markers = markers;+ b.input_ids.assign(size_t(rows * 3), 1);+ b.attention_mask.assign(size_t(rows * 3), 1);+ b.marker_pos.assign(size_t(rows * markers), 1);+ b.marker_mask.assign(size_t(rows * markers), 1);+ b.qtype.assign(size_t(rows), 0);+ return b;+}++OutTensor Tensor(const std::vector<float>& v, std::vector<int64_t> shape) {+ OutTensor t;+ t.shape = std::move(shape);+ t.data = v.data();+ t.count = int64_t(v.size());+ return t;+}++TEST(OnnxCollect, ACorrectPairIsCopied) {+ const Batch b = Tiny(2, 3);+ const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));+ ASSERT_TRUE(r.ok()) << r.error();+ EXPECT_EQ(r.value().logits, l);+ EXPECT_EQ(r.value().act_logits, a);+}++TEST(OnnxCollect, ASlicedOutputIsTheOutputOfTheRightRows) {+ const Batch b = Tiny(2, 3);+ const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));+ ASSERT_TRUE(r.ok());+ EXPECT_EQ(r.value().logits[3], 4.0f); // row 1, column 0+ EXPECT_EQ(r.value().act_logits[2], 9.0f);+}++TEST(OnnxCollect, ANaNIsAnErrorNotAnAnswer) {+ const Batch b = Tiny(1, 3);+ std::vector<float> l = {1, std::numeric_limits<float>::quiet_NaN(), 3}, a = {1, 2};+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("NaN"), std::string::npos);+ l[1] = 0;+ a[0] = std::numeric_limits<float>::quiet_NaN();+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());+}++TEST(OnnxCollect, WrongShapesAreErrors) {+ const Batch b = Tiny(2, 3);+ const std::vector<float> l(6, 1.f), a(4, 1.f), small(2, 1.f);+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {3, 2}), Tensor(a, {2, 2})).ok());+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {4, 1})).ok());+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {6}), Tensor(a, {2, 2})).ok());+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3, 1}), Tensor(a, {2, 2})).ok());+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(small, {2, 3}), Tensor(a, {2, 2})).ok()); // shape says 6, holds 2+}++// ---------------------------------------------------------------- options++TEST(OnnxOpen, ThreadCountsAreRange_Checked) {+ laya::onnx::Options o;+ o.intra_op_threads = -1;+ EXPECT_FALSE(laya::onnx::Open("/nonexistent/laya.onnx", o).ok());+ o.intra_op_threads = 0;+ o.inter_op_threads = laya::onnx::kMaxThreads + 1;+ const auto r = laya::onnx::Open("/nonexistent/laya.onnx", o);+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("thread counts"), std::string::npos);+}++TEST(OnnxOpen, AMissingGraphIsAnError) {+ const auto r = laya::onnx::Open("/nonexistent/dir/laya.onnx");+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("no such file"), std::string::npos);+}++TEST(OnnxOpen, AGarbageFileIsAnErrorNotAnException) {+ TempDir d;+ const auto r = laya::onnx::Open(d.File("laya.onnx", "this is not a protobuf \x01\x02\xff"));+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();+}++TEST(OnnxOpen, AnEmptyFileIsAnError) {+ TempDir d;+ EXPECT_FALSE(laya::onnx::Open(d.File("laya.onnx", "")).ok());+}++TEST(OnnxOpen, TheSplitLayoutIsRefusedByOpen) {+ TempDir d;+ d.File("encoder.onnx");+ d.File("head.onnx");+ const auto r = laya::onnx::Open(d.str() + "/encoder.onnx");+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("split encoder.onnx + head.onnx layout"), std::string::npos) << r.error();+}++TEST(OnnxRuntime, TheLoadedLibraryIsThePinnedVersion) { EXPECT_EQ(laya::onnx::RuntimeVersion(), LAYA_ORT_VERSION); }++// ---------------------------------------------------------------- real graphs++struct Rig {+ const char* checkpoint; // directory name under LAYA_CHECKPOINTS+ const char* graph_env; // names the exported laya.onnx+ const char* golden; // predict golden recorded from that graph+};+const Rig kRigs[] = {{"multilingual", "LAYA_ONNX_GRAPH", "predict.json"},+ {"english", "LAYA_ONNX_GRAPH_ENGLISH", "predict-english.json"}};++// "" and a recorded failure (or skip) when the rig is not there.+std::string Need(const char* var) {+ const char* v = std::getenv(var);+ if (v != nullptr && *v != 0) return v;+ if (laya_test::RequireGoldens()) ADD_FAILURE() << var << " is unset but LAYA_REQUIRE_GOLDENS=1";+ return "";+}++#define NEED_RIG(rig, graph, dir) \+ const std::string graph = Need(rig.graph_env); \+ const std::string root = Need("LAYA_CHECKPOINTS"); \+ if (graph.empty() || root.empty()) { \+ if (!HasFailure()) GTEST_SKIP() << "no ONNX rig (local run)"; \+ return; \+ } \+ const std::string dir = root + "/" + rig.checkpoint++std::mutex g_agents_mu;+std::map<std::string, std::unique_ptr<laya::agent::Agent>> g_agents;++// The agents hold ONNX Runtime, which must be released before static destruction begins (its own+// statics may already be gone by then), so the test environment closes them.+class CloseAgents : public ::testing::Environment {+ public:+ void TearDown() override {+ std::lock_guard<std::mutex> lock(g_agents_mu);+ for (auto& kv : g_agents) EXPECT_EQ(kv.second->Close(), "");+ g_agents.clear();+ }+};+const ::testing::Environment* const g_close_agents = ::testing::AddGlobalTestEnvironment(new CloseAgents);++// One agent per checkpoint for the whole process; the tests only read it.+laya::agent::Agent* Shared(const Rig& rig, const std::string& graph, const std::string& dir) {+ auto& agents = g_agents;+ std::lock_guard<std::mutex> lock(g_agents_mu);+ auto it = agents.find(rig.checkpoint);+ if (it != agents.end()) return it->second.get();+ auto a = laya::agent::OpenOnnx(dir, graph);+ if (!a.ok()) {+ ADD_FAILURE() << "open " << rig.checkpoint << ": " << a.error();+ return nullptr;+ }+ return (agents[rig.checkpoint] = std::move(a).value()).get();+}++constexpr double kTolerance = 2e-4; // probabilities, confidences, act_probability, noul+constexpr double kScoreTolerance = 8e-4; // score, the expectation over the probabilities+constexpr double kDecisiveGap = 1e-3; // the chosen label is compared when the top two differ by more+constexpr double kMaxDiffering = 0.05; // of all compared numbers may differ at all++struct Tally {+ int numbers = 0, differing = 0, labels = 0, ties = 0, label_diffs = 0, failures = 0;+ double worst = 0;+};++void Compare(const std::string& path, const std::string& key, const Value& got, const Value& want, Tally& t) {+ if (want.kind() == Kind::Object) {+ if (got.kind() != Kind::Object || got.size() != want.size()) {+ t.failures++;+ ADD_FAILURE() << path << ": object shape differs";+ return;+ }+ for (size_t i = 0; i < want.size(); i++) {+ const Value* g = got.Find(want.key(i));+ if (g == nullptr || got.key(i) != want.key(i)) {+ t.failures++;+ ADD_FAILURE() << path << ": key " << want.key(i) << " missing or out of order";+ continue;+ }+ Compare(path + "." + want.key(i), want.key(i), *g, want.value(i), t);+ }+ } else if (want.kind() == Kind::Array) {+ if (got.kind() != Kind::Array || got.size() != want.items().size()) {+ t.failures++;+ ADD_FAILURE() << path << ": array shape differs";+ return;+ }+ for (size_t i = 0; i < want.items().size(); i++) Compare(path + "[]", key, got.items()[i], want.items()[i], t);+ } else if (want.kind() == Kind::Int || want.kind() == Kind::Float) {+ if (got.kind() != Kind::Int && got.kind() != Kind::Float) {+ t.failures++;+ ADD_FAILURE() << path << ": not a number";+ return;+ }+ const double g = laya_test::Num(got), w = laya_test::Num(want), d = std::fabs(g - w);+ t.numbers++;+ if (d > 0) t.differing++;+ t.worst = std::max(t.worst, d);+ if (d > (key == "score" ? kScoreTolerance : kTolerance)) {+ t.failures++;+ ADD_FAILURE() << path << ": got " << g << " want " << w << " (off by " << d << ")";+ }+ } else if (got.kind() != want.kind() || laya::pyvalue::Repr(got) != laya::pyvalue::Repr(want)) {+ t.failures++;+ ADD_FAILURE() << path << ": " << laya::pyvalue::Repr(got) << " want " << laya::pyvalue::Repr(want);+ }+}++Value Without(const Value& obj, const std::string& drop) {+ Value out = Value::Object();+ for (size_t i = 0; i < obj.size(); i++) {+ if (obj.key(i) != drop) out.Set(obj.key(i), obj.value(i));+ }+ return out;+}++void CompareAnswer(const std::string& path, const Value& got, const Value& want, Tally& t) {+ const Value* type = want.Find("type");+ if (type != nullptr && type->as_string() == "choice") {+ std::vector<double> ps;+ for (const Value& p : want.Find("probabilities")->items()) ps.push_back(laya_test::Num(p));+ std::sort(ps.rbegin(), ps.rend());+ if (ps.size() > 1 && ps[0] - ps[1] <= kDecisiveGap) {+ t.ties++;+ Compare(path, "", Without(got, "choice"), Without(want, "choice"), t);+ return;+ }+ t.labels++;+ const Value* g = got.Find("choice");+ if (g == nullptr || laya::pyvalue::Repr(*g) != laya::pyvalue::Repr(*want.Find("choice"))) t.label_diffs++;+ }+ Compare(path, "", got, want, t);+}++void ComparePrediction(const std::string& path, const laya::schema::Prediction& p, const Value& want, Tally& t) {+ auto json = p.ToJson();+ ASSERT_TRUE(json.ok()) << json.error();+ auto parsed = laya::pyvalue::Parse(json.value());+ ASSERT_TRUE(parsed.ok()) << parsed.error();+ const Value& got = parsed.value();+ Compare(path + ".model", "model", *got.Find("model"), *want.Find("model"), t);+ Compare(path + ".usage", "usage", *got.Find("usage"), *want.Find("usage"), t);+ const Value& ga = *got.Find("answers");+ const Value& wa = *want.Find("answers");+ ASSERT_EQ(ga.size(), wa.size()) << path;+ for (size_t i = 0; i < wa.size(); i++) {+ ASSERT_EQ(ga.key(i), wa.key(i)) << path;+ CompareAnswer(path + "." + wa.key(i), ga.value(i), wa.value(i), t);+ }+}++laya::schema::Questions QuestionsOf(const Value& v) {+ auto text = laya::pyvalue::Dumps(v);+ EXPECT_TRUE(text.ok());+ auto qs = laya::schema::Questions::Parse(text.value());+ EXPECT_TRUE(qs.ok()) << qs.error();+ return qs.ok() ? qs.value() : laya::schema::Questions::Of({}).value();+}++void Finish(const char* name, const Tally& t, int min_numbers) {+ std::printf("[ tally ] %s: %d numbers, %d differ (worst %.2e), %d decisive labels (%d differ), %d near-ties\n", name, t.numbers,+ t.differing, t.worst, t.labels, t.label_diffs, t.ties);+ EXPECT_GE(t.numbers, min_numbers) << "too few numbers were compared";+ EXPECT_LE(double(t.differing), kMaxDiffering * t.numbers) << "more of the numbers differ at all than Python's CPU noise explains";+ EXPECT_EQ(t.label_diffs, 0) << "decisive labels differ from Python";+}++class OnnxRig : public ::testing::TestWithParam<int> {};+INSTANTIATE_TEST_SUITE_P(Checkpoints, OnnxRig, ::testing::Values(0, 1), [](const auto& i) { return std::string(kRigs[i.param].checkpoint); });++TEST_P(OnnxRig, SinglePredictionsMatchPython) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");+ LAYA_EXPECT_ROWS(singles, 6);+ laya::agent::Agent* a = Shared(rig, graph, dir);+ ASSERT_NE(a, nullptr);+ Tally t;+ for (size_t i = 0; i < singles->size(); i++) {+ const Value& row = singles->value(i);+ const std::string name = singles->key(i);+ auto pred = a->Predict(*row.Find("state"), QuestionsOf(*row.Find("questions")));+ ASSERT_TRUE(pred.ok()) << name << ": " << pred.error();+ ComparePrediction(name, pred.value(), row, t);+ }+ Finish("single", t, 60);+}++TEST_P(OnnxRig, BatchPredictionsMatchPythonInInputOrder) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ LAYA_GOLDEN_LIST(batches, rig.golden, "batch");+ LAYA_EXPECT_ROWS(batches, 3);+ laya::agent::Agent* a = Shared(rig, graph, dir);+ ASSERT_NE(a, nullptr);+ Tally t;+ for (const Value& rec : batches->items()) {+ laya::agent::CallOptions opts;+ if (!rec.Find("batch_size")->is_null()) opts.batch_size = rec.Find("batch_size")->as_int();+ opts.sort_by_length = rec.Find("sort_by_length")->kind() == Kind::Bool && rec.Find("sort_by_length")->as_bool();+ const std::vector<Value>& states = rec.Find("states")->items();+ auto preds = a->PredictBatch(states, QuestionsOf(*rec.Find("questions")), opts);+ ASSERT_TRUE(preds.ok()) << preds.error();+ const std::vector<Value>& want = rec.Find("results")->items();+ ASSERT_EQ(preds.value().size(), want.size());+ ASSERT_EQ(want.size(), 9u);+ for (size_t i = 0; i < want.size(); i++) ComparePrediction("batch/state" + std::to_string(i), preds.value()[i], want[i], t);+ }+ Finish("batch", t, 240);+}++TEST_P(OnnxRig, BatchOrderFollowsTheInputWhateverTheSortAndBatchSize) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ LAYA_GOLDEN_LIST(batches, rig.golden, "batch");+ laya::agent::Agent* a = Shared(rig, graph, dir);+ ASSERT_NE(a, nullptr);+ const Value& rec = batches->items()[0];+ std::vector<Value> states = rec.Find("states")->items();+ const auto qs = QuestionsOf(*rec.Find("questions"));+ auto fwd = a->PredictBatch(states, qs);+ ASSERT_TRUE(fwd.ok()) << fwd.error();+ std::vector<Value> rev(states.rbegin(), states.rend());+ laya::agent::CallOptions o;+ o.batch_size = 2;+ o.sort_by_length = true;+ auto back = a->PredictBatch(rev, qs, o);+ ASSERT_TRUE(back.ok()) << back.error();+ ASSERT_EQ(fwd.value().size(), back.value().size());+ const size_t n = states.size();+ for (size_t i = 0; i < n; i++) {+ auto x = fwd.value()[i].ToJson(), y = back.value()[n - 1 - i].ToJson();+ ASSERT_TRUE(x.ok() && y.ok());+ auto px = laya::pyvalue::Parse(x.value()), py = laya::pyvalue::Parse(y.value());+ Tally t;+ Compare("state" + std::to_string(i), "", py.value(), px.value(), t); // within the numeric tolerances+ EXPECT_EQ(t.failures, 0) << "state " << i << " changed with its position in the batch";+ }+}++TEST_P(OnnxRig, ConcurrentPredictAgreesWithASerialCall) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");+ laya::agent::Agent* a = Shared(rig, graph, dir);+ ASSERT_NE(a, nullptr);+ const size_t n = singles->size();+ std::vector<laya::schema::Questions> qs;+ std::vector<std::string> serial;+ for (size_t i = 0; i < n; i++) {+ qs.push_back(QuestionsOf(*singles->value(i).Find("questions")));+ auto p = a->Predict(*singles->value(i).Find("state"), qs.back());+ ASSERT_TRUE(p.ok()) << p.error();+ serial.push_back(p.value().ToJson().value());+ }+ std::atomic<int> bad{0};+ std::vector<std::thread> threads;+ for (int th = 0; th < 8; th++) {+ threads.emplace_back([&, th] {+ for (int round = 0; round < 3; round++) {+ for (size_t k = 0; k < n; k++) {+ const size_t i = (k + size_t(th)) % n;+ auto p = a->Predict(*singles->value(i).Find("state"), qs[i]);+ if (!p.ok() || p.value().ToJson().value() != serial[i]) bad++;+ }+ }+ });+ }+ for (auto& t : threads) t.join();+ EXPECT_EQ(bad.load(), 0);+}++TEST_P(OnnxRig, ACancelledCallStopsBeforeTheRuntime) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ LAYA_GOLDEN_LIST(singles, rig.golden, "single");+ laya::agent::Agent* a = Shared(rig, graph, dir);+ ASSERT_NE(a, nullptr);+ laya::agent::CallOptions o;+ o.cancel.Cancel();+ auto p = a->Predict(*singles->value(0).Find("state"), QuestionsOf(*singles->value(0).Find("questions")), o);+ ASSERT_FALSE(p.ok());+ EXPECT_TRUE(laya::agent::IsCancelled(p.error())) << p.error();+}++TEST_P(OnnxRig, SessionRunIsGuardedAgainstHostileBatches) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ laya::infer::Session& session = *s.value();+ const laya::CancelToken live;++ auto ok = session.Run(Tiny(2, 3), live);+ ASSERT_TRUE(ok.ok()) << ok.error();+ EXPECT_EQ(ok.value().logits.size(), 6u);+ EXPECT_EQ(ok.value().act_logits.size(), 4u);++ Batch wrong_rank = Tiny(2, 3);+ wrong_rank.input_ids.pop_back(); // a flattened slice that no longer matches rows*length+ EXPECT_FALSE(session.Run(wrong_rank, live).ok());++ Batch huge = Tiny(1, 3);+ huge.rows = int64_t(1) << 40; // far past the cap: refused before any allocation+ EXPECT_FALSE(session.Run(huge, live).ok());+ huge = Tiny(1, 3);+ huge.length = int64_t(1) << 40;+ EXPECT_FALSE(session.Run(huge, live).ok());+ Batch neg = Tiny(1, 3);+ neg.length = -1;+ EXPECT_FALSE(session.Run(neg, live).ok());++ Batch one_marker = Tiny(1, 1); // the exported head takes topk(2)+ EXPECT_FALSE(session.Run(one_marker, live).ok());+ Batch oob = Tiny(1, 3);+ oob.marker_pos[0] = 99;+ EXPECT_FALSE(session.Run(oob, live).ok());+ Batch mask2 = Tiny(1, 3);+ mask2.marker_mask[1] = 2; // not a bool+ EXPECT_FALSE(session.Run(mask2, live).ok());+ Batch bad_q = Tiny(1, 3);+ bad_q.qtype[0] = 7;+ EXPECT_FALSE(session.Run(bad_q, live).ok());+ Batch empty;+ EXPECT_FALSE(session.Run(empty, live).ok());++ laya::CancelToken fired;+ fired.Cancel();+ auto c = session.Run(Tiny(1, 3), fired);+ ASSERT_FALSE(c.ok());+ EXPECT_TRUE(laya::infer::IsCancelled(c.error()));++ // The session still works after every refusal.+ EXPECT_TRUE(session.Run(Tiny(1, 3), live).ok());+}++TEST_P(OnnxRig, CloseIsIdempotentAndRunAfterCloseIsAnError) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ EXPECT_EQ(s.value()->Close(), "");+ EXPECT_EQ(s.value()->Close(), "");+ auto r = s.value()->Run(Tiny(1, 3), laya::CancelToken());+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("closed"), std::string::npos) << r.error();+}++TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto first = laya::onnx::Open(graph);+ ASSERT_TRUE(first.ok()) << first.error();+ std::atomic<int> bad{0};+ std::thread worker([&] {+ for (int i = 0; i < 20; i++) {+ if (!first.value()->Run(Tiny(2, 3), laya::CancelToken()).ok()) bad++;+ }+ });+ for (int i = 0; i < 3; i++) {+ laya::onnx::Options o;+ o.intra_op_threads = 1;+ o.inter_op_threads = 1;+ auto other = laya::onnx::Open(graph, o); // a second session, dropped without Close+ if (!other.ok() || !other.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()) bad++;+ }+ worker.join();+ EXPECT_EQ(bad.load(), 0);+}++TEST_P(OnnxRig, OpenOnnxWithAWrongCheckpointDirIsAnError) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ TempDir empty;+ const auto a = laya::agent::OpenOnnx(empty.str(), graph);+ EXPECT_FALSE(a.ok());+}++TEST_P(OnnxRig, TheGraphWithoutItsDataFileIsAnError) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ TempDir d;+ fs::copy_file(graph, d.str() + "/laya.onnx"); // the .data file stays behind+ const auto r = laya::onnx::Open(d.str() + "/laya.onnx");+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();+}++} // namespace