Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cpp): onnx backend review round 1
Adeshina Lasisi6441cda7d66b
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex 9737d665..6e24ab7c 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -44,7 +44,7 @@ env:# compile_flags). Raise it with every test added.MIN_TESTS: "423"# The onnx job registers the same tests plus the laya_onnx_tests cases.- MIN_TESTS_ONNX: "466"+ MIN_TESTS_ONNX: "479"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints@@ -122,11 +122,12 @@ jobs:if: ${{ !cancelled() }}run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest.xml" --min-tests "$MIN_TESTS"- # The ONNX Runtime backend, mirroring go.yml's `onnx` job: the same graph cache keys as java.yml- # (so a graph exported there is reused), real graphs for both shipped checkpoints, goldens+ # The ONNX Runtime backend, mirroring go.yml's `onnx` job: real graphs for both shipped checkpoints, goldens# recorded by Python's ONNXAgent on those graphs, and a floor on the tests that must run. The# runtime comes from the release archive pinned by sha256 in laya-cpp/cmake/onnxruntime.cmake:- # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`.+ # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`,+ # and like go.yml's `onnx` job, which is also `continue-on-error`: the MIN_TESTS_ONNX floor blocks+ # nothing until this job and Go's are made blocking together.onnx:name: onnx (${{ matrix.compiler }}, ${{ matrix.preset }})runs-on: ubuntu-latest@@ -160,19 +161,22 @@ jobs:pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpupip install -r laya-dotnet/tools/requirements-regen.txt- # Keys are byte-identical to the predict cells in java.yml, so both share the entries.+ # One cache per checkpoint, at its own path, under cpp-specific keys. java.yml caches only its+ # multilingual graph, as a whole .work/onnx directory; sharing that entry would let this job+ # save the english graph into it, and the english graph has no java entry to share. The hash+ # inputs are java.yml's, so an exporter change invalidates both.- name: Cache the multilingual graphid: graph-multilingualuses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0with:- path: laya-java/.work/onnx- key: laya-java-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}+ path: laya-java/.work/onnx/multilingual+ key: laya-cpp-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}- name: Cache the english graphid: graph-englishuses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0with:- path: laya-java/.work/onnx- key: laya-java-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}+ path: laya-java/.work/onnx/english+ key: laya-cpp-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}- name: Prepare both checkpoints and their graphsenv:HF_TOKEN: ${{ secrets.HF_TOKEN }}diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex fe87d4b6..ed5999f3 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -73,7 +73,17 @@ if(LAYA_BUILD_TESTS)target_link_libraries(laya_onnx_tests PRIVATE laya_onnx laya_ort GTest::gtest_main)laya_target_flags(laya_onnx_tests)laya_ort_runtime_path(laya_onnx_tests)- target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}")+ add_executable(laya_onnx_static_holder tests/tools/onnx_static_holder.cpp)+ target_link_libraries(laya_onnx_static_holder PRIVATE laya_onnx laya_ort)+ laya_target_flags(laya_onnx_static_holder)+ laya_ort_runtime_path(laya_onnx_static_holder)+ add_dependencies(laya_onnx_tests laya_onnx_static_holder)+ add_executable(laya_c5_cli tests/tools/laya_c5_cli.cpp)+ target_link_libraries(laya_c5_cli PRIVATE laya_onnx laya_ort)+ laya_target_flags(laya_c5_cli)+ laya_ort_runtime_path(laya_c5_cli)+ target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}"+ LAYA_STATIC_HOLDER="$<TARGET_FILE:laya_onnx_static_holder>")endif()include(GoogleTest)@@ -82,6 +92,8 @@ if(LAYA_BUILD_TESTS)if(LAYA_ONNX)gtest_discover_tests(laya_onnx_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)+ # The refusals of cmake/onnxruntime.cmake (wrong version, missing header or library).+ add_test(NAME ort_root_check COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/check_ort_root.sh ${LAYA_ORT_ROOT})endif()# Listing must stay cheap (it runs before every ctest): bound it at 3 s with the goldens loaded.diff --git a/laya-cpp/cmake/onnxruntime.cmake b/laya-cpp/cmake/onnxruntime.cmakeindex 6d8178eb..3195cfa0 100644--- a/laya-cpp/cmake/onnxruntime.cmake+++ b/laya-cpp/cmake/onnxruntime.cmake@@ -29,10 +29,27 @@ endif()if(NOT EXISTS "${_ort_root}/include/onnxruntime_cxx_api.h")message(FATAL_ERROR "${_ort_root}/include/onnxruntime_cxx_api.h not found: not an ONNX Runtime release directory")endif()+# The pinned release is what the tests and goldens were measured against, so an override must be it.+string(REGEX MATCH "^[0-9]+\\.([0-9]+)\\." _ort_unused "${LAYA_ORT_VERSION}")+set(_ort_want_api "${CMAKE_MATCH_1}")+file(STRINGS "${_ort_root}/include/onnxruntime_c_api.h" _ort_api_line REGEX "^#define ORT_API_VERSION [0-9]+")+string(REGEX REPLACE "^#define ORT_API_VERSION ([0-9]+).*" "\\1" _ort_api "${_ort_api_line}")+if(NOT _ort_api STREQUAL _ort_want_api)+ message(FATAL_ERROR "${_ort_root} is ONNX Runtime API version '${_ort_api}', but laya_onnx is pinned to ${LAYA_ORT_VERSION} (API version ${_ort_want_api})")+endif()+if(EXISTS "${_ort_root}/VERSION_NUMBER")+ file(STRINGS "${_ort_root}/VERSION_NUMBER" _ort_vn LIMIT_COUNT 1)+ if(NOT _ort_vn STREQUAL LAYA_ORT_VERSION)+ message(FATAL_ERROR "${_ort_root}/VERSION_NUMBER says '${_ort_vn}', but laya_onnx is pinned to ${LAYA_ORT_VERSION}")+ endif()+endif()if(APPLE)set(_ort_lib "${_ort_root}/lib/libonnxruntime.${LAYA_ORT_VERSION}.dylib")if(NOT EXISTS "${_ort_lib}")file(GLOB _ort_lib "${_ort_root}/lib/libonnxruntime*.dylib")+ if(NOT _ort_lib)+ message(FATAL_ERROR "no libonnxruntime*.dylib under ${_ort_root}/lib")+ endif()list(GET _ort_lib 0 _ort_lib)endif()elseif(WIN32)@@ -48,6 +65,7 @@ message(STATUS "ONNX Runtime ${LAYA_ORT_VERSION}: ${_ort_lib}")add_library(laya_ort SHARED IMPORTED)set_target_properties(laya_ort PROPERTIES IMPORTED_LOCATION "${_ort_lib}" INTERFACE_INCLUDE_DIRECTORIES "${_ort_root}/include")set(LAYA_ORT_LIBDIR "${_ort_root}/lib")+set(LAYA_ORT_ROOT "${_ort_root}")# Build-tree executables find the shared library through an rpath to its directory.function(laya_ort_runtime_path target)diff --git a/laya-cpp/include/laya/onnx.hpp b/laya-cpp/include/laya/onnx.hppindex f391721b..64987ab1 100644--- a/laya-cpp/include/laya/onnx.hpp+++ b/laya-cpp/include/laya/onnx.hpp@@ -12,8 +12,10 @@ namespace laya {namespace onnx {struct Options {- int intra_op_threads = 0; // 0 leaves ONNX Runtime's default; at most kMaxThreads- int inter_op_threads = 0;+ // 0 leaves ONNX Runtime's default (one intra-op pool sized to the cores). Every session owns its+ // own pool and sessions do not share one, so many sessions in one process should set this. There+ // is no inter-op option: the graph runs sequentially, where it would have no effect.+ int intra_op_threads = 0; // at most kMaxThreads};constexpr int kMaxThreads = 4096;@@ -21,8 +23,10 @@ constexpr int kMaxThreads = 4096;std::string RuntimeVersion();// A Session over one fused laya.onnx (+ laya.onnx.data beside it). Checks the graph's inputs and-// outputs at once. Run is thread-safe; Close is idempotent and waits for runs in flight. No-// exception leaves this API.+// outputs at once. Run is thread-safe and stops in flight when its CancelToken fires (kCancelled).+// Close is idempotent, refuses new runs, terminates the runs in flight and returns once they have+// left. A Session or Agent may be held in a static: the process-wide ONNX Runtime environment is+// never destroyed. No exception leaves this API.Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options = {});} // namespace onnxdiff --git a/laya-cpp/scripts/check_ort_root.sh b/laya-cpp/scripts/check_ort_root.shnew file mode 100755index 00000000..1c4dc122--- /dev/null+++ b/laya-cpp/scripts/check_ort_root.sh@@ -0,0 +1,31 @@+#!/bin/sh+# Configure-only check of cmake/onnxruntime.cmake's refusals. usage: check_ort_root.sh <good-ort-root>+# A root with the wrong API version, a wrong VERSION_NUMBER, no header, or no library must each+# fail at configure time with the named message, and the good root must configure.+set -eu+good=$1+src=$(cd "$(dirname "$0")/.." && pwd)+tmp=$(mktemp -d)+trap 'rm -rf "$tmp"' EXIT+fail=0+try() { # name root want-substring ("" = must succeed)+ if out=$(cmake -S "$src" -B "$tmp/b-$1" -DLAYA_ONNX=ON -DLAYA_ONNXRUNTIME_ROOT="$2" 2>&1); then rc=0; else rc=1; fi+ if [ -z "$3" ]; then+ [ $rc -eq 0 ] || { echo "FAIL $1: good root refused"; fail=1; }+ else+ [ $rc -ne 0 ] || { echo "FAIL $1: accepted"; fail=1; return; }+ echo "$out" | tr -s ' \n' ' ' | grep -q "$3" || { echo "FAIL $1: wrong message"; echo "$out" | tail -4; fail=1; }+ fi+}+copy() { mkdir -p "$tmp/$1"; cp -R "$good/include" "$tmp/$1/include"; mkdir -p "$tmp/$1/lib"; cp -R "$good/lib/." "$tmp/$1/lib/"; cp "$good/VERSION_NUMBER" "$tmp/$1/" 2>/dev/null || true; }+try good "$good" ""+copy api; sed -i.bak 's/^#define ORT_API_VERSION [0-9]*/#define ORT_API_VERSION 99/' "$tmp/api/include/onnxruntime_c_api.h"+try wrong-api "$tmp/api" "API version '99'"+copy vn; echo 1.29.0 > "$tmp/vn/VERSION_NUMBER"+try wrong-version-number "$tmp/vn" "VERSION_NUMBER says"+copy nohdr; rm "$tmp/nohdr/include/onnxruntime_cxx_api.h"+try no-header "$tmp/nohdr" "not an ONNX Runtime release directory"+copy nolib; find "$tmp/nolib/lib" -maxdepth 1 -type f -name 'libonnxruntime*' -delete; find "$tmp/nolib/lib" -maxdepth 1 -type l -name 'libonnxruntime*' -delete+try no-library "$tmp/nolib" "library not found\|no libonnxruntime"+[ $fail -eq 0 ] && echo "check_ort_root: 5 cases ok"+exit $faildiff --git a/laya-cpp/src/onnx.cpp b/laya-cpp/src/onnx.cppindex 878e6f07..94d6b23f 100644--- a/laya-cpp/src/onnx.cpp+++ b/laya-cpp/src/onnx.cpp@@ -3,9 +3,13 @@#include <onnxruntime_cxx_api.h>#include <filesystem>+#include <atomic>+#include <chrono>+#include <condition_variable>#include <memory>#include <mutex>-#include <shared_mutex>+#include <set>+#include <thread>#include "laya/onnx_graph.hpp"@@ -13,18 +17,13 @@ namespace laya {namespace onnx {namespace {-// One Ort::Env per process, shared by every session and freed with the last one, so there is no-// static destructor racing a session's teardown. The mutex orders creation against that release.-std::shared_ptr<Ort::Env> SharedEnv() {- static std::mutex mu;- static std::weak_ptr<Ort::Env> weak;- std::lock_guard<std::mutex> lock(mu);- std::shared_ptr<Ort::Env> env = weak.lock();- if (!env) {- env = std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_ERROR, "laya");- weak = env;- }- return env;+// One Ort::Env per process, leaked on purpose. A session or agent may be held in a static whose+// destructor runs after any Env destructor would have, and ORT's own statics may already be gone;+// never destroying the Env means no destruction order can fault. The logging level is FATAL so a+// refused run is reported only through its Result.+Ort::Env& SharedEnv() {+ static Ort::Env* const env = new Ort::Env(ORT_LOGGING_LEVEL_FATAL, "laya");+ return *env;}std::string Dtype(ONNXTensorElementDataType t) {@@ -40,9 +39,45 @@ std::string Dtype(ONNXTensorElementDataType t) {const char* const kInputNames[5] = {"input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"};const char* const kOutputNames[2] = {"logits", "act_logits"};+// Fires SetTerminate on a run when the cancel token fires. The token is a polled flag, so the+// watcher waits on a condition variable with a short timeout; the run ending wakes it at once.+class Watcher {+ public:+ Watcher(const CancelToken& cancel, Ort::RunOptions& ro) : cancel_(cancel), ro_(ro), thread_([this] { Loop(); }) {}+ ~Watcher() {+ {+ std::lock_guard<std::mutex> lock(mu_);+ done_ = true;+ }+ cv_.notify_all();+ thread_.join();+ }+ bool terminated() const { return terminated_.load(); }++ private:+ void Loop() {+ std::unique_lock<std::mutex> lock(mu_);+ while (!done_) {+ if (cancel_.cancelled()) {+ terminated_ = true;+ ro_.SetTerminate();+ return;+ }+ cv_.wait_for(lock, std::chrono::milliseconds(2));+ }+ }+ CancelToken cancel_;+ Ort::RunOptions& ro_;+ std::mutex mu_;+ std::condition_variable cv_;+ bool done_ = false;+ std::atomic<bool> terminated_{false};+ std::thread thread_; // last: started after the rest is built+};+class OrtSession final : public infer::Session {public:- OrtSession(std::shared_ptr<Ort::Env> env, std::unique_ptr<Ort::Session> s) : env_(std::move(env)), session_(std::move(s)) {}+ explicit OrtSession(std::unique_ptr<Ort::Session> s) : session_(std::move(s)) {}~OrtSession() override { Close(); }Result<infer::Output> Run(const infer::Batch& b, const CancelToken& cancel) override {@@ -57,9 +92,13 @@ class OrtSession final : public infer::Session {}}+ // Refuses new runs, terminates the runs in flight, and releases the session once they have left.std::string Close() override {try {- std::unique_lock<std::shared_mutex> lock(mu_); // waits for runs in flight+ std::unique_lock<std::mutex> lock(mu_);+ closing_ = true;+ for (Ort::RunOptions* r : flights_) r->SetTerminate();+ idle_.wait(lock, [this] { return flights_.empty(); });session_.reset();return "";} catch (...) {@@ -68,6 +107,29 @@ class OrtSession final : public infer::Session {}private:+ // Registers a run for Close to terminate; the destructor leaves the list and wakes Close.+ class Flight {+ public:+ Flight(OrtSession& s, Ort::RunOptions& ro) : s_(s), ro_(ro) {+ std::lock_guard<std::mutex> lock(s_.mu_);+ if (s_.closing_ || !s_.session_) return;+ s_.flights_.insert(&ro_);+ ok_ = true;+ }+ ~Flight() {+ if (!ok_) return;+ std::lock_guard<std::mutex> lock(s_.mu_);+ s_.flights_.erase(&ro_);+ s_.idle_.notify_all();+ }+ bool ok() const { return ok_; }++ private:+ OrtSession& s_;+ Ort::RunOptions& ro_;+ bool ok_ = false;+ };+Result<infer::Output> RunImpl(const infer::Batch& b, const CancelToken& cancel) {using R = Result<infer::Output>;if (cancel.cancelled()) return R::Fail(infer::kCancelled);@@ -76,8 +138,9 @@ class OrtSession final : public infer::Session {for (uint8_t m : b.marker_mask) {if (m > 1) return R::Fail("onnx: marker_mask holds a value other than 0 or 1");}- std::shared_lock<std::shared_mutex> lock(mu_);- if (!session_) return R::Fail("onnx: session is closed");+ Ort::RunOptions ro;+ Flight flight(*this, ro);+ if (!flight.ok()) return R::Fail("onnx: session is closed");const Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);const int64_t shape2[2] = {b.rows, b.length};@@ -96,8 +159,14 @@ class OrtSession final : public infer::Session {shape_m, 2, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL));in.push_back(i64(b.qtype, shape1, 1));- Ort::RunOptions ro;- std::vector<Ort::Value> out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);+ Watcher watcher(cancel, ro);+ std::vector<Ort::Value> out;+ try {+ out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);+ } catch (const Ort::Exception& e) {+ if (watcher.terminated() || cancel.cancelled()) return R::Fail(infer::kCancelled);+ throw;+ }if (out.size() != 2) return R::Fail("onnx: run returned " + std::to_string(out.size()) + " outputs, want 2");if (cancel.cancelled()) return R::Fail(infer::kCancelled);@@ -115,8 +184,10 @@ class OrtSession final : public infer::Session {return CollectOutput(b, t[0], t[1]);}- std::shared_ptr<Ort::Env> env_; // keeps the environment alive past session_- std::shared_mutex mu_;+ std::mutex mu_; // guards closing_, flights_ and the release of session_+ std::condition_variable idle_;+ bool closing_ = false;+ std::set<Ort::RunOptions*> flights_;std::unique_ptr<Ort::Session> session_;};@@ -132,19 +203,17 @@ std::string RuntimeVersion() {Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options) {using R = Result<std::unique_ptr<infer::Session>>;- if (options.intra_op_threads < 0 || options.inter_op_threads < 0 || options.intra_op_threads > kMaxThreads ||- options.inter_op_threads > kMaxThreads) {- return R::Fail("onnx: thread counts must be between 0 and " + std::to_string(kMaxThreads));+ if (options.intra_op_threads < 0 || options.intra_op_threads > kMaxThreads) {+ return R::Fail("onnx: intra_op_threads must be between 0 and " + std::to_string(kMaxThreads));}const std::string path_problem = CheckPath(graph_path);if (!path_problem.empty()) return R::Fail("onnx: " + path_problem);try {- std::shared_ptr<Ort::Env> env = SharedEnv();+ Ort::Env& env = SharedEnv();Ort::SessionOptions so;if (options.intra_op_threads > 0) so.SetIntraOpNumThreads(options.intra_op_threads);- if (options.inter_op_threads > 0) so.SetInterOpNumThreads(options.inter_op_threads);const std::filesystem::path p(graph_path);- auto session = std::make_unique<Ort::Session>(*env, p.c_str(), so);+ auto session = std::make_unique<Ort::Session>(env, p.c_str(), so);Ort::AllocatorWithDefaultOptions alloc;auto describe = [&](bool inputs) {@@ -168,7 +237,7 @@ Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, cons};const std::string bad = CheckGraph(describe(true), describe(false));if (!bad.empty()) return R::Fail("onnx: graph " + graph_path + ": " + bad);- return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(env), std::move(session))));+ return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(session))));} catch (const Ort::Exception& e) {return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());} catch (const std::exception& e) {diff --git a/laya-cpp/src/onnx_graph.cpp b/laya-cpp/src/onnx_graph.cppindex 808c0b1f..9a498a7a 100644--- a/laya-cpp/src/onnx_graph.cpp+++ b/laya-cpp/src/onnx_graph.cpp@@ -118,7 +118,7 @@ Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logireturn Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " holds " + std::to_string(c.t.count) + " values");}for (int64_t i = 0; i < c.t.count; i++) {- if (std::isnan(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN");+ if (!std::isfinite(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN or Inf");}}infer::Output o;diff --git a/laya-cpp/tests/onnx_hostile_graphs.inc b/laya-cpp/tests/onnx_hostile_graphs.incnew file mode 100644index 00000000..5a6a61b8--- /dev/null+++ b/laya-cpp/tests/onnx_hostile_graphs.inc@@ -0,0 +1,5 @@+// A graph whose one initializer names external data outside the graph's directory.+const char kDotdotGraph[] = "\x08\x0e\x3a\x66\x0a\x10\x0a\x01\x77\x12\x01\x79\x22\x08\x49\x64\x65\x6e\x74\x69\x74\x79\x12\x01\x67\x2a\x3e\x08\x02\x10\x01\x42\x01\x77\x6a\x19\x0a\x08\x6c\x6f\x63\x61\x74\x69\x6f\x6e\x12\x0d\x2e\x2e\x2f\x73\x65\x63\x72\x65\x74\x2e\x62\x69\x6e\x6a\x0b\x0a\x06\x6f\x66\x66\x73\x65\x74\x12\x01\x30\x6a\x0b\x0a\x06\x6c\x65\x6e\x67\x74\x68\x12\x01\x38\x70\x01\x62\x0f\x0a\x01\x79\x12\x0a\x0a\x08\x08\x01\x12\x04\x0a\x02\x08\x02\x42\x02\x10\x1c";+constexpr size_t kDotdotGraphSize = 110;+const char kAbsGraph[] = "\x08\x0e\x3a\x63\x0a\x10\x0a\x01\x77\x12\x01\x79\x22\x08\x49\x64\x65\x6e\x74\x69\x74\x79\x12\x01\x67\x2a\x3b\x08\x02\x10\x01\x42\x01\x77\x6a\x16\x0a\x08\x6c\x6f\x63\x61\x74\x69\x6f\x6e\x12\x0a\x2f\x65\x74\x63\x2f\x68\x6f\x73\x74\x73\x6a\x0b\x0a\x06\x6f\x66\x66\x73\x65\x74\x12\x01\x30\x6a\x0b\x0a\x06\x6c\x65\x6e\x67\x74\x68\x12\x01\x38\x70\x01\x62\x0f\x0a\x01\x79\x12\x0a\x0a\x08\x08\x01\x12\x04\x0a\x02\x08\x02\x42\x02\x10\x1c";+constexpr size_t kAbsGraphSize = 107;diff --git a/laya-cpp/tests/onnx_test.cpp b/laya-cpp/tests/onnx_test.cppindex 72e33341..5755d7e1 100644--- a/laya-cpp/tests/onnx_test.cpp+++ b/laya-cpp/tests/onnx_test.cpp@@ -4,6 +4,7 @@#include <gtest/gtest.h>#include <atomic>+#include <chrono>#include <cmath>#include <cstdio>#include <filesystem>@@ -13,6 +14,7 @@#include <mutex>#include <thread>+#include <sys/wait.h>#include <unistd.h>#include "golden.hpp"@@ -218,6 +220,20 @@ TEST(OnnxCollect, ANaNIsAnErrorNotAnAnswer) {EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());}+TEST(OnnxCollect, AnInfinityIsAnErrorNotAnAnswer) {+ const Batch b = Tiny(1, 3);+ const float inf = std::numeric_limits<float>::infinity();+ for (float bad : {inf, -inf}) {+ std::vector<float> l = {1, bad, 3}, a = {1, 2};+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));+ ASSERT_FALSE(r.ok());+ EXPECT_NE(r.error().find("Inf"), std::string::npos);+ l[1] = 0;+ a[1] = bad;+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());+ }+}+TEST(OnnxCollect, WrongShapesAreErrors) {const Batch b = Tiny(2, 3);const std::vector<float> l(6, 1.f), a(4, 1.f), small(2, 1.f);@@ -234,11 +250,10 @@ TEST(OnnxOpen, ThreadCountsAreRange_Checked) {laya::onnx::Options o;o.intra_op_threads = -1;EXPECT_FALSE(laya::onnx::Open("/nonexistent/laya.onnx", o).ok());- o.intra_op_threads = 0;- o.inter_op_threads = laya::onnx::kMaxThreads + 1;+ o.intra_op_threads = laya::onnx::kMaxThreads + 1;const auto r = laya::onnx::Open("/nonexistent/laya.onnx", o);ASSERT_FALSE(r.ok());- EXPECT_NE(r.error().find("thread counts"), std::string::npos);+ EXPECT_NE(r.error().find("intra_op_threads"), std::string::npos);}TEST(OnnxOpen, AMissingGraphIsAnError) {@@ -259,6 +274,29 @@ TEST(OnnxOpen, AnEmptyFileIsAnError) {EXPECT_FALSE(laya::onnx::Open(d.File("laya.onnx", "")).ok());}+// ONNX Runtime refuses external data that leaves the graph's directory (`..` or an absolute path);+// this pins that, with a real file sitting where the graph points.+#include "onnx_hostile_graphs.inc"+TEST(OnnxOpen, ExternalDataOutsideTheGraphDirectoryIsRefused) {+ TempDir d;+ fs::create_directories(d.str() + "/sub");+ {+ std::ofstream(d.str() + "/secret.bin", std::ios::binary) << std::string(8, '\0');+ }+ const struct {+ const char* name;+ const char* bytes;+ size_t size;+ } cases[] = {{"dotdot", kDotdotGraph, kDotdotGraphSize}, {"absolute", kAbsGraph, kAbsGraphSize}};+ for (const auto& c : cases) {+ const std::string path = d.str() + "/sub/" + c.name + ".onnx";+ std::ofstream(path, std::ios::binary).write(c.bytes, std::streamsize(c.size));+ const auto r = laya::onnx::Open(path);+ ASSERT_FALSE(r.ok()) << c.name << " opened";+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();+ }+}+TEST(OnnxOpen, TheSplitLayoutIsRefusedByOpen) {TempDir d;d.File("encoder.onnx");@@ -300,18 +338,6 @@ std::string Need(const char* var) {std::mutex g_agents_mu;std::map<std::string, std::unique_ptr<laya::agent::Agent>> g_agents;-// The agents hold ONNX Runtime, which must be released before static destruction begins (its own-// statics may already be gone by then), so the test environment closes them.-class CloseAgents : public ::testing::Environment {- public:- void TearDown() override {- std::lock_guard<std::mutex> lock(g_agents_mu);- for (auto& kv : g_agents) EXPECT_EQ(kv.second->Close(), "");- g_agents.clear();- }-};-const ::testing::Environment* const g_close_agents = ::testing::AddGlobalTestEnvironment(new CloseAgents);-// One agent per checkpoint for the whole process; the tests only read it.laya::agent::Agent* Shared(const Rig& rig, const std::string& graph, const std::string& dir) {auto& agents = g_agents;@@ -597,6 +623,14 @@ TEST_P(OnnxRig, SessionRunIsGuardedAgainstHostileBatches) {Batch empty;EXPECT_FALSE(session.Run(empty, live).ok());+ // Passes every check of ours and fails inside ONNX Runtime (a token id past the vocabulary): the+ // failure must come back as a Result, not an exception.+ Batch past_vocab = Tiny(1, 3);+ past_vocab.input_ids[0] = 1000000000;+ const auto pv = session.Run(past_vocab, live);+ ASSERT_FALSE(pv.ok());+ EXPECT_EQ(pv.error().rfind("onnx: run:", 0), 0u) << pv.error();+laya::CancelToken fired;fired.Cancel();auto c = session.Run(Tiny(1, 3), fired);@@ -619,6 +653,136 @@ TEST_P(OnnxRig, CloseIsIdempotentAndRunAfterCloseIsAnError) {EXPECT_NE(r.error().find("closed"), std::string::npos) << r.error();}+// A batch that keeps the runtime busy for seconds.+Batch Long(int64_t rows, int64_t length) {+ Batch b = Tiny(rows, 3);+ b.length = length;+ b.input_ids.assign(size_t(rows * length), 5);+ b.attention_mask.assign(size_t(rows * length), 1);+ return b;+}+using Clock = std::chrono::steady_clock;+double Since(Clock::time_point t) { return std::chrono::duration<double>(Clock::now() - t).count(); }+constexpr int64_t kLongRows = 16, kLongLength = 192;++// Seconds one uncancelled long run takes; the tests below only mean something if it is long.+double LongRunSeconds(laya::infer::Session& s) {+ const auto t = Clock::now();+ const auto r = s.Run(Long(kLongRows, kLongLength), laya::CancelToken());+ EXPECT_TRUE(r.ok()) << (r.ok() ? "" : r.error());+ return Since(t);+}++TEST_P(OnnxRig, ACancelDuringARunStopsTheRun) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ const double full = LongRunSeconds(*s.value());+ ASSERT_GT(full, 0.8) << "the long batch is too short to tell a mid-run cancel from a late one";+ laya::CancelToken token;+ std::thread canceller([&] {+ std::this_thread::sleep_for(std::chrono::milliseconds(50));+ token.Cancel();+ });+ const auto t = Clock::now();+ const auto r = s.value()->Run(Long(kLongRows, kLongLength), token);+ const double took = Since(t);+ canceller.join();+ ASSERT_FALSE(r.ok());+ EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();+ EXPECT_LT(took, full / 3) << "cancelled run took " << took << " s of " << full << " s";+ EXPECT_TRUE(s.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()); // the session is still usable+}++TEST_P(OnnxRig, ACancelThatLandsAfterTheRunStillReportsCancelled) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ // The token fires inside the post-run window: the run is tiny, so a watcher cannot beat it+ // reliably; either outcome is allowed only if it is cancelled or a complete correct result.+ for (int i = 0; i < 50; i++) {+ laya::CancelToken token;+ std::thread canceller([&] { token.Cancel(); });+ const auto r = s.value()->Run(Tiny(2, 3), token);+ canceller.join();+ if (!r.ok()) EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();+ else EXPECT_EQ(r.value().logits.size(), 6u);+ }+}++TEST_P(OnnxRig, CloseInterruptsARunInFlight) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ const double full = LongRunSeconds(*s.value());+ ASSERT_GT(full, 0.8);+ std::atomic<bool> failed{false};+ std::thread runner([&] { failed = !s.value()->Run(Long(kLongRows, kLongLength), laya::CancelToken()).ok(); });+ std::this_thread::sleep_for(std::chrono::milliseconds(100));+ const auto t = Clock::now();+ EXPECT_EQ(s.value()->Close(), "");+ const double took = Since(t);+ runner.join();+ EXPECT_TRUE(failed.load());+ EXPECT_LT(took, full / 3) << "Close took " << took << " s of a " << full << " s run";+}++TEST_P(OnnxRig, CloseReturnsPromptlyUnderSustainedRunTraffic) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ auto s = laya::onnx::Open(graph);+ ASSERT_TRUE(s.ok()) << s.error();+ const auto t0 = Clock::now();+ ASSERT_TRUE(s.value()->Run(Long(4, 128), laya::CancelToken()).ok());+ const double one = Since(t0); // one such run alone+ ASSERT_GT(one, 0.15);+ std::atomic<bool> stop{false};+ std::atomic<int> ran{0}, refused{0}, other{0};+ std::vector<std::thread> threads;+ for (int i = 0; i < 4; i++) {+ threads.emplace_back([&] {+ while (!stop.load()) {+ const auto r = s.value()->Run(Long(4, 128), laya::CancelToken());+ if (r.ok()) ran++;+ else if (r.error().find("closed") != std::string::npos || r.error().find("run:") != std::string::npos) refused++;+ else other++;+ }+ });+ }+ std::this_thread::sleep_for(std::chrono::milliseconds(300));+ const auto t = Clock::now();+ EXPECT_EQ(s.value()->Close(), "");+ const double took = Since(t);+ stop = true;+ for (auto& th : threads) th.join();+ EXPECT_GT(ran.load() + refused.load(), 0); // every thread ended on a Result, none on a hang+ EXPECT_EQ(other.load(), 0);+ EXPECT_LT(took, one / 2) << "Close took " << took << " s with one run alone taking " << one << " s";+}++// A user program may hold a Session and an Agent in statics; it must exit cleanly, closed or not.+TEST_P(OnnxRig, AStaticSessionAndAgentExitCleanly) {+ const Rig& rig = kRigs[GetParam()];+ NEED_RIG(rig, graph, dir);+ for (const char* mode : {"", "close"}) {+ const pid_t pid = fork();+ ASSERT_GE(pid, 0);+ if (pid == 0) {+ const char* argv[] = {LAYA_STATIC_HOLDER, graph.c_str(), dir.c_str(), mode, nullptr};+ if (*mode == 0) argv[3] = nullptr;+ execv(LAYA_STATIC_HOLDER, const_cast<char* const*>(argv));+ _exit(127);+ }+ int status = 0;+ ASSERT_EQ(waitpid(pid, &status, 0), pid);+ EXPECT_TRUE(WIFEXITED(status)) << "mode '" << mode << "' died by signal " << (WIFSIGNALED(status) ? WTERMSIG(status) : 0);+ EXPECT_EQ(WEXITSTATUS(status), 0) << "mode '" << mode << "'";+ }+}+TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {const Rig& rig = kRigs[GetParam()];NEED_RIG(rig, graph, dir);@@ -633,7 +797,6 @@ TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {for (int i = 0; i < 3; i++) {laya::onnx::Options o;o.intra_op_threads = 1;- o.inter_op_threads = 1;auto other = laya::onnx::Open(graph, o); // a second session, dropped without Closeif (!other.ok() || !other.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()) bad++;}diff --git a/laya-cpp/tests/tools/laya_c5_cli.cpp b/laya-cpp/tests/tools/laya_c5_cli.cppnew file mode 100644index 00000000..2762e366--- /dev/null+++ b/laya-cpp/tests/tools/laya_c5_cli.cpp@@ -0,0 +1,56 @@+// A line protocol over laya::agent::OpenOnnx for the differential fuzz (laya-e2e/cpp-lane/fuzz-c5).+// argv: graph checkpoint_dir. Bytes travel as hex.+// pred <hex json state> <hex json questions> -> "err <hex>" | "ok <hex compact prediction json>"+#include <iostream>+#include <sstream>++#include "laya/onnx.hpp"+#include "laya/pyvalue.hpp"+#include "laya/schema.hpp"++namespace {+std::string FromHex(const std::string& h) {+ std::string out;+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));+ return out;+}+std::string Hex(const std::string& s) {+ static const char* d = "0123456789abcdef";+ std::string o;+ for (unsigned char c : s) {+ o.push_back(d[c >> 4]);+ o.push_back(d[c & 15]);+ }+ return o;+}+std::string Err(const std::string& e) { return "err " + Hex(e); }++std::string Pred(const laya::agent::Agent& a, const std::string& state, const std::string& questions) {+ auto sv = laya::pyvalue::Parse(FromHex(state));+ if (!sv.ok()) return Err(sv.error());+ auto qs = laya::schema::Questions::Parse(FromHex(questions));+ if (!qs.ok()) return Err(qs.error());+ auto p = a.Predict(sv.value(), qs.value());+ if (!p.ok()) return Err(p.error());+ auto js = p.value().ToJson();+ if (!js.ok()) return Err(js.error());+ return "ok " + Hex(js.value());+}+} // namespace++int main(int argc, char** argv) {+ if (argc < 3) return 64;+ auto agent = laya::agent::OpenOnnx(argv[2], argv[1]);+ if (!agent.ok()) {+ std::cerr << agent.error() << std::endl;+ return 2;+ }+ std::string line;+ while (std::getline(std::cin, line)) {+ std::stringstream ss(line);+ std::string cmd, state, questions;+ ss >> cmd >> state >> questions;+ std::cout << (cmd == "pred" ? Pred(*agent.value(), state, questions) : Err("unknown command")) << std::endl;+ }+ return 0;+}diff --git a/laya-cpp/tests/tools/onnx_static_holder.cpp b/laya-cpp/tests/tools/onnx_static_holder.cppnew file mode 100644index 00000000..35decfdc--- /dev/null+++ b/laya-cpp/tests/tools/onnx_static_holder.cpp@@ -0,0 +1,32 @@+// Holds a Session and an Agent in statics, as a user program may, then exits normally. Exit code 0+// means static destruction did not fault. argv: graph checkpoint_dir [close].+#include <cstdio>+#include <cstring>+#include <memory>++#include "laya/onnx.hpp"++namespace {+std::unique_ptr<laya::infer::Session> g_session; // constructed before any ONNX Runtime static+std::unique_ptr<laya::agent::Agent> g_agent;+} // namespace++int main(int argc, char** argv) {+ if (argc < 3) return 64;+ auto s = laya::onnx::Open(argv[1]);+ if (!s.ok()) {+ std::fprintf(stderr, "open: %s\n", s.error().c_str());+ return 2;+ }+ g_session = std::move(s).value();+ auto a = laya::agent::OpenOnnx(argv[2], argv[1]);+ if (!a.ok()) {+ std::fprintf(stderr, "agent: %s\n", a.error().c_str());+ return 3;+ }+ g_agent = std::move(a).value();+ if (argc > 3 && !std::strcmp(argv[3], "close")) {+ if (!g_session->Close().empty() || !g_agent->Close().empty()) return 4;+ }+ return 0;+}