Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cpp): onnx backend review round 1

Adeshina Lasisi6441cda7d66b
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 9737d665..6e24ab7c 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -44,7 +44,7 @@ env:
# compile_flags). Raise it with every test added.
MIN_TESTS: "423"
# The onnx job registers the same tests plus the laya_onnx_tests cases.
- MIN_TESTS_ONNX: "466"
+ MIN_TESTS_ONNX: "479"
GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints
@@ -122,11 +122,12 @@ jobs:
if: ${{ !cancelled() }}
run: python laya-cpp/scripts/check_test_xml.py "$RUNNER_TEMP/ctest.xml" --min-tests "$MIN_TESTS"
- # The ONNX Runtime backend, mirroring go.yml's `onnx` job: the same graph cache keys as java.yml
- # (so a graph exported there is reused), real graphs for both shipped checkpoints, goldens
+ # The ONNX Runtime backend, mirroring go.yml's `onnx` job: real graphs for both shipped checkpoints, goldens
# recorded by Python's ONNXAgent on those graphs, and a floor on the tests that must run. The
# runtime comes from the release archive pinned by sha256 in laya-cpp/cmake/onnxruntime.cmake:
- # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`.
+ # CMake (URL_HASH) verifies it at the moment it is fetched, in every cell. Advisory like `build`,
+ # and like go.yml's `onnx` job, which is also `continue-on-error`: the MIN_TESTS_ONNX floor blocks
+ # nothing until this job and Go's are made blocking together.
onnx:
name: onnx (${{ matrix.compiler }}, ${{ matrix.preset }})
runs-on: ubuntu-latest
@@ -160,19 +161,22 @@ jobs:
pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
pip install -r laya-dotnet/tools/requirements-regen.txt
- # Keys are byte-identical to the predict cells in java.yml, so both share the entries.
+ # One cache per checkpoint, at its own path, under cpp-specific keys. java.yml caches only its
+ # multilingual graph, as a whole .work/onnx directory; sharing that entry would let this job
+ # save the english graph into it, and the english graph has no java entry to share. The hash
+ # inputs are java.yml's, so an exporter change invalidates both.
- name: Cache the multilingual graph
id: graph-multilingual
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
- path: laya-java/.work/onnx
- key: laya-java-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
+ path: laya-java/.work/onnx/multilingual
+ key: laya-cpp-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
- name: Cache the english graph
id: graph-english
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
- path: laya-java/.work/onnx
- key: laya-java-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
+ path: laya-java/.work/onnx/english
+ key: laya-cpp-onnx-${{ runner.os }}-english-${{ hashFiles('laya-java/scripts/prepare_checkpoint.py', 'scripts/export_onnx.py', 'laya/common.py', 'laya-dotnet/tools/requirements-regen.txt') }}
- name: Prepare both checkpoints and their graphs
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index fe87d4b6..ed5999f3 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -73,7 +73,17 @@ if(LAYA_BUILD_TESTS)
target_link_libraries(laya_onnx_tests PRIVATE laya_onnx laya_ort GTest::gtest_main)
laya_target_flags(laya_onnx_tests)
laya_ort_runtime_path(laya_onnx_tests)
- target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}")
+ add_executable(laya_onnx_static_holder tests/tools/onnx_static_holder.cpp)
+ target_link_libraries(laya_onnx_static_holder PRIVATE laya_onnx laya_ort)
+ laya_target_flags(laya_onnx_static_holder)
+ laya_ort_runtime_path(laya_onnx_static_holder)
+ add_dependencies(laya_onnx_tests laya_onnx_static_holder)
+ add_executable(laya_c5_cli tests/tools/laya_c5_cli.cpp)
+ target_link_libraries(laya_c5_cli PRIVATE laya_onnx laya_ort)
+ laya_target_flags(laya_c5_cli)
+ laya_ort_runtime_path(laya_c5_cli)
+ target_compile_definitions(laya_onnx_tests PRIVATE LAYA_ORT_VERSION="${LAYA_ORT_VERSION}"
+ LAYA_STATIC_HOLDER="$<TARGET_FILE:laya_onnx_static_holder>")
endif()
include(GoogleTest)
@@ -82,6 +92,8 @@ if(LAYA_BUILD_TESTS)
if(LAYA_ONNX)
gtest_discover_tests(laya_onnx_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 120)
+ # The refusals of cmake/onnxruntime.cmake (wrong version, missing header or library).
+ add_test(NAME ort_root_check COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/check_ort_root.sh ${LAYA_ORT_ROOT})
endif()
# Listing must stay cheap (it runs before every ctest): bound it at 3 s with the goldens loaded.
diff --git a/laya-cpp/cmake/onnxruntime.cmake b/laya-cpp/cmake/onnxruntime.cmake
index 6d8178eb..3195cfa0 100644
--- a/laya-cpp/cmake/onnxruntime.cmake
+++ b/laya-cpp/cmake/onnxruntime.cmake
@@ -29,10 +29,27 @@ endif()
if(NOT EXISTS "${_ort_root}/include/onnxruntime_cxx_api.h")
message(FATAL_ERROR "${_ort_root}/include/onnxruntime_cxx_api.h not found: not an ONNX Runtime release directory")
endif()
+# The pinned release is what the tests and goldens were measured against, so an override must be it.
+string(REGEX MATCH "^[0-9]+\\.([0-9]+)\\." _ort_unused "${LAYA_ORT_VERSION}")
+set(_ort_want_api "${CMAKE_MATCH_1}")
+file(STRINGS "${_ort_root}/include/onnxruntime_c_api.h" _ort_api_line REGEX "^#define ORT_API_VERSION [0-9]+")
+string(REGEX REPLACE "^#define ORT_API_VERSION ([0-9]+).*" "\\1" _ort_api "${_ort_api_line}")
+if(NOT _ort_api STREQUAL _ort_want_api)
+ message(FATAL_ERROR "${_ort_root} is ONNX Runtime API version '${_ort_api}', but laya_onnx is pinned to ${LAYA_ORT_VERSION} (API version ${_ort_want_api})")
+endif()
+if(EXISTS "${_ort_root}/VERSION_NUMBER")
+ file(STRINGS "${_ort_root}/VERSION_NUMBER" _ort_vn LIMIT_COUNT 1)
+ if(NOT _ort_vn STREQUAL LAYA_ORT_VERSION)
+ message(FATAL_ERROR "${_ort_root}/VERSION_NUMBER says '${_ort_vn}', but laya_onnx is pinned to ${LAYA_ORT_VERSION}")
+ endif()
+endif()
if(APPLE)
set(_ort_lib "${_ort_root}/lib/libonnxruntime.${LAYA_ORT_VERSION}.dylib")
if(NOT EXISTS "${_ort_lib}")
file(GLOB _ort_lib "${_ort_root}/lib/libonnxruntime*.dylib")
+ if(NOT _ort_lib)
+ message(FATAL_ERROR "no libonnxruntime*.dylib under ${_ort_root}/lib")
+ endif()
list(GET _ort_lib 0 _ort_lib)
endif()
elseif(WIN32)
@@ -48,6 +65,7 @@ message(STATUS "ONNX Runtime ${LAYA_ORT_VERSION}: ${_ort_lib}")
add_library(laya_ort SHARED IMPORTED)
set_target_properties(laya_ort PROPERTIES IMPORTED_LOCATION "${_ort_lib}" INTERFACE_INCLUDE_DIRECTORIES "${_ort_root}/include")
set(LAYA_ORT_LIBDIR "${_ort_root}/lib")
+set(LAYA_ORT_ROOT "${_ort_root}")
# Build-tree executables find the shared library through an rpath to its directory.
function(laya_ort_runtime_path target)
diff --git a/laya-cpp/include/laya/onnx.hpp b/laya-cpp/include/laya/onnx.hpp
index f391721b..64987ab1 100644
--- a/laya-cpp/include/laya/onnx.hpp
+++ b/laya-cpp/include/laya/onnx.hpp
@@ -12,8 +12,10 @@ namespace laya {
namespace onnx {
struct Options {
- int intra_op_threads = 0; // 0 leaves ONNX Runtime's default; at most kMaxThreads
- int inter_op_threads = 0;
+ // 0 leaves ONNX Runtime's default (one intra-op pool sized to the cores). Every session owns its
+ // own pool and sessions do not share one, so many sessions in one process should set this. There
+ // is no inter-op option: the graph runs sequentially, where it would have no effect.
+ int intra_op_threads = 0; // at most kMaxThreads
};
constexpr int kMaxThreads = 4096;
@@ -21,8 +23,10 @@ constexpr int kMaxThreads = 4096;
std::string RuntimeVersion();
// A Session over one fused laya.onnx (+ laya.onnx.data beside it). Checks the graph's inputs and
-// outputs at once. Run is thread-safe; Close is idempotent and waits for runs in flight. No
-// exception leaves this API.
+// outputs at once. Run is thread-safe and stops in flight when its CancelToken fires (kCancelled).
+// Close is idempotent, refuses new runs, terminates the runs in flight and returns once they have
+// left. A Session or Agent may be held in a static: the process-wide ONNX Runtime environment is
+// never destroyed. No exception leaves this API.
Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options = {});
} // namespace onnx
diff --git a/laya-cpp/scripts/check_ort_root.sh b/laya-cpp/scripts/check_ort_root.sh
new file mode 100755
index 00000000..1c4dc122
--- /dev/null
+++ b/laya-cpp/scripts/check_ort_root.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Configure-only check of cmake/onnxruntime.cmake's refusals. usage: check_ort_root.sh <good-ort-root>
+# A root with the wrong API version, a wrong VERSION_NUMBER, no header, or no library must each
+# fail at configure time with the named message, and the good root must configure.
+set -eu
+good=$1
+src=$(cd "$(dirname "$0")/.." && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+fail=0
+try() { # name root want-substring ("" = must succeed)
+ if out=$(cmake -S "$src" -B "$tmp/b-$1" -DLAYA_ONNX=ON -DLAYA_ONNXRUNTIME_ROOT="$2" 2>&1); then rc=0; else rc=1; fi
+ if [ -z "$3" ]; then
+ [ $rc -eq 0 ] || { echo "FAIL $1: good root refused"; fail=1; }
+ else
+ [ $rc -ne 0 ] || { echo "FAIL $1: accepted"; fail=1; return; }
+ echo "$out" | tr -s ' \n' ' ' | grep -q "$3" || { echo "FAIL $1: wrong message"; echo "$out" | tail -4; fail=1; }
+ fi
+}
+copy() { mkdir -p "$tmp/$1"; cp -R "$good/include" "$tmp/$1/include"; mkdir -p "$tmp/$1/lib"; cp -R "$good/lib/." "$tmp/$1/lib/"; cp "$good/VERSION_NUMBER" "$tmp/$1/" 2>/dev/null || true; }
+try good "$good" ""
+copy api; sed -i.bak 's/^#define ORT_API_VERSION [0-9]*/#define ORT_API_VERSION 99/' "$tmp/api/include/onnxruntime_c_api.h"
+try wrong-api "$tmp/api" "API version '99'"
+copy vn; echo 1.29.0 > "$tmp/vn/VERSION_NUMBER"
+try wrong-version-number "$tmp/vn" "VERSION_NUMBER says"
+copy nohdr; rm "$tmp/nohdr/include/onnxruntime_cxx_api.h"
+try no-header "$tmp/nohdr" "not an ONNX Runtime release directory"
+copy nolib; find "$tmp/nolib/lib" -maxdepth 1 -type f -name 'libonnxruntime*' -delete; find "$tmp/nolib/lib" -maxdepth 1 -type l -name 'libonnxruntime*' -delete
+try no-library "$tmp/nolib" "library not found\|no libonnxruntime"
+[ $fail -eq 0 ] && echo "check_ort_root: 5 cases ok"
+exit $fail
diff --git a/laya-cpp/src/onnx.cpp b/laya-cpp/src/onnx.cpp
index 878e6f07..94d6b23f 100644
--- a/laya-cpp/src/onnx.cpp
+++ b/laya-cpp/src/onnx.cpp
@@ -3,9 +3,13 @@
#include <onnxruntime_cxx_api.h>
#include <filesystem>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
#include <memory>
#include <mutex>
-#include <shared_mutex>
+#include <set>
+#include <thread>
#include "laya/onnx_graph.hpp"
@@ -13,18 +17,13 @@ namespace laya {
namespace onnx {
namespace {
-// One Ort::Env per process, shared by every session and freed with the last one, so there is no
-// static destructor racing a session's teardown. The mutex orders creation against that release.
-std::shared_ptr<Ort::Env> SharedEnv() {
- static std::mutex mu;
- static std::weak_ptr<Ort::Env> weak;
- std::lock_guard<std::mutex> lock(mu);
- std::shared_ptr<Ort::Env> env = weak.lock();
- if (!env) {
- env = std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_ERROR, "laya");
- weak = env;
- }
- return env;
+// One Ort::Env per process, leaked on purpose. A session or agent may be held in a static whose
+// destructor runs after any Env destructor would have, and ORT's own statics may already be gone;
+// never destroying the Env means no destruction order can fault. The logging level is FATAL so a
+// refused run is reported only through its Result.
+Ort::Env& SharedEnv() {
+ static Ort::Env* const env = new Ort::Env(ORT_LOGGING_LEVEL_FATAL, "laya");
+ return *env;
}
std::string Dtype(ONNXTensorElementDataType t) {
@@ -40,9 +39,45 @@ std::string Dtype(ONNXTensorElementDataType t) {
const char* const kInputNames[5] = {"input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"};
const char* const kOutputNames[2] = {"logits", "act_logits"};
+// Fires SetTerminate on a run when the cancel token fires. The token is a polled flag, so the
+// watcher waits on a condition variable with a short timeout; the run ending wakes it at once.
+class Watcher {
+ public:
+ Watcher(const CancelToken& cancel, Ort::RunOptions& ro) : cancel_(cancel), ro_(ro), thread_([this] { Loop(); }) {}
+ ~Watcher() {
+ {
+ std::lock_guard<std::mutex> lock(mu_);
+ done_ = true;
+ }
+ cv_.notify_all();
+ thread_.join();
+ }
+ bool terminated() const { return terminated_.load(); }
+
+ private:
+ void Loop() {
+ std::unique_lock<std::mutex> lock(mu_);
+ while (!done_) {
+ if (cancel_.cancelled()) {
+ terminated_ = true;
+ ro_.SetTerminate();
+ return;
+ }
+ cv_.wait_for(lock, std::chrono::milliseconds(2));
+ }
+ }
+ CancelToken cancel_;
+ Ort::RunOptions& ro_;
+ std::mutex mu_;
+ std::condition_variable cv_;
+ bool done_ = false;
+ std::atomic<bool> terminated_{false};
+ std::thread thread_; // last: started after the rest is built
+};
+
class OrtSession final : public infer::Session {
public:
- OrtSession(std::shared_ptr<Ort::Env> env, std::unique_ptr<Ort::Session> s) : env_(std::move(env)), session_(std::move(s)) {}
+ explicit OrtSession(std::unique_ptr<Ort::Session> s) : session_(std::move(s)) {}
~OrtSession() override { Close(); }
Result<infer::Output> Run(const infer::Batch& b, const CancelToken& cancel) override {
@@ -57,9 +92,13 @@ class OrtSession final : public infer::Session {
}
}
+ // Refuses new runs, terminates the runs in flight, and releases the session once they have left.
std::string Close() override {
try {
- std::unique_lock<std::shared_mutex> lock(mu_); // waits for runs in flight
+ std::unique_lock<std::mutex> lock(mu_);
+ closing_ = true;
+ for (Ort::RunOptions* r : flights_) r->SetTerminate();
+ idle_.wait(lock, [this] { return flights_.empty(); });
session_.reset();
return "";
} catch (...) {
@@ -68,6 +107,29 @@ class OrtSession final : public infer::Session {
}
private:
+ // Registers a run for Close to terminate; the destructor leaves the list and wakes Close.
+ class Flight {
+ public:
+ Flight(OrtSession& s, Ort::RunOptions& ro) : s_(s), ro_(ro) {
+ std::lock_guard<std::mutex> lock(s_.mu_);
+ if (s_.closing_ || !s_.session_) return;
+ s_.flights_.insert(&ro_);
+ ok_ = true;
+ }
+ ~Flight() {
+ if (!ok_) return;
+ std::lock_guard<std::mutex> lock(s_.mu_);
+ s_.flights_.erase(&ro_);
+ s_.idle_.notify_all();
+ }
+ bool ok() const { return ok_; }
+
+ private:
+ OrtSession& s_;
+ Ort::RunOptions& ro_;
+ bool ok_ = false;
+ };
+
Result<infer::Output> RunImpl(const infer::Batch& b, const CancelToken& cancel) {
using R = Result<infer::Output>;
if (cancel.cancelled()) return R::Fail(infer::kCancelled);
@@ -76,8 +138,9 @@ class OrtSession final : public infer::Session {
for (uint8_t m : b.marker_mask) {
if (m > 1) return R::Fail("onnx: marker_mask holds a value other than 0 or 1");
}
- std::shared_lock<std::shared_mutex> lock(mu_);
- if (!session_) return R::Fail("onnx: session is closed");
+ Ort::RunOptions ro;
+ Flight flight(*this, ro);
+ if (!flight.ok()) return R::Fail("onnx: session is closed");
const Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
const int64_t shape2[2] = {b.rows, b.length};
@@ -96,8 +159,14 @@ class OrtSession final : public infer::Session {
shape_m, 2, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL));
in.push_back(i64(b.qtype, shape1, 1));
- Ort::RunOptions ro;
- std::vector<Ort::Value> out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);
+ Watcher watcher(cancel, ro);
+ std::vector<Ort::Value> out;
+ try {
+ out = session_->Run(ro, kInputNames, in.data(), in.size(), kOutputNames, 2);
+ } catch (const Ort::Exception& e) {
+ if (watcher.terminated() || cancel.cancelled()) return R::Fail(infer::kCancelled);
+ throw;
+ }
if (out.size() != 2) return R::Fail("onnx: run returned " + std::to_string(out.size()) + " outputs, want 2");
if (cancel.cancelled()) return R::Fail(infer::kCancelled);
@@ -115,8 +184,10 @@ class OrtSession final : public infer::Session {
return CollectOutput(b, t[0], t[1]);
}
- std::shared_ptr<Ort::Env> env_; // keeps the environment alive past session_
- std::shared_mutex mu_;
+ std::mutex mu_; // guards closing_, flights_ and the release of session_
+ std::condition_variable idle_;
+ bool closing_ = false;
+ std::set<Ort::RunOptions*> flights_;
std::unique_ptr<Ort::Session> session_;
};
@@ -132,19 +203,17 @@ std::string RuntimeVersion() {
Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, const Options& options) {
using R = Result<std::unique_ptr<infer::Session>>;
- if (options.intra_op_threads < 0 || options.inter_op_threads < 0 || options.intra_op_threads > kMaxThreads ||
- options.inter_op_threads > kMaxThreads) {
- return R::Fail("onnx: thread counts must be between 0 and " + std::to_string(kMaxThreads));
+ if (options.intra_op_threads < 0 || options.intra_op_threads > kMaxThreads) {
+ return R::Fail("onnx: intra_op_threads must be between 0 and " + std::to_string(kMaxThreads));
}
const std::string path_problem = CheckPath(graph_path);
if (!path_problem.empty()) return R::Fail("onnx: " + path_problem);
try {
- std::shared_ptr<Ort::Env> env = SharedEnv();
+ Ort::Env& env = SharedEnv();
Ort::SessionOptions so;
if (options.intra_op_threads > 0) so.SetIntraOpNumThreads(options.intra_op_threads);
- if (options.inter_op_threads > 0) so.SetInterOpNumThreads(options.inter_op_threads);
const std::filesystem::path p(graph_path);
- auto session = std::make_unique<Ort::Session>(*env, p.c_str(), so);
+ auto session = std::make_unique<Ort::Session>(env, p.c_str(), so);
Ort::AllocatorWithDefaultOptions alloc;
auto describe = [&](bool inputs) {
@@ -168,7 +237,7 @@ Result<std::unique_ptr<infer::Session>> Open(const std::string& graph_path, cons
};
const std::string bad = CheckGraph(describe(true), describe(false));
if (!bad.empty()) return R::Fail("onnx: graph " + graph_path + ": " + bad);
- return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(env), std::move(session))));
+ return R::Ok(std::unique_ptr<infer::Session>(new OrtSession(std::move(session))));
} catch (const Ort::Exception& e) {
return R::Fail("onnx: cannot open " + graph_path + ": " + e.what());
} catch (const std::exception& e) {
diff --git a/laya-cpp/src/onnx_graph.cpp b/laya-cpp/src/onnx_graph.cpp
index 808c0b1f..9a498a7a 100644
--- a/laya-cpp/src/onnx_graph.cpp
+++ b/laya-cpp/src/onnx_graph.cpp
@@ -118,7 +118,7 @@ Result<infer::Output> CollectOutput(const infer::Batch& b, const OutTensor& logi
return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " holds " + std::to_string(c.t.count) + " values");
}
for (int64_t i = 0; i < c.t.count; i++) {
- if (std::isnan(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN");
+ if (!std::isfinite(c.t.data[i])) return Result<infer::Output>::Fail(std::string("onnx: output ") + c.name + " contains NaN or Inf");
}
}
infer::Output o;
diff --git a/laya-cpp/tests/onnx_hostile_graphs.inc b/laya-cpp/tests/onnx_hostile_graphs.inc
new file mode 100644
index 00000000..5a6a61b8
--- /dev/null
+++ b/laya-cpp/tests/onnx_hostile_graphs.inc
@@ -0,0 +1,5 @@
+// A graph whose one initializer names external data outside the graph's directory.
+const char kDotdotGraph[] = "\x08\x0e\x3a\x66\x0a\x10\x0a\x01\x77\x12\x01\x79\x22\x08\x49\x64\x65\x6e\x74\x69\x74\x79\x12\x01\x67\x2a\x3e\x08\x02\x10\x01\x42\x01\x77\x6a\x19\x0a\x08\x6c\x6f\x63\x61\x74\x69\x6f\x6e\x12\x0d\x2e\x2e\x2f\x73\x65\x63\x72\x65\x74\x2e\x62\x69\x6e\x6a\x0b\x0a\x06\x6f\x66\x66\x73\x65\x74\x12\x01\x30\x6a\x0b\x0a\x06\x6c\x65\x6e\x67\x74\x68\x12\x01\x38\x70\x01\x62\x0f\x0a\x01\x79\x12\x0a\x0a\x08\x08\x01\x12\x04\x0a\x02\x08\x02\x42\x02\x10\x1c";
+constexpr size_t kDotdotGraphSize = 110;
+const char kAbsGraph[] = "\x08\x0e\x3a\x63\x0a\x10\x0a\x01\x77\x12\x01\x79\x22\x08\x49\x64\x65\x6e\x74\x69\x74\x79\x12\x01\x67\x2a\x3b\x08\x02\x10\x01\x42\x01\x77\x6a\x16\x0a\x08\x6c\x6f\x63\x61\x74\x69\x6f\x6e\x12\x0a\x2f\x65\x74\x63\x2f\x68\x6f\x73\x74\x73\x6a\x0b\x0a\x06\x6f\x66\x66\x73\x65\x74\x12\x01\x30\x6a\x0b\x0a\x06\x6c\x65\x6e\x67\x74\x68\x12\x01\x38\x70\x01\x62\x0f\x0a\x01\x79\x12\x0a\x0a\x08\x08\x01\x12\x04\x0a\x02\x08\x02\x42\x02\x10\x1c";
+constexpr size_t kAbsGraphSize = 107;
diff --git a/laya-cpp/tests/onnx_test.cpp b/laya-cpp/tests/onnx_test.cpp
index 72e33341..5755d7e1 100644
--- a/laya-cpp/tests/onnx_test.cpp
+++ b/laya-cpp/tests/onnx_test.cpp
@@ -4,6 +4,7 @@
#include <gtest/gtest.h>
#include <atomic>
+#include <chrono>
#include <cmath>
#include <cstdio>
#include <filesystem>
@@ -13,6 +14,7 @@
#include <mutex>
#include <thread>
+#include <sys/wait.h>
#include <unistd.h>
#include "golden.hpp"
@@ -218,6 +220,20 @@ TEST(OnnxCollect, ANaNIsAnErrorNotAnAnswer) {
EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());
}
+TEST(OnnxCollect, AnInfinityIsAnErrorNotAnAnswer) {
+ const Batch b = Tiny(1, 3);
+ const float inf = std::numeric_limits<float>::infinity();
+ for (float bad : {inf, -inf}) {
+ std::vector<float> l = {1, bad, 3}, a = {1, 2};
+ auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("Inf"), std::string::npos);
+ l[1] = 0;
+ a[1] = bad;
+ EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());
+ }
+}
+
TEST(OnnxCollect, WrongShapesAreErrors) {
const Batch b = Tiny(2, 3);
const std::vector<float> l(6, 1.f), a(4, 1.f), small(2, 1.f);
@@ -234,11 +250,10 @@ TEST(OnnxOpen, ThreadCountsAreRange_Checked) {
laya::onnx::Options o;
o.intra_op_threads = -1;
EXPECT_FALSE(laya::onnx::Open("/nonexistent/laya.onnx", o).ok());
- o.intra_op_threads = 0;
- o.inter_op_threads = laya::onnx::kMaxThreads + 1;
+ o.intra_op_threads = laya::onnx::kMaxThreads + 1;
const auto r = laya::onnx::Open("/nonexistent/laya.onnx", o);
ASSERT_FALSE(r.ok());
- EXPECT_NE(r.error().find("thread counts"), std::string::npos);
+ EXPECT_NE(r.error().find("intra_op_threads"), std::string::npos);
}
TEST(OnnxOpen, AMissingGraphIsAnError) {
@@ -259,6 +274,29 @@ TEST(OnnxOpen, AnEmptyFileIsAnError) {
EXPECT_FALSE(laya::onnx::Open(d.File("laya.onnx", "")).ok());
}
+// ONNX Runtime refuses external data that leaves the graph's directory (`..` or an absolute path);
+// this pins that, with a real file sitting where the graph points.
+#include "onnx_hostile_graphs.inc"
+TEST(OnnxOpen, ExternalDataOutsideTheGraphDirectoryIsRefused) {
+ TempDir d;
+ fs::create_directories(d.str() + "/sub");
+ {
+ std::ofstream(d.str() + "/secret.bin", std::ios::binary) << std::string(8, '\0');
+ }
+ const struct {
+ const char* name;
+ const char* bytes;
+ size_t size;
+ } cases[] = {{"dotdot", kDotdotGraph, kDotdotGraphSize}, {"absolute", kAbsGraph, kAbsGraphSize}};
+ for (const auto& c : cases) {
+ const std::string path = d.str() + "/sub/" + c.name + ".onnx";
+ std::ofstream(path, std::ios::binary).write(c.bytes, std::streamsize(c.size));
+ const auto r = laya::onnx::Open(path);
+ ASSERT_FALSE(r.ok()) << c.name << " opened";
+ EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
+ }
+}
+
TEST(OnnxOpen, TheSplitLayoutIsRefusedByOpen) {
TempDir d;
d.File("encoder.onnx");
@@ -300,18 +338,6 @@ std::string Need(const char* var) {
std::mutex g_agents_mu;
std::map<std::string, std::unique_ptr<laya::agent::Agent>> g_agents;
-// The agents hold ONNX Runtime, which must be released before static destruction begins (its own
-// statics may already be gone by then), so the test environment closes them.
-class CloseAgents : public ::testing::Environment {
- public:
- void TearDown() override {
- std::lock_guard<std::mutex> lock(g_agents_mu);
- for (auto& kv : g_agents) EXPECT_EQ(kv.second->Close(), "");
- g_agents.clear();
- }
-};
-const ::testing::Environment* const g_close_agents = ::testing::AddGlobalTestEnvironment(new CloseAgents);
-
// One agent per checkpoint for the whole process; the tests only read it.
laya::agent::Agent* Shared(const Rig& rig, const std::string& graph, const std::string& dir) {
auto& agents = g_agents;
@@ -597,6 +623,14 @@ TEST_P(OnnxRig, SessionRunIsGuardedAgainstHostileBatches) {
Batch empty;
EXPECT_FALSE(session.Run(empty, live).ok());
+ // Passes every check of ours and fails inside ONNX Runtime (a token id past the vocabulary): the
+ // failure must come back as a Result, not an exception.
+ Batch past_vocab = Tiny(1, 3);
+ past_vocab.input_ids[0] = 1000000000;
+ const auto pv = session.Run(past_vocab, live);
+ ASSERT_FALSE(pv.ok());
+ EXPECT_EQ(pv.error().rfind("onnx: run:", 0), 0u) << pv.error();
+
laya::CancelToken fired;
fired.Cancel();
auto c = session.Run(Tiny(1, 3), fired);
@@ -619,6 +653,136 @@ TEST_P(OnnxRig, CloseIsIdempotentAndRunAfterCloseIsAnError) {
EXPECT_NE(r.error().find("closed"), std::string::npos) << r.error();
}
+// A batch that keeps the runtime busy for seconds.
+Batch Long(int64_t rows, int64_t length) {
+ Batch b = Tiny(rows, 3);
+ b.length = length;
+ b.input_ids.assign(size_t(rows * length), 5);
+ b.attention_mask.assign(size_t(rows * length), 1);
+ return b;
+}
+using Clock = std::chrono::steady_clock;
+double Since(Clock::time_point t) { return std::chrono::duration<double>(Clock::now() - t).count(); }
+constexpr int64_t kLongRows = 16, kLongLength = 192;
+
+// Seconds one uncancelled long run takes; the tests below only mean something if it is long.
+double LongRunSeconds(laya::infer::Session& s) {
+ const auto t = Clock::now();
+ const auto r = s.Run(Long(kLongRows, kLongLength), laya::CancelToken());
+ EXPECT_TRUE(r.ok()) << (r.ok() ? "" : r.error());
+ return Since(t);
+}
+
+TEST_P(OnnxRig, ACancelDuringARunStopsTheRun) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ const double full = LongRunSeconds(*s.value());
+ ASSERT_GT(full, 0.8) << "the long batch is too short to tell a mid-run cancel from a late one";
+ laya::CancelToken token;
+ std::thread canceller([&] {
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ token.Cancel();
+ });
+ const auto t = Clock::now();
+ const auto r = s.value()->Run(Long(kLongRows, kLongLength), token);
+ const double took = Since(t);
+ canceller.join();
+ ASSERT_FALSE(r.ok());
+ EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();
+ EXPECT_LT(took, full / 3) << "cancelled run took " << took << " s of " << full << " s";
+ EXPECT_TRUE(s.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()); // the session is still usable
+}
+
+TEST_P(OnnxRig, ACancelThatLandsAfterTheRunStillReportsCancelled) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ // The token fires inside the post-run window: the run is tiny, so a watcher cannot beat it
+ // reliably; either outcome is allowed only if it is cancelled or a complete correct result.
+ for (int i = 0; i < 50; i++) {
+ laya::CancelToken token;
+ std::thread canceller([&] { token.Cancel(); });
+ const auto r = s.value()->Run(Tiny(2, 3), token);
+ canceller.join();
+ if (!r.ok()) EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();
+ else EXPECT_EQ(r.value().logits.size(), 6u);
+ }
+}
+
+TEST_P(OnnxRig, CloseInterruptsARunInFlight) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ const double full = LongRunSeconds(*s.value());
+ ASSERT_GT(full, 0.8);
+ std::atomic<bool> failed{false};
+ std::thread runner([&] { failed = !s.value()->Run(Long(kLongRows, kLongLength), laya::CancelToken()).ok(); });
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ const auto t = Clock::now();
+ EXPECT_EQ(s.value()->Close(), "");
+ const double took = Since(t);
+ runner.join();
+ EXPECT_TRUE(failed.load());
+ EXPECT_LT(took, full / 3) << "Close took " << took << " s of a " << full << " s run";
+}
+
+TEST_P(OnnxRig, CloseReturnsPromptlyUnderSustainedRunTraffic) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ auto s = laya::onnx::Open(graph);
+ ASSERT_TRUE(s.ok()) << s.error();
+ const auto t0 = Clock::now();
+ ASSERT_TRUE(s.value()->Run(Long(4, 128), laya::CancelToken()).ok());
+ const double one = Since(t0); // one such run alone
+ ASSERT_GT(one, 0.15);
+ std::atomic<bool> stop{false};
+ std::atomic<int> ran{0}, refused{0}, other{0};
+ std::vector<std::thread> threads;
+ for (int i = 0; i < 4; i++) {
+ threads.emplace_back([&] {
+ while (!stop.load()) {
+ const auto r = s.value()->Run(Long(4, 128), laya::CancelToken());
+ if (r.ok()) ran++;
+ else if (r.error().find("closed") != std::string::npos || r.error().find("run:") != std::string::npos) refused++;
+ else other++;
+ }
+ });
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds(300));
+ const auto t = Clock::now();
+ EXPECT_EQ(s.value()->Close(), "");
+ const double took = Since(t);
+ stop = true;
+ for (auto& th : threads) th.join();
+ EXPECT_GT(ran.load() + refused.load(), 0); // every thread ended on a Result, none on a hang
+ EXPECT_EQ(other.load(), 0);
+ EXPECT_LT(took, one / 2) << "Close took " << took << " s with one run alone taking " << one << " s";
+}
+
+// A user program may hold a Session and an Agent in statics; it must exit cleanly, closed or not.
+TEST_P(OnnxRig, AStaticSessionAndAgentExitCleanly) {
+ const Rig& rig = kRigs[GetParam()];
+ NEED_RIG(rig, graph, dir);
+ for (const char* mode : {"", "close"}) {
+ const pid_t pid = fork();
+ ASSERT_GE(pid, 0);
+ if (pid == 0) {
+ const char* argv[] = {LAYA_STATIC_HOLDER, graph.c_str(), dir.c_str(), mode, nullptr};
+ if (*mode == 0) argv[3] = nullptr;
+ execv(LAYA_STATIC_HOLDER, const_cast<char* const*>(argv));
+ _exit(127);
+ }
+ int status = 0;
+ ASSERT_EQ(waitpid(pid, &status, 0), pid);
+ EXPECT_TRUE(WIFEXITED(status)) << "mode '" << mode << "' died by signal " << (WIFSIGNALED(status) ? WTERMSIG(status) : 0);
+ EXPECT_EQ(WEXITSTATUS(status), 0) << "mode '" << mode << "'";
+ }
+}
+
TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {
const Rig& rig = kRigs[GetParam()];
NEED_RIG(rig, graph, dir);
@@ -633,7 +797,6 @@ TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {
for (int i = 0; i < 3; i++) {
laya::onnx::Options o;
o.intra_op_threads = 1;
- o.inter_op_threads = 1;
auto other = laya::onnx::Open(graph, o); // a second session, dropped without Close
if (!other.ok() || !other.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()) bad++;
}
diff --git a/laya-cpp/tests/tools/laya_c5_cli.cpp b/laya-cpp/tests/tools/laya_c5_cli.cpp
new file mode 100644
index 00000000..2762e366
--- /dev/null
+++ b/laya-cpp/tests/tools/laya_c5_cli.cpp
@@ -0,0 +1,56 @@
+// A line protocol over laya::agent::OpenOnnx for the differential fuzz (laya-e2e/cpp-lane/fuzz-c5).
+// argv: graph checkpoint_dir. Bytes travel as hex.
+// pred <hex json state> <hex json questions> -> "err <hex>" | "ok <hex compact prediction json>"
+#include <iostream>
+#include <sstream>
+
+#include "laya/onnx.hpp"
+#include "laya/pyvalue.hpp"
+#include "laya/schema.hpp"
+
+namespace {
+std::string FromHex(const std::string& h) {
+ std::string out;
+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));
+ return out;
+}
+std::string Hex(const std::string& s) {
+ static const char* d = "0123456789abcdef";
+ std::string o;
+ for (unsigned char c : s) {
+ o.push_back(d[c >> 4]);
+ o.push_back(d[c & 15]);
+ }
+ return o;
+}
+std::string Err(const std::string& e) { return "err " + Hex(e); }
+
+std::string Pred(const laya::agent::Agent& a, const std::string& state, const std::string& questions) {
+ auto sv = laya::pyvalue::Parse(FromHex(state));
+ if (!sv.ok()) return Err(sv.error());
+ auto qs = laya::schema::Questions::Parse(FromHex(questions));
+ if (!qs.ok()) return Err(qs.error());
+ auto p = a.Predict(sv.value(), qs.value());
+ if (!p.ok()) return Err(p.error());
+ auto js = p.value().ToJson();
+ if (!js.ok()) return Err(js.error());
+ return "ok " + Hex(js.value());
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ if (argc < 3) return 64;
+ auto agent = laya::agent::OpenOnnx(argv[2], argv[1]);
+ if (!agent.ok()) {
+ std::cerr << agent.error() << std::endl;
+ return 2;
+ }
+ std::string line;
+ while (std::getline(std::cin, line)) {
+ std::stringstream ss(line);
+ std::string cmd, state, questions;
+ ss >> cmd >> state >> questions;
+ std::cout << (cmd == "pred" ? Pred(*agent.value(), state, questions) : Err("unknown command")) << std::endl;
+ }
+ return 0;
+}
diff --git a/laya-cpp/tests/tools/onnx_static_holder.cpp b/laya-cpp/tests/tools/onnx_static_holder.cpp
new file mode 100644
index 00000000..35decfdc
--- /dev/null
+++ b/laya-cpp/tests/tools/onnx_static_holder.cpp
@@ -0,0 +1,32 @@
+// Holds a Session and an Agent in statics, as a user program may, then exits normally. Exit code 0
+// means static destruction did not fault. argv: graph checkpoint_dir [close].
+#include <cstdio>
+#include <cstring>
+#include <memory>
+
+#include "laya/onnx.hpp"
+
+namespace {
+std::unique_ptr<laya::infer::Session> g_session; // constructed before any ONNX Runtime static
+std::unique_ptr<laya::agent::Agent> g_agent;
+} // namespace
+
+int main(int argc, char** argv) {
+ if (argc < 3) return 64;
+ auto s = laya::onnx::Open(argv[1]);
+ if (!s.ok()) {
+ std::fprintf(stderr, "open: %s\n", s.error().c_str());
+ return 2;
+ }
+ g_session = std::move(s).value();
+ auto a = laya::agent::OpenOnnx(argv[2], argv[1]);
+ if (!a.ok()) {
+ std::fprintf(stderr, "agent: %s\n", a.error().c_str());
+ return 3;
+ }
+ g_agent = std::move(a).value();
+ if (argc > 3 && !std::strcmp(argv[3], "close")) {
+ if (!g_session->Close().empty() || !g_agent->Close().empty()) return 4;
+ }
+ return 0;
+}