Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cpp): decode and agent review round 1
Adeshina Lasisi140c33491199
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex 4036563d..a9fd5a7b 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -42,7 +42,7 @@ concurrency:env:# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus# compile_flags). Raise it with every test added.- MIN_TESTS: "409"+ MIN_TESTS: "423"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints@@ -60,6 +60,10 @@ jobs:matrix:compiler: [g++, clang++]preset: [debug, asan-ubsan]+ include:+ # The Agent claims thread safety; the whole suite runs under ThreadSanitizer once.+ - compiler: clang+++ preset: tsansteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex ff41f10f..3f8bfe42 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -59,6 +59,9 @@ if(LAYA_BUILD_TESTS)add_executable(laya_c3_cli tests/tools/laya_c3_cli.cpp)target_link_libraries(laya_c3_cli PRIVATE laya)laya_target_flags(laya_c3_cli)+ add_executable(laya_c4_cli tests/tools/laya_c4_cli.cpp)+ target_link_libraries(laya_c4_cli PRIVATE laya)+ laya_target_flags(laya_c4_cli)include(GoogleTest)# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.diff --git a/laya-cpp/include/laya/agent.hpp b/laya-cpp/include/laya/agent.hppindex ac30a4fe..e48df935 100644--- a/laya-cpp/include/laya/agent.hpp+++ b/laya-cpp/include/laya/agent.hpp@@ -3,9 +3,12 @@// PredictBatch states against a question set. Hooks, the confidence gate, predict_long, calibration// and digests are later changes.//-// Errors are returned as Result<T>; no exception crosses this API. An Agent is safe for concurrent-// use: any number of threads may call Predict while another calls Close, which waits for the calls-// in flight and then closes the Session.+// Errors are returned as Result<T>; no exception crosses this API, and an exception thrown by a+// Session is caught and reported as an error. An Agent is safe for concurrent use: any number of+// threads may call Predict while another calls Close.+//+// Deferred to later changes: per-language temperatures (CallOptions has no `lang` until then), the+// confidence gate, predict_long, hooks, calibration and digests.#include <atomic>#include <condition_variable>#include <cstdint>@@ -46,8 +49,9 @@ struct Config {// Per-call settings. Zero values mean the defaults.struct CallOptions {- std::string lang; // language tag selecting a temperature override- int64_t batch_size = 0; // states per session run; <= 0 runs all states at once+ // States per session run. <= 0 runs all states at once, split only where one run would exceed+ // infer::kMaxTensorElements in any tensor. A positive value is used as given; a run over the cap fails.+ int64_t batch_size = 0;bool sort_by_length = false; // group similar lengths per run; results stay in input orderint64_t max_len = 0; // > 0 overrides the checkpoint'sint64_t head_max_len = 0; // > 0 overrides the checkpoint's@@ -75,8 +79,11 @@ class Agent {Result<std::vector<schema::Prediction>> PredictBatch(const std::vector<pyvalue::Value>& states,const schema::Questions& qs,const CallOptions& opts = {}) const;- // Waits for the calls in flight, then closes the session. Idempotent; later calls fail with kClosed.- // Returns the session's close error, "" on success.+ // Refuses new calls at once and waits for each call in flight to return; a call with runs left is+ // cut short with kClosed at its next run. Then closes the session exactly once, outside the lock;+ // concurrent and later Closes return the same result. Later calls fail with kClosed. Returns the+ // session's close error, "" on success. Must not be called from inside a Session (it would wait+ // for itself).std::string Close();private:@@ -100,6 +107,7 @@ class Agent {mutable std::condition_variable cv_;mutable int64_t inflight_ = 0;bool shut_ = false;+ bool close_started_ = false;bool close_done_ = false;std::string close_err_;std::atomic<bool> closed_{false};diff --git a/laya-cpp/include/laya/cancel.hpp b/laya-cpp/include/laya/cancel.hppindex 18e06064..6328e4ee 100644--- a/laya-cpp/include/laya/cancel.hpp+++ b/laya-cpp/include/laya/cancel.hpp@@ -1,6 +1,6 @@#pragma once// A cancellation token. Copies share one flag, so the caller keeps a copy and cancels it from any-// thread while a call holds another. A default token can never be cancelled.+// thread while a call holds another. Every token, default included, owns a flag.#include <atomic>#include <memory>@@ -8,16 +8,10 @@ namespace laya {class CancelToken {public:- CancelToken() = default;- static CancelToken Make() {- CancelToken t;- t.flag_ = std::make_shared<std::atomic<bool>>(false);- return t;- }- void Cancel() const {- if (flag_) flag_->store(true, std::memory_order_release);- }- bool cancelled() const { return flag_ && flag_->load(std::memory_order_acquire); }+ CancelToken() : flag_(std::make_shared<std::atomic<bool>>(false)) {}+ static CancelToken Make() { return CancelToken(); }+ void Cancel() const { flag_->store(true, std::memory_order_release); }+ bool cancelled() const { return flag_->load(std::memory_order_acquire); }private:std::shared_ptr<std::atomic<bool>> flag_;diff --git a/laya-cpp/include/laya/collate.hpp b/laya-cpp/include/laya/collate.hppindex f442cd83..c3046e69 100644--- a/laya-cpp/include/laya/collate.hpp+++ b/laya-cpp/include/laya/collate.hpp@@ -27,5 +27,10 @@ struct Row {// id, and a batch that would exceed infer::kMaxTensorElements.Result<infer::Batch> Collate(const std::vector<Row>& rows, int32_t pad_id);+// How many states each session run takes when one run must not exceed infer::kMaxTensorElements in+// any tensor. Each state contributes rows_per_state rows of length[i] tokens and width[i] markers,+// taken in order; a state that alone exceeds the cap still gets a run (Collate refuses it).+std::vector<size_t> PlanRuns(size_t rows_per_state, const std::vector<int64_t>& length, const std::vector<int64_t>& width);+} // namespace collate} // namespace layadiff --git a/laya-cpp/include/laya/decode.hpp b/laya-cpp/include/laya/decode.hppindex d49a11bd..de807bfd 100644--- a/laya-cpp/include/laya/decode.hpp+++ b/laya-cpp/include/laya/decode.hpp@@ -4,8 +4,8 @@// does), softmax with the maximum subtracted, undo the option order, derive the type's fields and// round every number to four decimals as Python's round(x, 4) does.//-// Everything after the float32 quotient is double here; Python computes exp and the sum in float32.-// The choice is the first maximum, so an exact tie falls to the lower index as numpy.argmax does.+// Reported numbers are double after the float32 quotient. The choice is made on float32 probabilities+// (exp rounded to float32, numpy's pairwise float32 sum), first maximum wins, as numpy.argmax does.#include <array>#include <cstdint>#include <map>@@ -43,7 +43,7 @@ struct LanguageTemperatures {std::map<std::string, double> by_options;};-// A checkpoint's temperature tables, every value already clamped. Immutable once built.+// A checkpoint's temperature tables, every value already clamped. Immutable once built apart from SetBinning.class Temperatures {public:Temperatures() = default; // all 1.0@@ -57,9 +57,12 @@ class Temperatures {double For(schema::Type t, int64_t k, std::string_view lang, bool* override_used = nullptr) const;const std::array<double, 3>& by_type() const { return by_type_; }const std::map<std::string, double>& by_options() const { return by_options_; }- std::map<std::string, Bins> binning; // empty: no recalibration+ // Recalibration bins by bucket; empty means none.+ const std::map<std::string, Bins>& binning() const { return binning_; }+ void SetBinning(std::map<std::string, Bins> b) { binning_ = std::move(b); }private:+ std::map<std::string, Bins> binning_;std::array<double, 3> by_type_{1, 1, 1};std::map<std::string, double> by_options_;std::map<std::string, LanguageTemperatures> by_language_;diff --git a/laya-cpp/include/laya/infer.hpp b/laya-cpp/include/laya/infer.hppindex 2e7eb42c..0b04d885 100644--- a/laya-cpp/include/laya/infer.hpp+++ b/laya-cpp/include/laya/infer.hpp@@ -45,6 +45,10 @@ struct Output {std::string Check(const Batch& b) const; // empty when both hold the lengths b calls for};+// What a Session returns from Run when the cancel token fired and it stopped early.+extern const char* const kCancelled;+bool IsCancelled(const std::string& error);+// Run may be called from several threads at once. Close is idempotent and never runs concurrently// with Run (the agent guarantees it). A Session is owned through std::unique_ptr, so it is moved,// never copied.@@ -54,7 +58,8 @@ class Session {Session(const Session&) = delete;Session& operator=(const Session&) = delete;virtual ~Session() = default;- // Must not modify or retain b. Should return early with an error when cancel fires.+ // Must not modify or retain b. Should return early with kCancelled when cancel fires. No Session+ // method may throw; the agent catches anyway and reports the exception as an error.virtual Result<Output> Run(const Batch& b, const CancelToken& cancel) = 0;// Releases the runtime; "" on success, else the error. Run after Close fails.virtual std::string Close() = 0;diff --git a/laya-cpp/include/laya/schema.hpp b/laya-cpp/include/laya/schema.hppindex bb5a3fe9..30a20e48 100644--- a/laya-cpp/include/laya/schema.hpp+++ b/laya-cpp/include/laya/schema.hpp@@ -170,6 +170,7 @@ struct Usage {bool truncated = false;std::vector<std::string> truncated_questions;std::vector<CollapsedQuestion> collapsed_options;+ bool bare = false; // only input_tokens and output_tokens (a call with no questions)int64_t windows = 0;int64_t truncated_windows = 0; // with windows > 1, the "truncated" key is this countResult<std::string> ToJson() const;diff --git a/laya-cpp/src/agent.cpp b/laya-cpp/src/agent.cppindex 28f0c558..e54ddb0f 100644--- a/laya-cpp/src/agent.cpp+++ b/laya-cpp/src/agent.cpp@@ -1,6 +1,7 @@#include "laya/agent.hpp"#include <algorithm>+#include <exception>#include <fstream>#include <sstream>@@ -54,11 +55,28 @@ Result<int64_t> LengthField(const Value& obj, const char* key, int64_t dflt) {}+} // namespace++namespace {++// Closes the session on every failure path of Open; empty once the agent owns it.+struct SessionGuard {+ std::unique_ptr<infer::Session>& s;+ ~SessionGuard() {+ if (!s) return;+ try {+ s->Close();+ } catch (...) {+ }+ }+};+} // namespaceResult<std::unique_ptr<Agent>> Agent::Open(const std::string& dir, std::unique_ptr<infer::Session> session) {using R = Result<std::unique_ptr<Agent>>;if (!session) return R::Fail("agent: Open needs a session");+ SessionGuard guard{session};std::ifstream in(dir + "/" + kConfigFile, std::ios::binary);if (!in) return R::Fail(std::string("agent: ") + kConfigFile + ": cannot read " + dir + "/" + kConfigFile);std::stringstream ss;@@ -72,6 +90,7 @@ Result<std::unique_ptr<Agent>> Agent::OpenParts(std::string_view config_json, tostd::unique_ptr<infer::Session> session) {using R = Result<std::unique_ptr<Agent>>;if (!session) return R::Fail("agent: Open needs a session");+ SessionGuard guard{session};const std::string where = std::string("agent: ") + kConfigFile + ": ";auto parsed = pyvalue::Parse(config_json);if (!parsed.ok()) return R::Fail(where + parsed.error());@@ -154,10 +173,24 @@ std::string Agent::Close() {shut_ = true;closed_.store(true);cv_.wait(lock, [&] { return inflight_ == 0; });- if (!close_done_) {- close_done_ = true;- if (sess_) close_err_ = sess_->Close();+ if (close_started_) {+ cv_.wait(lock, [&] { return close_done_; });+ return close_err_;+ }+ close_started_ = true;+ lock.unlock();+ std::string err;+ try {+ if (sess_) err = sess_->Close();+ } catch (const std::exception& e) {+ err = std::string("agent: session Close threw: ") + e.what();+ } catch (...) {+ err = "agent: session Close threw";}+ lock.lock();+ close_err_ = err;+ close_done_ = true;+ cv_.notify_all();return close_err_;}@@ -195,6 +228,7 @@ Result<std::vector<schema::Prediction>> Agent::PredictBatch(const std::vector<Vaif (states.empty()) return R::Ok({});if (qs.size() == 0) {std::vector<schema::Prediction> preds(states.size());+ for (auto& p : preds) p.usage.bare = true; // what the Python ONNX agent returns with no questionsreturn R::Ok(std::move(preds));}return Chunks(states, qs, opts);@@ -245,10 +279,25 @@ Result<std::vector<schema::Prediction>> Agent::Chunks(const std::vector<Value>&}std::stable_sort(order.begin(), order.end(), [&](size_t x, size_t y) { return longest[x] < longest[y]; });}- for (size_t off = 0; off < len; off += chunk) {+ std::vector<size_t> sizes; // states per run+ if (c.batch_size > 0) {+ for (size_t off = 0; off < len; off += chunk) sizes.push_back(std::min(chunk, len - off));+ } else {+ std::vector<int64_t> length(len), width(len);+ for (size_t i = 0; i < len; i++) {+ for (const auto& s : encoded[order[i]]) {+ length[i] = std::max(length[i], int64_t(s.ids.size()));+ width[i] = std::max(width[i], int64_t(s.markers.size()));+ }+ }+ sizes = collate::PlanRuns(qs.size(), length, width);+ }+ size_t off = 0;+ for (size_t take : sizes) {if (c.cancel.cancelled()) return R::Fail(kCancelled);if (closed_.load()) return R::Fail(kClosed);- std::vector<size_t> idx(order.begin() + std::ptrdiff_t(off), order.begin() + std::ptrdiff_t(std::min(off + chunk, len)));+ std::vector<size_t> idx(order.begin() + std::ptrdiff_t(off), order.begin() + std::ptrdiff_t(off + take));+ off += take;auto preds = Run(qs, encoded, idx, c);if (!preds.ok()) return R::Fail(preds.error());for (size_t j = 0; j < idx.size(); j++) results[start + idx[j]] = std::move(preds.value()[j]);@@ -301,8 +350,15 @@ Result<std::vector<schema::Prediction>> Agent::Run(const schema::Questions& qs,std::to_string(b.markers) + " marker slots: " + why);};if (std::string bad = b.Validate(); !bad.empty()) return fail(bad);- auto out = sess_->Run(b, c.cancel);- if (!out.ok()) return c.cancel.cancelled() ? R::Fail(kCancelled) : fail(out.error());+ Result<infer::Output> out = Result<infer::Output>::Fail("");+ try {+ out = sess_->Run(b, c.cancel);+ } catch (const std::exception& e) {+ return fail(std::string("session threw: ") + e.what());+ } catch (...) {+ return fail("session threw");+ }+ if (!out.ok()) return infer::IsCancelled(out.error()) ? R::Fail(kCancelled) : fail(out.error());if (std::string bad = out.value().Check(b); !bad.empty()) return fail(bad);const infer::Output& o = out.value();@@ -322,7 +378,7 @@ Result<std::vector<schema::Prediction>> Agent::Run(const schema::Questions& qs,o.logits.begin() + std::ptrdiff_t((at + 1) * size_t(b.markers)));dr.k = int64_t(seqs[r].markers.size());dr.act = act.value()[0];- auto ans = decode::Decode(e.question, dr, c.lang, &temps_);+ auto ans = decode::Decode(e.question, dr, "", &temps_);if (!ans.ok()) return fail("question " + e.id + ": " + ans.error());preds[j].answers.push_back({e.id, std::move(ans).value()});}diff --git a/laya-cpp/src/collate.cpp b/laya-cpp/src/collate.cppindex 82d3794c..bcd74368 100644--- a/laya-cpp/src/collate.cpp+++ b/laya-cpp/src/collate.cpp@@ -56,5 +56,26 @@ Result<infer::Batch> Collate(const std::vector<Row>& rows, int32_t pad_id) {return R::Ok(std::move(b));}+std::vector<size_t> PlanRuns(size_t rows_per_state, const std::vector<int64_t>& length, const std::vector<int64_t>& width) {+ std::vector<size_t> runs;+ const int64_t per = int64_t(rows_per_state);+ size_t count = 0;+ int64_t longest = 0, widest = kMinMarkerSlots;+ for (size_t i = 0; i < length.size(); i++) {+ const int64_t l = std::max(longest, length[i]), w = std::max(widest, width[i]);+ const int64_t rows = int64_t(count + 1) * per;+ if (count > 0 && (l > infer::kMaxTensorElements / rows || w > infer::kMaxTensorElements / rows)) {+ runs.push_back(count);+ count = 0;+ longest = 0, widest = kMinMarkerSlots;+ }+ longest = std::max(longest, length[i]);+ widest = std::max(widest, width[i]);+ count++;+ }+ if (count > 0) runs.push_back(count);+ return runs;+}+} // namespace collate} // namespace layadiff --git a/laya-cpp/src/decode.cpp b/laya-cpp/src/decode.cppindex e931580d..b043c57b 100644--- a/laya-cpp/src/decode.cpp+++ b/laya-cpp/src/decode.cpp@@ -190,6 +190,29 @@ double Temperatures::For(schema::Type typ, int64_t k, std::string_view lang, boonamespace {+// numpy's float32 sum of a contiguous row (pairwise, eight accumulators from eight values up).+float PairwiseSum(const float* a, size_t n) {+ if (n < 8) {+ float r = 0;+ for (size_t i = 0; i < n; i++) r += a[i];+ return r;+ }+ if (n <= 128) {+ float r[8];+ for (size_t j = 0; j < 8; j++) r[j] = a[j];+ size_t i = 8;+ for (; i < n - (n % 8); i += 8) {+ for (size_t j = 0; j < 8; j++) r[j] += a[i + j];+ }+ float res = ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7]));+ for (; i < n; i++) res += a[i];+ return res;+ }+ size_t half = n / 2;+ half -= half % 8;+ return PairwiseSum(a, half) + PairwiseSum(a + half, n - half);+}+std::vector<schema::Probability> Probabilities(const std::vector<schema::Label>& labels, const std::vector<double>& p) {std::vector<schema::Probability> out;out.reserve(p.size());@@ -250,17 +273,26 @@ Result<schema::Answer> Decode(const schema::Question& q, const Row& row, std::stsum += p[i];}for (double& v : p) v /= sum;+ // The choice is made on float32 probabilities as numpy computes them: a gap below float32 resolution+ // is an exact tie that argmax gives to the lower index. Reported numbers stay double.+ std::vector<float> pf(z.size());+ for (size_t i = 0; i < z.size(); i++) pf[i] = float(std::exp(double(z[i] - peak)));+ const float total = PairwiseSum(pf.data(), pf.size());+ for (float& v : pf) v /= total;// The row is in slot order; everything below indexes by option.const std::vector<int>& order = q.OptionOrder();if (order.size() == p.size()) {std::vector<double> canonical(p.size());for (size_t slot = 0; slot < order.size(); slot++) canonical[size_t(order[slot])] = p[slot];p = std::move(canonical);+ std::vector<float> canonical_f(pf.size());+ for (size_t slot = 0; slot < order.size(); slot++) canonical_f[size_t(order[slot])] = pf[slot];+ pf = std::move(canonical_f);}schema::Answer a;a.type = typ;double conf = AnswerConfidence(p);- if (t != nullptr && !override_used) conf = ApplyBinning(conf, Bucket(typ, k), t->binning);+ if (t != nullptr && !override_used) conf = ApplyBinning(conf, Bucket(typ, k), t->binning());a.answer_confidence = Round4(conf);a.act_probability = Round4(double(row.act));if (typ == schema::Type::Noul) {@@ -276,8 +308,8 @@ Result<schema::Answer> Decode(const schema::Question& q, const Row& row, std::sta.probabilities = Probabilities(labels, p);if (typ == schema::Type::Choice) {size_t best = 0;- for (size_t i = 0; i < p.size(); i++) {- if (p[i] > p[best]) best = i; // strict: the first maximum wins, as numpy.argmax breaks a tie+ for (size_t i = 0; i < pf.size(); i++) {+ if (pf[i] > pf[best]) best = i; // strict: the first maximum wins, as numpy.argmax breaks a tie}a.choice = labels[best];} else {diff --git a/laya-cpp/src/infer.cpp b/laya-cpp/src/infer.cppindex eeecacd6..da17118b 100644--- a/laya-cpp/src/infer.cpp+++ b/laya-cpp/src/infer.cpp@@ -11,6 +11,9 @@ const char* const kQType = "qtype";const char* const kLogits = "logits";const char* const kActLogits = "act_logits";+const char* const kCancelled = "infer: cancelled";+bool IsCancelled(const std::string& e) { return e == kCancelled; }+namespace {std::string Shape(const char* name, size_t got, int64_t want) {return std::string("infer: ") + name + " has " + std::to_string(got) + " values, want " + std::to_string(want);@@ -57,6 +60,9 @@ std::string Batch::Validate() const {}std::string Output::Check(const Batch& b) const {+ if (b.rows < 0 || b.markers < 0 || (b.rows > 0 && (b.markers > kMaxTensorElements / b.rows || b.rows > kMaxTensorElements))) {+ return "infer: batch dimensions " + std::to_string(b.rows) + " x " + std::to_string(b.markers) + " are out of range";+ }if (int64_t(logits.size()) != b.rows * b.markers) return Shape(kLogits, logits.size(), b.rows * b.markers);if (int64_t(act_logits.size()) != b.rows * 2) return Shape(kActLogits, act_logits.size(), b.rows * 2);return "";diff --git a/laya-cpp/src/schema.cpp b/laya-cpp/src/schema.cppindex ce67e5b8..ffc04366 100644--- a/laya-cpp/src/schema.cpp+++ b/laya-cpp/src/schema.cpp@@ -897,6 +897,7 @@ Result<Answer> Answer::FromJson(const Value& v) {}Result<std::string> Usage::ToJson() const {+ if (bare) return Result<std::string>::Ok("{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) + "}");bool scanned = windows > 1;if (scanned && state_tokens == 0 && state_tokens_dropped == 0 && truncated_windows == 0 &&truncated_questions.empty() && collapsed_options.empty()) {diff --git a/laya-cpp/tests/agent_test.cpp b/laya-cpp/tests/agent_test.cppindex 67b43eed..cfe60535 100644--- a/laya-cpp/tests/agent_test.cpp+++ b/laya-cpp/tests/agent_test.cpp@@ -6,6 +6,7 @@#include <cmath>#include <fstream>#include <sstream>+#include <stdexcept>#include <thread>#include "checkpoint.hpp"@@ -103,6 +104,30 @@ TEST(Infer, ValidateCatchesEachShape) {EXPECT_NE(bad([](in::Batch& b) { b.rows = int64_t(1) << 62; }), "");}+TEST(Infer, OutputCheckDoesNotOverflowOnAHostileBatch) {+ in::Batch b;+ b.rows = int64_t(1) << 40;+ b.markers = int64_t(1) << 40;+ in::Output o;+ EXPECT_NE(o.Check(b), "");+ b.rows = -1;+ EXPECT_NE(o.Check(b), "");+}++TEST(Collate, PlanRunsSplitsOnlyAtTheTensorCap) {+ using laya::collate::PlanRuns;+ // 32769 rows of 512 tokens: one row over 2^24 elements.+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(32768, 512), std::vector<int64_t>(32768, 2)), (std::vector<size_t>{32768}));+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(32769, 512), std::vector<int64_t>(32769, 2)), (std::vector<size_t>{32768, 1}));+ // three rows per state: 10922 states fit, the 10923rd starts a new run+ EXPECT_EQ(PlanRuns(3, std::vector<int64_t>(10923, 512), std::vector<int64_t>(10923, 2)), (std::vector<size_t>{10922, 1}));+ // the marker width splits too+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(3, 4), std::vector<int64_t>(3, 6000000)), (std::vector<size_t>{2, 1}));+ // one state over the cap alone is still one run (collate refuses it)+ EXPECT_EQ(PlanRuns(1, {int64_t(1) << 25}, {2}), (std::vector<size_t>{1}));+ EXPECT_TRUE(PlanRuns(1, {}, {}).empty());+}+TEST(Infer, OutputCheck) {in::Batch b = laya::collate::Collate({Seq({1, 2}, {0, 1})}, 0).value();in::Output o;@@ -113,10 +138,11 @@ TEST(Infer, OutputCheck) {EXPECT_NE(o.Check(b), "");}-TEST(CancelToken, CopiesShareAFlagAndADefaultNeverFires) {+TEST(CancelToken, CopiesShareAFlagAndADefaultFires) {laya::CancelToken none;- none.Cancel();EXPECT_FALSE(none.cancelled());+ none.Cancel();+ EXPECT_TRUE(none.cancelled());laya::CancelToken a = laya::CancelToken::Make(), b = a;EXPECT_FALSE(b.cancelled());a.Cancel();@@ -260,7 +286,7 @@ TEST(AgentGolden, PredictPlan) {ag::CallOptions o;if (!rec.Find("batch_size")->is_null()) o.batch_size = rec.Find("batch_size")->as_int();o.sort_by_length = rec.Find("sort")->as_bool();- if (!rec.Find("lang")->is_null()) o.lang = rec.Find("lang")->as_string();+ // The checkpoints set no language temperatures, so the recorded lang changes nothing.rig.st->runs.clear();auto preds = rig.agent->PredictBatch(rec.Find("states")->items(), qs.value(), o);ASSERT_TRUE(preds.ok()) << t.where << ": " << preds.error();@@ -516,6 +542,189 @@ TEST(Agent, CloseWaitsForCallsInFlightAndNoneStartsAfter) {EXPECT_GT(ok + closed, 0);}+// ---- review round 1 ----++class ThrowingSession : public in::Session {+ public:+ ThrowingSession(bool run, bool close, int* closes) : run_(run), close_(close), closes_(closes) {}+ laya::Result<in::Output> Run(const in::Batch&, const laya::CancelToken&) override {+ if (run_) throw std::runtime_error("run blew up");+ return laya::Result<in::Output>::Fail("unreachable");+ }+ std::string Close() override {+ (*closes_)++;+ if (close_) throw std::runtime_error("close blew up");+ return "";+ }++ private:+ bool run_, close_;+ int* closes_;+};++Rig OpenWith(const char* model, std::unique_ptr<in::Session> s) {+ Rig r;+ const laya::tokenizer::Tokenizer* tok = laya_test::LoadCheckpoint(model);+ if (tok == nullptr) return r;+ auto a = ag::Agent::OpenParts(kConfig, *tok, std::move(s));+ EXPECT_TRUE(a.ok());+ if (a.ok()) r.agent = std::move(a).value();+ return r;+}++TEST(Agent, ASessionThatThrowsIsAnErrorNotATerminate) {+ int closes = 0;+ LAYA_RIG_OPEN(rig, OpenWith("english", std::make_unique<ThrowingSession>(true, true, &closes)));+ auto p = rig.agent->Predict(Value::String("alpha"), Mix());+ ASSERT_FALSE(p.ok());+ EXPECT_NE(p.error().find("run blew up"), std::string::npos) << p.error();+ const std::string first = rig.agent->Close();+ EXPECT_NE(first.find("close blew up"), std::string::npos) << first;+ EXPECT_EQ(rig.agent->Close(), first); // the second Close reports the same error+ EXPECT_EQ(closes, 1);+ rig.agent.reset(); // the destructor closes again without terminating+ EXPECT_EQ(closes, 1);+}++TEST(Agent, DestroyingWithAThrowingCloseDoesNotTerminate) {+ int closes = 0;+ {+ LAYA_RIG_OPEN(rig, OpenWith("english", std::make_unique<ThrowingSession>(false, true, &closes)));+ }+ EXPECT_EQ(closes, 1);+}++TEST(Agent, CloseReturnsTheSessionErrorToEveryCaller) {+ LAYA_RIG(rig, "english");+ rig.st->close_error = "disk gone";+ std::string a, b;+ std::thread t1([&] { a = rig.agent->Close(); }), t2([&] { b = rig.agent->Close(); });+ t1.join();+ t2.join();+ EXPECT_EQ(a, "disk gone");+ EXPECT_EQ(b, "disk gone");+ EXPECT_EQ(rig.st->closes, 1);+}++TEST(Agent, NoQuestionsUsageIsTheBareTwoCounters) {+ LAYA_RIG(rig, "english");+ auto p = rig.agent->Predict(Value::String("x"), sc::Questions());+ ASSERT_TRUE(p.ok());+ // The JSON the Python ONNX agent returns for an empty question set.+ EXPECT_EQ(p.value().ToJson().value(), R"({"model":"laya-rl-agent-onnx","answers":{},"usage":{"input_tokens":0,"output_tokens":0}})");+}++TEST(Agent, CancelledCallsReportCancelledEvenWithNothingToDo) {+ LAYA_RIG(rig, "english");+ ag::CallOptions o;+ o.cancel.Cancel(); // a default token fires too+ auto none = rig.agent->PredictBatch({}, Mix(), o);+ ASSERT_FALSE(none.ok());+ EXPECT_TRUE(ag::IsCancelled(none.error()));+ auto noq = rig.agent->PredictBatch(States(2), sc::Questions(), o);+ ASSERT_FALSE(noq.ok());+ EXPECT_TRUE(ag::IsCancelled(noq.error()));+}++TEST(Agent, ASessionThatIgnoresCancelIsStoppedBeforeTheNextRun) {+ LAYA_RIG(rig, "english");+ rig.st->ignore_cancel = true;+ ag::CallOptions o;+ o.batch_size = 2;+ o.sort_by_length = true;+ o.cancel = laya::CancelToken::Make();+ rig.st->on_run = [c = o.cancel] { c.Cancel(); };+ auto p = rig.agent->PredictBatch(States(6), Mix(), o);+ ASSERT_FALSE(p.ok());+ EXPECT_TRUE(ag::IsCancelled(p.error())) << p.error();+ EXPECT_EQ(rig.st->runs.size(), 1u);+}++// The last run of window one cancels; window two opens with a state that cannot be encoded, and the+// call must report the cancel, not that state's error.+TEST(Agent, CancelIsCheckedBeforeEncodingTheNextWindow) {+ LAYA_RIG(rig, "english");+ auto states = States(8);+ Value deep = Value::String("x");+ for (int i = 0; i < laya::pyvalue::kMaxDepth + 8; i++) deep = Value::Array({std::move(deep)}); // too deep to serialize+ states.push_back(std::move(deep));+ ag::CallOptions o;+ o.batch_size = 1;+ o.sort_by_length = true; // windows of eight states+ o.cancel = laya::CancelToken::Make();+ rig.st->ignore_cancel = true;+ ASSERT_FALSE(rig.agent->Predict(states.back(), Mix()).ok()); // it does fail to encode+ rig.st->runs.clear();+ int runs = 0;+ rig.st->on_run = [&, c = o.cancel] { // the last run of window one cancels+ if (++runs == 8) c.Cancel();+ };+ auto p = rig.agent->PredictBatch(states, Mix(), o);+ ASSERT_FALSE(p.ok());+ EXPECT_TRUE(ag::IsCancelled(p.error())) << p.error();+}++TEST(Agent, ClosedAgentRefusesEmptyAndQuestionlessCalls) {+ LAYA_RIG(rig, "english");+ rig.agent->Close();+ auto a = rig.agent->PredictBatch({}, Mix());+ ASSERT_FALSE(a.ok());+ EXPECT_TRUE(ag::IsClosed(a.error()));+ auto b = rig.agent->PredictBatch(States(2), sc::Questions());+ ASSERT_FALSE(b.ok());+ EXPECT_TRUE(ag::IsClosed(b.error()));+}++TEST(Agent, ASessionErrorSurvivesAFiredToken) {+ LAYA_RIG(rig, "english");+ rig.st->ignore_cancel = true;+ rig.st->fail = "disk on fire";+ ag::CallOptions o;+ o.cancel = laya::CancelToken::Make();+ rig.st->on_run = [c = o.cancel] { c.Cancel(); };+ auto p = rig.agent->Predict(Value::String("alpha"), Mix(), o);+ ASSERT_FALSE(p.ok());+ EXPECT_FALSE(ag::IsCancelled(p.error()));+ EXPECT_NE(p.error().find("disk on fire"), std::string::npos) << p.error();+}++// Close does not wait for a call to finish its runs: it waits for the run in flight and the call+// then fails with kClosed at the next one.+TEST(Agent, CloseCutsAnInFlightCallShortAtItsNextRun) {+ LAYA_RIG(rig, "english");+ std::mutex mu;+ std::condition_variable cv;+ bool in_run = false, release = false;+ rig.st->on_run = [&] {+ std::unique_lock<std::mutex> l(mu);+ if (in_run) return; // only the first run is gated+ in_run = true;+ cv.notify_all();+ cv.wait(l, [&] { return release; });+ };+ ag::CallOptions o;+ o.batch_size = 1;+ laya::Result<std::vector<sc::Prediction>> got = laya::Result<std::vector<sc::Prediction>>::Fail("unset");+ std::thread call([&] { got = rig.agent->PredictBatch(States(4), Mix(), o); });+ {+ std::unique_lock<std::mutex> l(mu);+ cv.wait(l, [&] { return in_run; });+ }+ std::thread closer([&] { rig.agent->Close(); });+ while (rig.agent->PredictBatch({}, Mix()).ok()) std::this_thread::yield(); // until Close has begun+ {+ std::lock_guard<std::mutex> l(mu);+ release = true;+ }+ cv.notify_all();+ call.join();+ closer.join();+ ASSERT_FALSE(got.ok());+ EXPECT_TRUE(ag::IsClosed(got.error())) << got.error();+ EXPECT_EQ(rig.st->runs.size(), 1u);+ EXPECT_EQ(rig.st->closes, 1);+}+// ---- Open ----TEST(AgentOpen, ReadsTheCheckpointDirectory) {@@ -540,7 +749,15 @@ TEST(AgentOpen, FailureDestroysTheSession) {auto a = ag::Agent::Open("/nonexistent/checkpoint", std::move(s));EXPECT_FALSE(a.ok());EXPECT_TRUE(st->destroyed);+ EXPECT_EQ(st->closes, 1); // a failed Open still closes the sessionEXPECT_FALSE(ag::Agent::Open("/x", nullptr).ok());+ auto s2 = std::make_unique<StubSession>();+ auto st2 = s2->state();+ const laya::tokenizer::Tokenizer* tok = laya_test::LoadCheckpoint("english");+ if (tok != nullptr) {+ EXPECT_FALSE(ag::Agent::OpenParts("[]", *tok, std::move(s2)).ok());+ EXPECT_EQ(st2->closes, 1);+ }}TEST(AgentOpen, RefusesABadConfig) {diff --git a/laya-cpp/tests/decode_test.cpp b/laya-cpp/tests/decode_test.cppindex 2be11603..e0d74987 100644--- a/laya-cpp/tests/decode_test.cpp+++ b/laya-cpp/tests/decode_test.cpp@@ -50,7 +50,7 @@ TEST(DecodeGolden, DecodeJson) {for (const char* name : kModels) {LAYA_GOLDEN_LIST(entry, "decode.json", name);const Value* cases = entry->Find("cases");- LAYA_EXPECT_ROWS(cases, 20);+ LAYA_EXPECT_ROWS(cases, 24);const dc::Temperatures temps = TempsOf(*entry->Find("config"));for (size_t i = 0; i < cases->size(); i++) {const Value& rec = cases->value(i);@@ -344,6 +344,18 @@ TEST(Decode, QuotientTieFallsToTheFirst) {}}+// numpy keeps exp and the sum in float32, so a logit gap below float32 resolution is an exact tie.+TEST(Decode, SubUlpGapIsAFloat32TieForTheChoice) {+ const float lo = 1e-6f, hi = std::nextafter(lo, 1.0f);+ auto t = Uniform(1);+ auto key = [&](int n, std::vector<float> v) { return dc::Decode(Pick(n), RowOf(v), "", &t).value().choice.Key(); };+ EXPECT_EQ(key(2, {lo, hi}), "o0");+ EXPECT_EQ(key(2, {hi, lo}), "o0");+ EXPECT_EQ(key(3, {lo, hi, hi}), "o0");+ EXPECT_EQ(key(3, {lo, 5.0f, hi}), "o1"); // a real gap still decides+ EXPECT_EQ(key(3, {-3.0f, lo, hi}), "o1");+}+TEST(Decode, FirstMaximumWinsAnExactTie) {auto t = Uniform(1);EXPECT_EQ(dc::Decode(Pick(3), RowOf({1, 1, 1}), "", &t).value().choice.Key(), "o0");@@ -393,16 +405,24 @@ TEST(Decode, OptionOrderIsUndone) {EXPECT_EQ(a.value().choice.Key(), "o2");}+TEST(Temperatures, SamePrimarySubtagLastWins) {+ auto js = laya::pyvalue::Parse(R"({"zh-CN":{"temperature":[2,2,2]},"zh-TW":{"temperature":[4,4,4]}})").value();+ auto t = dc::Temperatures::Of(nullptr, nullptr, &js).value();+ EXPECT_EQ(t.For(sc::Type::Choice, 2, "zh", nullptr), 4.0);+}+TEST(Decode, BinningReplacesTheConfidenceOnlyWithoutALanguageOverride) {auto js = laya::pyvalue::Parse(R"({"l":{"zh":null}})").value();auto t = dc::Temperatures::Of(nullptr, nullptr, js.Find("l")).value();- t.binning["choice:2"].values = {0.25, 0.75};+ std::map<std::string, dc::Bins> bins;+ bins["choice:2"].values = {0.25, 0.75};+ t.SetBinning(bins);auto plain = dc::Decode(Pick(2), RowOf({4, 0}), "", &t).value();EXPECT_EQ(plain.answer_confidence, 0.75); // 0.98 falls in the top of two binsauto zh = dc::Decode(Pick(2), RowOf({4, 0}), "zh", &t).value();EXPECT_NE(zh.answer_confidence, 0.75);- EXPECT_EQ(dc::ApplyBinning(1.0, "choice:2", t.binning), 0.75); // clamped into the last bin- EXPECT_EQ(dc::ApplyBinning(0.3, "absent", t.binning), 0.3);+ EXPECT_EQ(dc::ApplyBinning(1.0, "choice:2", t.binning()), 0.75); // clamped into the last bin+ EXPECT_EQ(dc::ApplyBinning(0.3, "absent", t.binning()), 0.3);}TEST(ActionProbabilities, SoftmaxAndRefusals) {diff --git a/laya-cpp/tests/stub_session.hpp b/laya-cpp/tests/stub_session.hppindex 15c0c9b2..bf0d17a2 100644--- a/laya-cpp/tests/stub_session.hpp+++ b/laya-cpp/tests/stub_session.hpp@@ -28,6 +28,8 @@ class StubSession : public laya::infer::Session {std::function<void()> on_run; // called at the start of every Runstd::function<void(laya::infer::Output*)> tamper; // may break the outputstd::string fail; // non-empty: Run fails with this+ bool ignore_cancel = false; // Run does not look at the token+ std::string close_error; // what Close returns};explicit StubSession(std::shared_ptr<State> s = std::make_shared<State>()) : st_(std::move(s)) {}~StubSession() override {@@ -45,7 +47,7 @@ class StubSession : public laya::infer::Session {hook = st_->on_run;}if (hook) hook();- if (cancel.cancelled()) return R::Fail("cancelled");+ if (!st_->ignore_cancel && cancel.cancelled()) return R::Fail(laya::infer::kCancelled);RunRecord rec;rec.rows = b.rows, rec.length = b.length, rec.markers = b.markers;laya::infer::Output out;@@ -75,7 +77,7 @@ class StubSession : public laya::infer::Session {std::lock_guard<std::mutex> l(st_->mu);st_->closes++;st_->closed = true;- return "";+ return st_->close_error;}private:diff --git a/laya-cpp/tests/tools/laya_c4_cli.cpp b/laya-cpp/tests/tools/laya_c4_cli.cppnew file mode 100644index 00000000..40e893c5--- /dev/null+++ b/laya-cpp/tests/tools/laya_c4_cli.cpp@@ -0,0 +1,74 @@+// A line protocol over laya::decode for the differential fuzz (laya-e2e/cpp-lane/fuzz-c4); bytes travel as hex.+// dec <hex question wire def> <order|-> <hex json {"logits","act","temperature","tbo"}>+// -> "err <hex>" | "ok <hex compact answer json>"+#include <iostream>+#include <sstream>++#include "laya/decode.hpp"+#include "laya/pyvalue.hpp"+#include "laya/schema.hpp"++namespace {+using laya::pyvalue::Value;+namespace dc = laya::decode;+namespace sc = laya::schema;++std::string FromHex(const std::string& h) {+ std::string out;+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));+ return out;+}+std::string Hex(const std::string& s) {+ static const char* d = "0123456789abcdef";+ std::string o;+ for (unsigned char c : s) {+ o.push_back(d[c >> 4]);+ o.push_back(d[c & 15]);+ }+ return o;+}+std::string Err(const std::string& e) { return "err " + Hex(e); }++double Num(const Value& v) { return v.kind() == Value::Kind::Int ? double(v.as_int()) : v.as_float(); }++std::string Dec(const std::string& def, const std::string& order, const std::string& arg) {+ auto wire = laya::pyvalue::Parse(FromHex(def));+ if (!wire.ok()) return Err(wire.error());+ auto q = sc::Question::FromWire(wire.value());+ if (!q.ok()) return Err(q.error());+ sc::Question question = q.value();+ if (order != "-") {+ std::vector<int> o;+ std::stringstream ss(order);+ for (std::string part; std::getline(ss, part, ',');) o.push_back(std::stoi(part));+ auto w = question.WithOptionOrder(o);+ if (!w.ok()) return Err(w.error());+ question = w.value();+ }+ auto a = laya::pyvalue::Parse(FromHex(arg));+ if (!a.ok()) return Err(a.error());+ auto temps = dc::Temperatures::Of(a.value().Find("temperature"), a.value().Find("tbo"));+ if (!temps.ok()) return Err(temps.error());+ dc::Row row;+ for (const Value& v : a.value().Find("logits")->items()) row.logits.push_back(float(Num(v)));+ row.k = int64_t(row.logits.size());+ row.act = float(Num(*a.value().Find("act")));+ auto ans = dc::Decode(question, row, "", &temps.value());+ if (!ans.ok()) return Err(ans.error());+ auto js = ans.value().ToJson();+ if (!js.ok()) return Err(js.error());+ auto back = laya::pyvalue::Parse(js.value());+ return "ok " + Hex(laya::pyvalue::Dumps(back.value()).value());+}+} // namespace++int main() {+ std::string line;+ while (std::getline(std::cin, line)) {+ std::stringstream ss(line);+ std::string cmd, def, order, arg;+ ss >> cmd >> def >> order >> arg;+ std::cout << (cmd == "dec" ? Dec(def, order, arg) : Err("unknown command")) << std::endl;+ }+ return 0;+}diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.pyindex 1b69bd5e..689678fb 100644--- a/laya-java/scripts/gen_fixtures.py+++ b/laya-java/scripts/gen_fixtures.py@@ -1768,6 +1768,9 @@ def decode_answers():def row(values):return np.asarray(values, dtype=np.float32)+ _LO = np.float32(1e-6)+ _HI = np.nextafter(_LO, np.float32(1))+# (id, question, option_order, lang, logits)cases = [("choice-2", {"t": "choice", "ins": "i", "crit": {"a": "x", "b": "y"}},@@ -1794,6 +1797,16 @@ def decode_answers():None, None, [700.0, 699.0, -700.0]),("choice-tiny-spread", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},None, None, [1.0, 1.0 + 1e-7]),+ # Adjacent float32 logits: numpy's float32 exp and sum make p an exact tie, so argmax+ # takes the first option; a double softmax would pick the larger logit.+ ("choice-ulp-tie", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},+ None, None, [float(_LO), float(_HI)]),+ ("choice-ulp-tie-reversed", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},+ None, None, [float(_HI), float(_LO)]),+ ("choice-ulp-tie-3", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2", "c": "3"}},+ None, None, [float(_LO), float(_HI), float(_HI)]),+ ("choice-ulp-tie-permuted", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},+ [1, 0], None, [float(_LO), float(_HI)]),("choice-lang-override", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},None, "zh-Hans", [2.0, 1.0]),("choice-lang-override-prefix", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},