Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cpp): decode and agent review round 1

Adeshina Lasisi140c33491199
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 4036563d..a9fd5a7b 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -42,7 +42,7 @@ concurrency:
env:
# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus
# compile_flags). Raise it with every test added.
- MIN_TESTS: "409"
+ MIN_TESTS: "423"
GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints
@@ -60,6 +60,10 @@ jobs:
matrix:
compiler: [g++, clang++]
preset: [debug, asan-ubsan]
+ include:
+ # The Agent claims thread safety; the whole suite runs under ThreadSanitizer once.
+ - compiler: clang++
+ preset: tsan
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index ff41f10f..3f8bfe42 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -59,6 +59,9 @@ if(LAYA_BUILD_TESTS)
add_executable(laya_c3_cli tests/tools/laya_c3_cli.cpp)
target_link_libraries(laya_c3_cli PRIVATE laya)
laya_target_flags(laya_c3_cli)
+ add_executable(laya_c4_cli tests/tools/laya_c4_cli.cpp)
+ target_link_libraries(laya_c4_cli PRIVATE laya)
+ laya_target_flags(laya_c4_cli)
include(GoogleTest)
# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.
diff --git a/laya-cpp/include/laya/agent.hpp b/laya-cpp/include/laya/agent.hpp
index ac30a4fe..e48df935 100644
--- a/laya-cpp/include/laya/agent.hpp
+++ b/laya-cpp/include/laya/agent.hpp
@@ -3,9 +3,12 @@
// PredictBatch states against a question set. Hooks, the confidence gate, predict_long, calibration
// and digests are later changes.
//
-// Errors are returned as Result<T>; no exception crosses this API. An Agent is safe for concurrent
-// use: any number of threads may call Predict while another calls Close, which waits for the calls
-// in flight and then closes the Session.
+// Errors are returned as Result<T>; no exception crosses this API, and an exception thrown by a
+// Session is caught and reported as an error. An Agent is safe for concurrent use: any number of
+// threads may call Predict while another calls Close.
+//
+// Deferred to later changes: per-language temperatures (CallOptions has no `lang` until then), the
+// confidence gate, predict_long, hooks, calibration and digests.
#include <atomic>
#include <condition_variable>
#include <cstdint>
@@ -46,8 +49,9 @@ struct Config {
// Per-call settings. Zero values mean the defaults.
struct CallOptions {
- std::string lang; // language tag selecting a temperature override
- int64_t batch_size = 0; // states per session run; <= 0 runs all states at once
+ // States per session run. <= 0 runs all states at once, split only where one run would exceed
+ // infer::kMaxTensorElements in any tensor. A positive value is used as given; a run over the cap fails.
+ int64_t batch_size = 0;
bool sort_by_length = false; // group similar lengths per run; results stay in input order
int64_t max_len = 0; // > 0 overrides the checkpoint's
int64_t head_max_len = 0; // > 0 overrides the checkpoint's
@@ -75,8 +79,11 @@ class Agent {
Result<std::vector<schema::Prediction>> PredictBatch(const std::vector<pyvalue::Value>& states,
const schema::Questions& qs,
const CallOptions& opts = {}) const;
- // Waits for the calls in flight, then closes the session. Idempotent; later calls fail with kClosed.
- // Returns the session's close error, "" on success.
+ // Refuses new calls at once and waits for each call in flight to return; a call with runs left is
+ // cut short with kClosed at its next run. Then closes the session exactly once, outside the lock;
+ // concurrent and later Closes return the same result. Later calls fail with kClosed. Returns the
+ // session's close error, "" on success. Must not be called from inside a Session (it would wait
+ // for itself).
std::string Close();
private:
@@ -100,6 +107,7 @@ class Agent {
mutable std::condition_variable cv_;
mutable int64_t inflight_ = 0;
bool shut_ = false;
+ bool close_started_ = false;
bool close_done_ = false;
std::string close_err_;
std::atomic<bool> closed_{false};
diff --git a/laya-cpp/include/laya/cancel.hpp b/laya-cpp/include/laya/cancel.hpp
index 18e06064..6328e4ee 100644
--- a/laya-cpp/include/laya/cancel.hpp
+++ b/laya-cpp/include/laya/cancel.hpp
@@ -1,6 +1,6 @@
#pragma once
// A cancellation token. Copies share one flag, so the caller keeps a copy and cancels it from any
-// thread while a call holds another. A default token can never be cancelled.
+// thread while a call holds another. Every token, default included, owns a flag.
#include <atomic>
#include <memory>
@@ -8,16 +8,10 @@ namespace laya {
class CancelToken {
public:
- CancelToken() = default;
- static CancelToken Make() {
- CancelToken t;
- t.flag_ = std::make_shared<std::atomic<bool>>(false);
- return t;
- }
- void Cancel() const {
- if (flag_) flag_->store(true, std::memory_order_release);
- }
- bool cancelled() const { return flag_ && flag_->load(std::memory_order_acquire); }
+ CancelToken() : flag_(std::make_shared<std::atomic<bool>>(false)) {}
+ static CancelToken Make() { return CancelToken(); }
+ void Cancel() const { flag_->store(true, std::memory_order_release); }
+ bool cancelled() const { return flag_->load(std::memory_order_acquire); }
private:
std::shared_ptr<std::atomic<bool>> flag_;
diff --git a/laya-cpp/include/laya/collate.hpp b/laya-cpp/include/laya/collate.hpp
index f442cd83..c3046e69 100644
--- a/laya-cpp/include/laya/collate.hpp
+++ b/laya-cpp/include/laya/collate.hpp
@@ -27,5 +27,10 @@ struct Row {
// id, and a batch that would exceed infer::kMaxTensorElements.
Result<infer::Batch> Collate(const std::vector<Row>& rows, int32_t pad_id);
+// How many states each session run takes when one run must not exceed infer::kMaxTensorElements in
+// any tensor. Each state contributes rows_per_state rows of length[i] tokens and width[i] markers,
+// taken in order; a state that alone exceeds the cap still gets a run (Collate refuses it).
+std::vector<size_t> PlanRuns(size_t rows_per_state, const std::vector<int64_t>& length, const std::vector<int64_t>& width);
+
} // namespace collate
} // namespace laya
diff --git a/laya-cpp/include/laya/decode.hpp b/laya-cpp/include/laya/decode.hpp
index d49a11bd..de807bfd 100644
--- a/laya-cpp/include/laya/decode.hpp
+++ b/laya-cpp/include/laya/decode.hpp
@@ -4,8 +4,8 @@
// does), softmax with the maximum subtracted, undo the option order, derive the type's fields and
// round every number to four decimals as Python's round(x, 4) does.
//
-// Everything after the float32 quotient is double here; Python computes exp and the sum in float32.
-// The choice is the first maximum, so an exact tie falls to the lower index as numpy.argmax does.
+// Reported numbers are double after the float32 quotient. The choice is made on float32 probabilities
+// (exp rounded to float32, numpy's pairwise float32 sum), first maximum wins, as numpy.argmax does.
#include <array>
#include <cstdint>
#include <map>
@@ -43,7 +43,7 @@ struct LanguageTemperatures {
std::map<std::string, double> by_options;
};
-// A checkpoint's temperature tables, every value already clamped. Immutable once built.
+// A checkpoint's temperature tables, every value already clamped. Immutable once built apart from SetBinning.
class Temperatures {
public:
Temperatures() = default; // all 1.0
@@ -57,9 +57,12 @@ class Temperatures {
double For(schema::Type t, int64_t k, std::string_view lang, bool* override_used = nullptr) const;
const std::array<double, 3>& by_type() const { return by_type_; }
const std::map<std::string, double>& by_options() const { return by_options_; }
- std::map<std::string, Bins> binning; // empty: no recalibration
+ // Recalibration bins by bucket; empty means none.
+ const std::map<std::string, Bins>& binning() const { return binning_; }
+ void SetBinning(std::map<std::string, Bins> b) { binning_ = std::move(b); }
private:
+ std::map<std::string, Bins> binning_;
std::array<double, 3> by_type_{1, 1, 1};
std::map<std::string, double> by_options_;
std::map<std::string, LanguageTemperatures> by_language_;
diff --git a/laya-cpp/include/laya/infer.hpp b/laya-cpp/include/laya/infer.hpp
index 2e7eb42c..0b04d885 100644
--- a/laya-cpp/include/laya/infer.hpp
+++ b/laya-cpp/include/laya/infer.hpp
@@ -45,6 +45,10 @@ struct Output {
std::string Check(const Batch& b) const; // empty when both hold the lengths b calls for
};
+// What a Session returns from Run when the cancel token fired and it stopped early.
+extern const char* const kCancelled;
+bool IsCancelled(const std::string& error);
+
// Run may be called from several threads at once. Close is idempotent and never runs concurrently
// with Run (the agent guarantees it). A Session is owned through std::unique_ptr, so it is moved,
// never copied.
@@ -54,7 +58,8 @@ class Session {
Session(const Session&) = delete;
Session& operator=(const Session&) = delete;
virtual ~Session() = default;
- // Must not modify or retain b. Should return early with an error when cancel fires.
+ // Must not modify or retain b. Should return early with kCancelled when cancel fires. No Session
+ // method may throw; the agent catches anyway and reports the exception as an error.
virtual Result<Output> Run(const Batch& b, const CancelToken& cancel) = 0;
// Releases the runtime; "" on success, else the error. Run after Close fails.
virtual std::string Close() = 0;
diff --git a/laya-cpp/include/laya/schema.hpp b/laya-cpp/include/laya/schema.hpp
index bb5a3fe9..30a20e48 100644
--- a/laya-cpp/include/laya/schema.hpp
+++ b/laya-cpp/include/laya/schema.hpp
@@ -170,6 +170,7 @@ struct Usage {
bool truncated = false;
std::vector<std::string> truncated_questions;
std::vector<CollapsedQuestion> collapsed_options;
+ bool bare = false; // only input_tokens and output_tokens (a call with no questions)
int64_t windows = 0;
int64_t truncated_windows = 0; // with windows > 1, the "truncated" key is this count
Result<std::string> ToJson() const;
diff --git a/laya-cpp/src/agent.cpp b/laya-cpp/src/agent.cpp
index 28f0c558..e54ddb0f 100644
--- a/laya-cpp/src/agent.cpp
+++ b/laya-cpp/src/agent.cpp
@@ -1,6 +1,7 @@
#include "laya/agent.hpp"
#include <algorithm>
+#include <exception>
#include <fstream>
#include <sstream>
@@ -54,11 +55,28 @@ Result<int64_t> LengthField(const Value& obj, const char* key, int64_t dflt) {
}
+} // namespace
+
+namespace {
+
+// Closes the session on every failure path of Open; empty once the agent owns it.
+struct SessionGuard {
+ std::unique_ptr<infer::Session>& s;
+ ~SessionGuard() {
+ if (!s) return;
+ try {
+ s->Close();
+ } catch (...) {
+ }
+ }
+};
+
} // namespace
Result<std::unique_ptr<Agent>> Agent::Open(const std::string& dir, std::unique_ptr<infer::Session> session) {
using R = Result<std::unique_ptr<Agent>>;
if (!session) return R::Fail("agent: Open needs a session");
+ SessionGuard guard{session};
std::ifstream in(dir + "/" + kConfigFile, std::ios::binary);
if (!in) return R::Fail(std::string("agent: ") + kConfigFile + ": cannot read " + dir + "/" + kConfigFile);
std::stringstream ss;
@@ -72,6 +90,7 @@ Result<std::unique_ptr<Agent>> Agent::OpenParts(std::string_view config_json, to
std::unique_ptr<infer::Session> session) {
using R = Result<std::unique_ptr<Agent>>;
if (!session) return R::Fail("agent: Open needs a session");
+ SessionGuard guard{session};
const std::string where = std::string("agent: ") + kConfigFile + ": ";
auto parsed = pyvalue::Parse(config_json);
if (!parsed.ok()) return R::Fail(where + parsed.error());
@@ -154,10 +173,24 @@ std::string Agent::Close() {
shut_ = true;
closed_.store(true);
cv_.wait(lock, [&] { return inflight_ == 0; });
- if (!close_done_) {
- close_done_ = true;
- if (sess_) close_err_ = sess_->Close();
+ if (close_started_) {
+ cv_.wait(lock, [&] { return close_done_; });
+ return close_err_;
+ }
+ close_started_ = true;
+ lock.unlock();
+ std::string err;
+ try {
+ if (sess_) err = sess_->Close();
+ } catch (const std::exception& e) {
+ err = std::string("agent: session Close threw: ") + e.what();
+ } catch (...) {
+ err = "agent: session Close threw";
}
+ lock.lock();
+ close_err_ = err;
+ close_done_ = true;
+ cv_.notify_all();
return close_err_;
}
@@ -195,6 +228,7 @@ Result<std::vector<schema::Prediction>> Agent::PredictBatch(const std::vector<Va
if (states.empty()) return R::Ok({});
if (qs.size() == 0) {
std::vector<schema::Prediction> preds(states.size());
+ for (auto& p : preds) p.usage.bare = true; // what the Python ONNX agent returns with no questions
return R::Ok(std::move(preds));
}
return Chunks(states, qs, opts);
@@ -245,10 +279,25 @@ Result<std::vector<schema::Prediction>> Agent::Chunks(const std::vector<Value>&
}
std::stable_sort(order.begin(), order.end(), [&](size_t x, size_t y) { return longest[x] < longest[y]; });
}
- for (size_t off = 0; off < len; off += chunk) {
+ std::vector<size_t> sizes; // states per run
+ if (c.batch_size > 0) {
+ for (size_t off = 0; off < len; off += chunk) sizes.push_back(std::min(chunk, len - off));
+ } else {
+ std::vector<int64_t> length(len), width(len);
+ for (size_t i = 0; i < len; i++) {
+ for (const auto& s : encoded[order[i]]) {
+ length[i] = std::max(length[i], int64_t(s.ids.size()));
+ width[i] = std::max(width[i], int64_t(s.markers.size()));
+ }
+ }
+ sizes = collate::PlanRuns(qs.size(), length, width);
+ }
+ size_t off = 0;
+ for (size_t take : sizes) {
if (c.cancel.cancelled()) return R::Fail(kCancelled);
if (closed_.load()) return R::Fail(kClosed);
- std::vector<size_t> idx(order.begin() + std::ptrdiff_t(off), order.begin() + std::ptrdiff_t(std::min(off + chunk, len)));
+ std::vector<size_t> idx(order.begin() + std::ptrdiff_t(off), order.begin() + std::ptrdiff_t(off + take));
+ off += take;
auto preds = Run(qs, encoded, idx, c);
if (!preds.ok()) return R::Fail(preds.error());
for (size_t j = 0; j < idx.size(); j++) results[start + idx[j]] = std::move(preds.value()[j]);
@@ -301,8 +350,15 @@ Result<std::vector<schema::Prediction>> Agent::Run(const schema::Questions& qs,
std::to_string(b.markers) + " marker slots: " + why);
};
if (std::string bad = b.Validate(); !bad.empty()) return fail(bad);
- auto out = sess_->Run(b, c.cancel);
- if (!out.ok()) return c.cancel.cancelled() ? R::Fail(kCancelled) : fail(out.error());
+ Result<infer::Output> out = Result<infer::Output>::Fail("");
+ try {
+ out = sess_->Run(b, c.cancel);
+ } catch (const std::exception& e) {
+ return fail(std::string("session threw: ") + e.what());
+ } catch (...) {
+ return fail("session threw");
+ }
+ if (!out.ok()) return infer::IsCancelled(out.error()) ? R::Fail(kCancelled) : fail(out.error());
if (std::string bad = out.value().Check(b); !bad.empty()) return fail(bad);
const infer::Output& o = out.value();
@@ -322,7 +378,7 @@ Result<std::vector<schema::Prediction>> Agent::Run(const schema::Questions& qs,
o.logits.begin() + std::ptrdiff_t((at + 1) * size_t(b.markers)));
dr.k = int64_t(seqs[r].markers.size());
dr.act = act.value()[0];
- auto ans = decode::Decode(e.question, dr, c.lang, &temps_);
+ auto ans = decode::Decode(e.question, dr, "", &temps_);
if (!ans.ok()) return fail("question " + e.id + ": " + ans.error());
preds[j].answers.push_back({e.id, std::move(ans).value()});
}
diff --git a/laya-cpp/src/collate.cpp b/laya-cpp/src/collate.cpp
index 82d3794c..bcd74368 100644
--- a/laya-cpp/src/collate.cpp
+++ b/laya-cpp/src/collate.cpp
@@ -56,5 +56,26 @@ Result<infer::Batch> Collate(const std::vector<Row>& rows, int32_t pad_id) {
return R::Ok(std::move(b));
}
+std::vector<size_t> PlanRuns(size_t rows_per_state, const std::vector<int64_t>& length, const std::vector<int64_t>& width) {
+ std::vector<size_t> runs;
+ const int64_t per = int64_t(rows_per_state);
+ size_t count = 0;
+ int64_t longest = 0, widest = kMinMarkerSlots;
+ for (size_t i = 0; i < length.size(); i++) {
+ const int64_t l = std::max(longest, length[i]), w = std::max(widest, width[i]);
+ const int64_t rows = int64_t(count + 1) * per;
+ if (count > 0 && (l > infer::kMaxTensorElements / rows || w > infer::kMaxTensorElements / rows)) {
+ runs.push_back(count);
+ count = 0;
+ longest = 0, widest = kMinMarkerSlots;
+ }
+ longest = std::max(longest, length[i]);
+ widest = std::max(widest, width[i]);
+ count++;
+ }
+ if (count > 0) runs.push_back(count);
+ return runs;
+}
+
} // namespace collate
} // namespace laya
diff --git a/laya-cpp/src/decode.cpp b/laya-cpp/src/decode.cpp
index e931580d..b043c57b 100644
--- a/laya-cpp/src/decode.cpp
+++ b/laya-cpp/src/decode.cpp
@@ -190,6 +190,29 @@ double Temperatures::For(schema::Type typ, int64_t k, std::string_view lang, boo
namespace {
+// numpy's float32 sum of a contiguous row (pairwise, eight accumulators from eight values up).
+float PairwiseSum(const float* a, size_t n) {
+ if (n < 8) {
+ float r = 0;
+ for (size_t i = 0; i < n; i++) r += a[i];
+ return r;
+ }
+ if (n <= 128) {
+ float r[8];
+ for (size_t j = 0; j < 8; j++) r[j] = a[j];
+ size_t i = 8;
+ for (; i < n - (n % 8); i += 8) {
+ for (size_t j = 0; j < 8; j++) r[j] += a[i + j];
+ }
+ float res = ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7]));
+ for (; i < n; i++) res += a[i];
+ return res;
+ }
+ size_t half = n / 2;
+ half -= half % 8;
+ return PairwiseSum(a, half) + PairwiseSum(a + half, n - half);
+}
+
std::vector<schema::Probability> Probabilities(const std::vector<schema::Label>& labels, const std::vector<double>& p) {
std::vector<schema::Probability> out;
out.reserve(p.size());
@@ -250,17 +273,26 @@ Result<schema::Answer> Decode(const schema::Question& q, const Row& row, std::st
sum += p[i];
}
for (double& v : p) v /= sum;
+ // The choice is made on float32 probabilities as numpy computes them: a gap below float32 resolution
+ // is an exact tie that argmax gives to the lower index. Reported numbers stay double.
+ std::vector<float> pf(z.size());
+ for (size_t i = 0; i < z.size(); i++) pf[i] = float(std::exp(double(z[i] - peak)));
+ const float total = PairwiseSum(pf.data(), pf.size());
+ for (float& v : pf) v /= total;
// The row is in slot order; everything below indexes by option.
const std::vector<int>& order = q.OptionOrder();
if (order.size() == p.size()) {
std::vector<double> canonical(p.size());
for (size_t slot = 0; slot < order.size(); slot++) canonical[size_t(order[slot])] = p[slot];
p = std::move(canonical);
+ std::vector<float> canonical_f(pf.size());
+ for (size_t slot = 0; slot < order.size(); slot++) canonical_f[size_t(order[slot])] = pf[slot];
+ pf = std::move(canonical_f);
}
schema::Answer a;
a.type = typ;
double conf = AnswerConfidence(p);
- if (t != nullptr && !override_used) conf = ApplyBinning(conf, Bucket(typ, k), t->binning);
+ if (t != nullptr && !override_used) conf = ApplyBinning(conf, Bucket(typ, k), t->binning());
a.answer_confidence = Round4(conf);
a.act_probability = Round4(double(row.act));
if (typ == schema::Type::Noul) {
@@ -276,8 +308,8 @@ Result<schema::Answer> Decode(const schema::Question& q, const Row& row, std::st
a.probabilities = Probabilities(labels, p);
if (typ == schema::Type::Choice) {
size_t best = 0;
- for (size_t i = 0; i < p.size(); i++) {
- if (p[i] > p[best]) best = i; // strict: the first maximum wins, as numpy.argmax breaks a tie
+ for (size_t i = 0; i < pf.size(); i++) {
+ if (pf[i] > pf[best]) best = i; // strict: the first maximum wins, as numpy.argmax breaks a tie
}
a.choice = labels[best];
} else {
diff --git a/laya-cpp/src/infer.cpp b/laya-cpp/src/infer.cpp
index eeecacd6..da17118b 100644
--- a/laya-cpp/src/infer.cpp
+++ b/laya-cpp/src/infer.cpp
@@ -11,6 +11,9 @@ const char* const kQType = "qtype";
const char* const kLogits = "logits";
const char* const kActLogits = "act_logits";
+const char* const kCancelled = "infer: cancelled";
+bool IsCancelled(const std::string& e) { return e == kCancelled; }
+
namespace {
std::string Shape(const char* name, size_t got, int64_t want) {
return std::string("infer: ") + name + " has " + std::to_string(got) + " values, want " + std::to_string(want);
@@ -57,6 +60,9 @@ std::string Batch::Validate() const {
}
std::string Output::Check(const Batch& b) const {
+ if (b.rows < 0 || b.markers < 0 || (b.rows > 0 && (b.markers > kMaxTensorElements / b.rows || b.rows > kMaxTensorElements))) {
+ return "infer: batch dimensions " + std::to_string(b.rows) + " x " + std::to_string(b.markers) + " are out of range";
+ }
if (int64_t(logits.size()) != b.rows * b.markers) return Shape(kLogits, logits.size(), b.rows * b.markers);
if (int64_t(act_logits.size()) != b.rows * 2) return Shape(kActLogits, act_logits.size(), b.rows * 2);
return "";
diff --git a/laya-cpp/src/schema.cpp b/laya-cpp/src/schema.cpp
index ce67e5b8..ffc04366 100644
--- a/laya-cpp/src/schema.cpp
+++ b/laya-cpp/src/schema.cpp
@@ -897,6 +897,7 @@ Result<Answer> Answer::FromJson(const Value& v) {
}
Result<std::string> Usage::ToJson() const {
+ if (bare) return Result<std::string>::Ok("{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) + "}");
bool scanned = windows > 1;
if (scanned && state_tokens == 0 && state_tokens_dropped == 0 && truncated_windows == 0 &&
truncated_questions.empty() && collapsed_options.empty()) {
diff --git a/laya-cpp/tests/agent_test.cpp b/laya-cpp/tests/agent_test.cpp
index 67b43eed..cfe60535 100644
--- a/laya-cpp/tests/agent_test.cpp
+++ b/laya-cpp/tests/agent_test.cpp
@@ -6,6 +6,7 @@
#include <cmath>
#include <fstream>
#include <sstream>
+#include <stdexcept>
#include <thread>
#include "checkpoint.hpp"
@@ -103,6 +104,30 @@ TEST(Infer, ValidateCatchesEachShape) {
EXPECT_NE(bad([](in::Batch& b) { b.rows = int64_t(1) << 62; }), "");
}
+TEST(Infer, OutputCheckDoesNotOverflowOnAHostileBatch) {
+ in::Batch b;
+ b.rows = int64_t(1) << 40;
+ b.markers = int64_t(1) << 40;
+ in::Output o;
+ EXPECT_NE(o.Check(b), "");
+ b.rows = -1;
+ EXPECT_NE(o.Check(b), "");
+}
+
+TEST(Collate, PlanRunsSplitsOnlyAtTheTensorCap) {
+ using laya::collate::PlanRuns;
+ // 32769 rows of 512 tokens: one row over 2^24 elements.
+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(32768, 512), std::vector<int64_t>(32768, 2)), (std::vector<size_t>{32768}));
+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(32769, 512), std::vector<int64_t>(32769, 2)), (std::vector<size_t>{32768, 1}));
+ // three rows per state: 10922 states fit, the 10923rd starts a new run
+ EXPECT_EQ(PlanRuns(3, std::vector<int64_t>(10923, 512), std::vector<int64_t>(10923, 2)), (std::vector<size_t>{10922, 1}));
+ // the marker width splits too
+ EXPECT_EQ(PlanRuns(1, std::vector<int64_t>(3, 4), std::vector<int64_t>(3, 6000000)), (std::vector<size_t>{2, 1}));
+ // one state over the cap alone is still one run (collate refuses it)
+ EXPECT_EQ(PlanRuns(1, {int64_t(1) << 25}, {2}), (std::vector<size_t>{1}));
+ EXPECT_TRUE(PlanRuns(1, {}, {}).empty());
+}
+
TEST(Infer, OutputCheck) {
in::Batch b = laya::collate::Collate({Seq({1, 2}, {0, 1})}, 0).value();
in::Output o;
@@ -113,10 +138,11 @@ TEST(Infer, OutputCheck) {
EXPECT_NE(o.Check(b), "");
}
-TEST(CancelToken, CopiesShareAFlagAndADefaultNeverFires) {
+TEST(CancelToken, CopiesShareAFlagAndADefaultFires) {
laya::CancelToken none;
- none.Cancel();
EXPECT_FALSE(none.cancelled());
+ none.Cancel();
+ EXPECT_TRUE(none.cancelled());
laya::CancelToken a = laya::CancelToken::Make(), b = a;
EXPECT_FALSE(b.cancelled());
a.Cancel();
@@ -260,7 +286,7 @@ TEST(AgentGolden, PredictPlan) {
ag::CallOptions o;
if (!rec.Find("batch_size")->is_null()) o.batch_size = rec.Find("batch_size")->as_int();
o.sort_by_length = rec.Find("sort")->as_bool();
- if (!rec.Find("lang")->is_null()) o.lang = rec.Find("lang")->as_string();
+ // The checkpoints set no language temperatures, so the recorded lang changes nothing.
rig.st->runs.clear();
auto preds = rig.agent->PredictBatch(rec.Find("states")->items(), qs.value(), o);
ASSERT_TRUE(preds.ok()) << t.where << ": " << preds.error();
@@ -516,6 +542,189 @@ TEST(Agent, CloseWaitsForCallsInFlightAndNoneStartsAfter) {
EXPECT_GT(ok + closed, 0);
}
+// ---- review round 1 ----
+
+class ThrowingSession : public in::Session {
+ public:
+ ThrowingSession(bool run, bool close, int* closes) : run_(run), close_(close), closes_(closes) {}
+ laya::Result<in::Output> Run(const in::Batch&, const laya::CancelToken&) override {
+ if (run_) throw std::runtime_error("run blew up");
+ return laya::Result<in::Output>::Fail("unreachable");
+ }
+ std::string Close() override {
+ (*closes_)++;
+ if (close_) throw std::runtime_error("close blew up");
+ return "";
+ }
+
+ private:
+ bool run_, close_;
+ int* closes_;
+};
+
+Rig OpenWith(const char* model, std::unique_ptr<in::Session> s) {
+ Rig r;
+ const laya::tokenizer::Tokenizer* tok = laya_test::LoadCheckpoint(model);
+ if (tok == nullptr) return r;
+ auto a = ag::Agent::OpenParts(kConfig, *tok, std::move(s));
+ EXPECT_TRUE(a.ok());
+ if (a.ok()) r.agent = std::move(a).value();
+ return r;
+}
+
+TEST(Agent, ASessionThatThrowsIsAnErrorNotATerminate) {
+ int closes = 0;
+ LAYA_RIG_OPEN(rig, OpenWith("english", std::make_unique<ThrowingSession>(true, true, &closes)));
+ auto p = rig.agent->Predict(Value::String("alpha"), Mix());
+ ASSERT_FALSE(p.ok());
+ EXPECT_NE(p.error().find("run blew up"), std::string::npos) << p.error();
+ const std::string first = rig.agent->Close();
+ EXPECT_NE(first.find("close blew up"), std::string::npos) << first;
+ EXPECT_EQ(rig.agent->Close(), first); // the second Close reports the same error
+ EXPECT_EQ(closes, 1);
+ rig.agent.reset(); // the destructor closes again without terminating
+ EXPECT_EQ(closes, 1);
+}
+
+TEST(Agent, DestroyingWithAThrowingCloseDoesNotTerminate) {
+ int closes = 0;
+ {
+ LAYA_RIG_OPEN(rig, OpenWith("english", std::make_unique<ThrowingSession>(false, true, &closes)));
+ }
+ EXPECT_EQ(closes, 1);
+}
+
+TEST(Agent, CloseReturnsTheSessionErrorToEveryCaller) {
+ LAYA_RIG(rig, "english");
+ rig.st->close_error = "disk gone";
+ std::string a, b;
+ std::thread t1([&] { a = rig.agent->Close(); }), t2([&] { b = rig.agent->Close(); });
+ t1.join();
+ t2.join();
+ EXPECT_EQ(a, "disk gone");
+ EXPECT_EQ(b, "disk gone");
+ EXPECT_EQ(rig.st->closes, 1);
+}
+
+TEST(Agent, NoQuestionsUsageIsTheBareTwoCounters) {
+ LAYA_RIG(rig, "english");
+ auto p = rig.agent->Predict(Value::String("x"), sc::Questions());
+ ASSERT_TRUE(p.ok());
+ // The JSON the Python ONNX agent returns for an empty question set.
+ EXPECT_EQ(p.value().ToJson().value(), R"({"model":"laya-rl-agent-onnx","answers":{},"usage":{"input_tokens":0,"output_tokens":0}})");
+}
+
+TEST(Agent, CancelledCallsReportCancelledEvenWithNothingToDo) {
+ LAYA_RIG(rig, "english");
+ ag::CallOptions o;
+ o.cancel.Cancel(); // a default token fires too
+ auto none = rig.agent->PredictBatch({}, Mix(), o);
+ ASSERT_FALSE(none.ok());
+ EXPECT_TRUE(ag::IsCancelled(none.error()));
+ auto noq = rig.agent->PredictBatch(States(2), sc::Questions(), o);
+ ASSERT_FALSE(noq.ok());
+ EXPECT_TRUE(ag::IsCancelled(noq.error()));
+}
+
+TEST(Agent, ASessionThatIgnoresCancelIsStoppedBeforeTheNextRun) {
+ LAYA_RIG(rig, "english");
+ rig.st->ignore_cancel = true;
+ ag::CallOptions o;
+ o.batch_size = 2;
+ o.sort_by_length = true;
+ o.cancel = laya::CancelToken::Make();
+ rig.st->on_run = [c = o.cancel] { c.Cancel(); };
+ auto p = rig.agent->PredictBatch(States(6), Mix(), o);
+ ASSERT_FALSE(p.ok());
+ EXPECT_TRUE(ag::IsCancelled(p.error())) << p.error();
+ EXPECT_EQ(rig.st->runs.size(), 1u);
+}
+
+// The last run of window one cancels; window two opens with a state that cannot be encoded, and the
+// call must report the cancel, not that state's error.
+TEST(Agent, CancelIsCheckedBeforeEncodingTheNextWindow) {
+ LAYA_RIG(rig, "english");
+ auto states = States(8);
+ Value deep = Value::String("x");
+ for (int i = 0; i < laya::pyvalue::kMaxDepth + 8; i++) deep = Value::Array({std::move(deep)}); // too deep to serialize
+ states.push_back(std::move(deep));
+ ag::CallOptions o;
+ o.batch_size = 1;
+ o.sort_by_length = true; // windows of eight states
+ o.cancel = laya::CancelToken::Make();
+ rig.st->ignore_cancel = true;
+ ASSERT_FALSE(rig.agent->Predict(states.back(), Mix()).ok()); // it does fail to encode
+ rig.st->runs.clear();
+ int runs = 0;
+ rig.st->on_run = [&, c = o.cancel] { // the last run of window one cancels
+ if (++runs == 8) c.Cancel();
+ };
+ auto p = rig.agent->PredictBatch(states, Mix(), o);
+ ASSERT_FALSE(p.ok());
+ EXPECT_TRUE(ag::IsCancelled(p.error())) << p.error();
+}
+
+TEST(Agent, ClosedAgentRefusesEmptyAndQuestionlessCalls) {
+ LAYA_RIG(rig, "english");
+ rig.agent->Close();
+ auto a = rig.agent->PredictBatch({}, Mix());
+ ASSERT_FALSE(a.ok());
+ EXPECT_TRUE(ag::IsClosed(a.error()));
+ auto b = rig.agent->PredictBatch(States(2), sc::Questions());
+ ASSERT_FALSE(b.ok());
+ EXPECT_TRUE(ag::IsClosed(b.error()));
+}
+
+TEST(Agent, ASessionErrorSurvivesAFiredToken) {
+ LAYA_RIG(rig, "english");
+ rig.st->ignore_cancel = true;
+ rig.st->fail = "disk on fire";
+ ag::CallOptions o;
+ o.cancel = laya::CancelToken::Make();
+ rig.st->on_run = [c = o.cancel] { c.Cancel(); };
+ auto p = rig.agent->Predict(Value::String("alpha"), Mix(), o);
+ ASSERT_FALSE(p.ok());
+ EXPECT_FALSE(ag::IsCancelled(p.error()));
+ EXPECT_NE(p.error().find("disk on fire"), std::string::npos) << p.error();
+}
+
+// Close does not wait for a call to finish its runs: it waits for the run in flight and the call
+// then fails with kClosed at the next one.
+TEST(Agent, CloseCutsAnInFlightCallShortAtItsNextRun) {
+ LAYA_RIG(rig, "english");
+ std::mutex mu;
+ std::condition_variable cv;
+ bool in_run = false, release = false;
+ rig.st->on_run = [&] {
+ std::unique_lock<std::mutex> l(mu);
+ if (in_run) return; // only the first run is gated
+ in_run = true;
+ cv.notify_all();
+ cv.wait(l, [&] { return release; });
+ };
+ ag::CallOptions o;
+ o.batch_size = 1;
+ laya::Result<std::vector<sc::Prediction>> got = laya::Result<std::vector<sc::Prediction>>::Fail("unset");
+ std::thread call([&] { got = rig.agent->PredictBatch(States(4), Mix(), o); });
+ {
+ std::unique_lock<std::mutex> l(mu);
+ cv.wait(l, [&] { return in_run; });
+ }
+ std::thread closer([&] { rig.agent->Close(); });
+ while (rig.agent->PredictBatch({}, Mix()).ok()) std::this_thread::yield(); // until Close has begun
+ {
+ std::lock_guard<std::mutex> l(mu);
+ release = true;
+ }
+ cv.notify_all();
+ call.join();
+ closer.join();
+ ASSERT_FALSE(got.ok());
+ EXPECT_TRUE(ag::IsClosed(got.error())) << got.error();
+ EXPECT_EQ(rig.st->runs.size(), 1u);
+ EXPECT_EQ(rig.st->closes, 1);
+}
+
// ---- Open ----
TEST(AgentOpen, ReadsTheCheckpointDirectory) {
@@ -540,7 +749,15 @@ TEST(AgentOpen, FailureDestroysTheSession) {
auto a = ag::Agent::Open("/nonexistent/checkpoint", std::move(s));
EXPECT_FALSE(a.ok());
EXPECT_TRUE(st->destroyed);
+ EXPECT_EQ(st->closes, 1); // a failed Open still closes the session
EXPECT_FALSE(ag::Agent::Open("/x", nullptr).ok());
+ auto s2 = std::make_unique<StubSession>();
+ auto st2 = s2->state();
+ const laya::tokenizer::Tokenizer* tok = laya_test::LoadCheckpoint("english");
+ if (tok != nullptr) {
+ EXPECT_FALSE(ag::Agent::OpenParts("[]", *tok, std::move(s2)).ok());
+ EXPECT_EQ(st2->closes, 1);
+ }
}
TEST(AgentOpen, RefusesABadConfig) {
diff --git a/laya-cpp/tests/decode_test.cpp b/laya-cpp/tests/decode_test.cpp
index 2be11603..e0d74987 100644
--- a/laya-cpp/tests/decode_test.cpp
+++ b/laya-cpp/tests/decode_test.cpp
@@ -50,7 +50,7 @@ TEST(DecodeGolden, DecodeJson) {
for (const char* name : kModels) {
LAYA_GOLDEN_LIST(entry, "decode.json", name);
const Value* cases = entry->Find("cases");
- LAYA_EXPECT_ROWS(cases, 20);
+ LAYA_EXPECT_ROWS(cases, 24);
const dc::Temperatures temps = TempsOf(*entry->Find("config"));
for (size_t i = 0; i < cases->size(); i++) {
const Value& rec = cases->value(i);
@@ -344,6 +344,18 @@ TEST(Decode, QuotientTieFallsToTheFirst) {
}
}
+// numpy keeps exp and the sum in float32, so a logit gap below float32 resolution is an exact tie.
+TEST(Decode, SubUlpGapIsAFloat32TieForTheChoice) {
+ const float lo = 1e-6f, hi = std::nextafter(lo, 1.0f);
+ auto t = Uniform(1);
+ auto key = [&](int n, std::vector<float> v) { return dc::Decode(Pick(n), RowOf(v), "", &t).value().choice.Key(); };
+ EXPECT_EQ(key(2, {lo, hi}), "o0");
+ EXPECT_EQ(key(2, {hi, lo}), "o0");
+ EXPECT_EQ(key(3, {lo, hi, hi}), "o0");
+ EXPECT_EQ(key(3, {lo, 5.0f, hi}), "o1"); // a real gap still decides
+ EXPECT_EQ(key(3, {-3.0f, lo, hi}), "o1");
+}
+
TEST(Decode, FirstMaximumWinsAnExactTie) {
auto t = Uniform(1);
EXPECT_EQ(dc::Decode(Pick(3), RowOf({1, 1, 1}), "", &t).value().choice.Key(), "o0");
@@ -393,16 +405,24 @@ TEST(Decode, OptionOrderIsUndone) {
EXPECT_EQ(a.value().choice.Key(), "o2");
}
+TEST(Temperatures, SamePrimarySubtagLastWins) {
+ auto js = laya::pyvalue::Parse(R"({"zh-CN":{"temperature":[2,2,2]},"zh-TW":{"temperature":[4,4,4]}})").value();
+ auto t = dc::Temperatures::Of(nullptr, nullptr, &js).value();
+ EXPECT_EQ(t.For(sc::Type::Choice, 2, "zh", nullptr), 4.0);
+}
+
TEST(Decode, BinningReplacesTheConfidenceOnlyWithoutALanguageOverride) {
auto js = laya::pyvalue::Parse(R"({"l":{"zh":null}})").value();
auto t = dc::Temperatures::Of(nullptr, nullptr, js.Find("l")).value();
- t.binning["choice:2"].values = {0.25, 0.75};
+ std::map<std::string, dc::Bins> bins;
+ bins["choice:2"].values = {0.25, 0.75};
+ t.SetBinning(bins);
auto plain = dc::Decode(Pick(2), RowOf({4, 0}), "", &t).value();
EXPECT_EQ(plain.answer_confidence, 0.75); // 0.98 falls in the top of two bins
auto zh = dc::Decode(Pick(2), RowOf({4, 0}), "zh", &t).value();
EXPECT_NE(zh.answer_confidence, 0.75);
- EXPECT_EQ(dc::ApplyBinning(1.0, "choice:2", t.binning), 0.75); // clamped into the last bin
- EXPECT_EQ(dc::ApplyBinning(0.3, "absent", t.binning), 0.3);
+ EXPECT_EQ(dc::ApplyBinning(1.0, "choice:2", t.binning()), 0.75); // clamped into the last bin
+ EXPECT_EQ(dc::ApplyBinning(0.3, "absent", t.binning()), 0.3);
}
TEST(ActionProbabilities, SoftmaxAndRefusals) {
diff --git a/laya-cpp/tests/stub_session.hpp b/laya-cpp/tests/stub_session.hpp
index 15c0c9b2..bf0d17a2 100644
--- a/laya-cpp/tests/stub_session.hpp
+++ b/laya-cpp/tests/stub_session.hpp
@@ -28,6 +28,8 @@ class StubSession : public laya::infer::Session {
std::function<void()> on_run; // called at the start of every Run
std::function<void(laya::infer::Output*)> tamper; // may break the output
std::string fail; // non-empty: Run fails with this
+ bool ignore_cancel = false; // Run does not look at the token
+ std::string close_error; // what Close returns
};
explicit StubSession(std::shared_ptr<State> s = std::make_shared<State>()) : st_(std::move(s)) {}
~StubSession() override {
@@ -45,7 +47,7 @@ class StubSession : public laya::infer::Session {
hook = st_->on_run;
}
if (hook) hook();
- if (cancel.cancelled()) return R::Fail("cancelled");
+ if (!st_->ignore_cancel && cancel.cancelled()) return R::Fail(laya::infer::kCancelled);
RunRecord rec;
rec.rows = b.rows, rec.length = b.length, rec.markers = b.markers;
laya::infer::Output out;
@@ -75,7 +77,7 @@ class StubSession : public laya::infer::Session {
std::lock_guard<std::mutex> l(st_->mu);
st_->closes++;
st_->closed = true;
- return "";
+ return st_->close_error;
}
private:
diff --git a/laya-cpp/tests/tools/laya_c4_cli.cpp b/laya-cpp/tests/tools/laya_c4_cli.cpp
new file mode 100644
index 00000000..40e893c5
--- /dev/null
+++ b/laya-cpp/tests/tools/laya_c4_cli.cpp
@@ -0,0 +1,74 @@
+// A line protocol over laya::decode for the differential fuzz (laya-e2e/cpp-lane/fuzz-c4); bytes travel as hex.
+// dec <hex question wire def> <order|-> <hex json {"logits","act","temperature","tbo"}>
+// -> "err <hex>" | "ok <hex compact answer json>"
+#include <iostream>
+#include <sstream>
+
+#include "laya/decode.hpp"
+#include "laya/pyvalue.hpp"
+#include "laya/schema.hpp"
+
+namespace {
+using laya::pyvalue::Value;
+namespace dc = laya::decode;
+namespace sc = laya::schema;
+
+std::string FromHex(const std::string& h) {
+ std::string out;
+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));
+ return out;
+}
+std::string Hex(const std::string& s) {
+ static const char* d = "0123456789abcdef";
+ std::string o;
+ for (unsigned char c : s) {
+ o.push_back(d[c >> 4]);
+ o.push_back(d[c & 15]);
+ }
+ return o;
+}
+std::string Err(const std::string& e) { return "err " + Hex(e); }
+
+double Num(const Value& v) { return v.kind() == Value::Kind::Int ? double(v.as_int()) : v.as_float(); }
+
+std::string Dec(const std::string& def, const std::string& order, const std::string& arg) {
+ auto wire = laya::pyvalue::Parse(FromHex(def));
+ if (!wire.ok()) return Err(wire.error());
+ auto q = sc::Question::FromWire(wire.value());
+ if (!q.ok()) return Err(q.error());
+ sc::Question question = q.value();
+ if (order != "-") {
+ std::vector<int> o;
+ std::stringstream ss(order);
+ for (std::string part; std::getline(ss, part, ',');) o.push_back(std::stoi(part));
+ auto w = question.WithOptionOrder(o);
+ if (!w.ok()) return Err(w.error());
+ question = w.value();
+ }
+ auto a = laya::pyvalue::Parse(FromHex(arg));
+ if (!a.ok()) return Err(a.error());
+ auto temps = dc::Temperatures::Of(a.value().Find("temperature"), a.value().Find("tbo"));
+ if (!temps.ok()) return Err(temps.error());
+ dc::Row row;
+ for (const Value& v : a.value().Find("logits")->items()) row.logits.push_back(float(Num(v)));
+ row.k = int64_t(row.logits.size());
+ row.act = float(Num(*a.value().Find("act")));
+ auto ans = dc::Decode(question, row, "", &temps.value());
+ if (!ans.ok()) return Err(ans.error());
+ auto js = ans.value().ToJson();
+ if (!js.ok()) return Err(js.error());
+ auto back = laya::pyvalue::Parse(js.value());
+ return "ok " + Hex(laya::pyvalue::Dumps(back.value()).value());
+}
+} // namespace
+
+int main() {
+ std::string line;
+ while (std::getline(std::cin, line)) {
+ std::stringstream ss(line);
+ std::string cmd, def, order, arg;
+ ss >> cmd >> def >> order >> arg;
+ std::cout << (cmd == "dec" ? Dec(def, order, arg) : Err("unknown command")) << std::endl;
+ }
+ return 0;
+}
diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py
index 1b69bd5e..689678fb 100644
--- a/laya-java/scripts/gen_fixtures.py
+++ b/laya-java/scripts/gen_fixtures.py
@@ -1768,6 +1768,9 @@ def decode_answers():
def row(values):
return np.asarray(values, dtype=np.float32)
+ _LO = np.float32(1e-6)
+ _HI = np.nextafter(_LO, np.float32(1))
+
# (id, question, option_order, lang, logits)
cases = [
("choice-2", {"t": "choice", "ins": "i", "crit": {"a": "x", "b": "y"}},
@@ -1794,6 +1797,16 @@ def decode_answers():
None, None, [700.0, 699.0, -700.0]),
("choice-tiny-spread", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},
None, None, [1.0, 1.0 + 1e-7]),
+ # Adjacent float32 logits: numpy's float32 exp and sum make p an exact tie, so argmax
+ # takes the first option; a double softmax would pick the larger logit.
+ ("choice-ulp-tie", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},
+ None, None, [float(_LO), float(_HI)]),
+ ("choice-ulp-tie-reversed", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},
+ None, None, [float(_HI), float(_LO)]),
+ ("choice-ulp-tie-3", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2", "c": "3"}},
+ None, None, [float(_LO), float(_HI), float(_HI)]),
+ ("choice-ulp-tie-permuted", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},
+ [1, 0], None, [float(_LO), float(_HI)]),
("choice-lang-override", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},
None, "zh-Hans", [2.0, 1.0]),
("choice-lang-override-prefix", {"t": "choice", "ins": "i", "crit": {"a": "1", "b": "2"}},