Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(cpp): schema and sequence

Adeshina Lasisid7491a3ae238
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index cb24e6b3..076166ff 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -42,7 +42,7 @@ concurrency:
env:
# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus
# compile_flags). Raise it with every test added.
- MIN_TESTS: "253"
+ MIN_TESTS: "352"
GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints
@@ -93,7 +93,7 @@ jobs:
LAYA_FIXTURE_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints
PYTHONPATH: ${{ github.workspace }}
run: |
- python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json
+ python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json --only sequences.json --only window_plan.json
python laya-go/scripts/gen_go_fixtures.py --out "$GOLDENS"
- name: Compiled C++ tables match CPython
env:
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index 77981fe5..f76be6e7 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -26,7 +26,7 @@ function(laya_target_flags target)
endfunction()
add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp
- src/tokenizer.cpp)
+ src/tokenizer.cpp src/schema.cpp src/sequence.cpp)
target_include_directories(laya PUBLIC include)
laya_target_flags(laya)
@@ -46,7 +46,7 @@ if(LAYA_BUILD_TESTS)
enable_testing()
add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp
- tests/tokenizer_test.cpp)
+ tests/tokenizer_test.cpp tests/schema_test.cpp tests/sequence_test.cpp)
target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)
laya_target_flags(laya_tests)
@@ -56,6 +56,9 @@ if(LAYA_BUILD_TESTS)
add_executable(laya_c2_cli tests/tools/laya_c2_cli.cpp)
target_link_libraries(laya_c2_cli PRIVATE laya)
laya_target_flags(laya_c2_cli)
+ add_executable(laya_c3_cli tests/tools/laya_c3_cli.cpp)
+ target_link_libraries(laya_c3_cli PRIVATE laya)
+ laya_target_flags(laya_c3_cli)
include(GoogleTest)
# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.
diff --git a/laya-cpp/include/laya/schema.hpp b/laya-cpp/include/laya/schema.hpp
new file mode 100644
index 00000000..bb5a3fe9
--- /dev/null
+++ b/laya-cpp/include/laya/schema.hpp
@@ -0,0 +1,188 @@
+#pragma once
+// Question, answer and prediction types (laya-go/schema). Wording of every refusal is laya's own
+// Agent._check_question; JSON is spelled as Python's json.dumps spells it, keys in Python's order.
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "laya/pyvalue.hpp"
+#include "laya/result.hpp"
+
+namespace laya {
+namespace schema {
+
+using pyvalue::Value;
+
+enum class Type { Invalid, Choice, Score, Noul };
+std::string_view Name(Type t); // "choice", "score", "noul", "" for Invalid
+int Code(Type t); // the head's integer: 0, 1, 2; -1 for Invalid
+
+// A choice label: a string, or the bool, int or float Python also allows.
+class Label {
+ public:
+ enum class Kind { Text, Bool, Int, Float };
+ Label() = default;
+ static Label Text(std::string s);
+ static Label OfBool(bool b);
+ static Label OfInt(int64_t i);
+ static Label OfFloat(double d);
+ static Result<Label> Of(const Value& v); // string, bool, int or float; null and containers refused
+
+ Kind kind() const { return kind_; }
+ std::string String() const; // str(label): what the prompt shows
+ const std::string& Key() const { return text_; } // json.dumps spelling: the answer key
+ std::string Repr() const; // repr(label)
+ std::string EqKey() const; // equal for the labels Python's dict keys as one (1, 1.0, True)
+ std::string Json() const; // the label as a JSON value
+ bool operator==(const Label& o) const { return kind_ == o.kind_ && text_ == o.text_; }
+
+ private:
+ Kind kind_ = Kind::Text;
+ std::string text_;
+};
+
+struct Option {
+ Label label;
+ Value criterion; // Null and "" mean no description
+};
+
+struct NoulSpec {
+ Value true_text, false_text; // Null or "" keep the default texts
+ std::string true_label, false_label; // both empty keeps "true" and "false"
+};
+
+// A description snapshotted when the question is built.
+struct Criterion {
+ bool set = false;
+ std::string text; // render_criterion of the value
+ std::string raw; // the value as JSON, for the wire form
+};
+
+struct ChoiceOption {
+ Label label;
+ Criterion crit;
+};
+
+// The reason alone is the error of a constructor; ParseQuestions prefixes "question 'id': ".
+class Question {
+ public:
+ static Result<Question> Choice(std::string instructions, const std::vector<Option>& options);
+ static Result<Question> ChoiceOf(std::string instructions, const std::vector<std::string>& labels);
+ static Result<Question> Score(std::string instructions, const std::vector<Value>& levels);
+ static Result<Question> Noul(std::string instructions, const NoulSpec& spec = {});
+ // One definition in the Python wire form ({"type", "instructions", "criteria", ...}).
+ static Result<Question> FromWire(const Value& definition);
+
+ Result<Question> WithOptionOrder(const std::vector<int>& order) const; // a permutation of 0..n-1, else refused
+ Question WithoutOptionOrder() const; // restores label order
+ Result<Question> Subset(const std::vector<int>& indices) const;
+
+ Type type() const { return typ_; }
+ const std::string& instructions() const { return instructions_; }
+ size_t NumOptions() const;
+ const std::vector<int>& OptionOrder() const { return order_; } // empty: label order
+ std::vector<std::string> Labels() const; // str() of each label, label-index order
+ std::vector<Label> AnswerLabels() const;
+ std::vector<std::string> RenderedOptions() const; // render_options: the text the model reads
+ std::vector<std::string> Legend() const; // score only
+ Result<std::string> ToJson() const;
+
+ private:
+ Type typ_ = Type::Invalid;
+ std::string instructions_;
+ std::vector<ChoiceOption> options_;
+ std::vector<Criterion> levels_;
+ Criterion noul_[2]; // false side, true side
+ std::string false_label_, true_label_;
+ bool custom_labels_ = false;
+ std::vector<int> order_;
+};
+
+struct Entry {
+ std::string id;
+ Question question;
+};
+
+class Questions {
+ public:
+ static Result<Questions> Of(const std::vector<Entry>& entries);
+ static Result<Questions> Parse(std::string_view json);
+ size_t size() const { return entries_.size(); }
+ const Entry& at(size_t i) const { return entries_[i]; }
+ const Question* Get(std::string_view id) const;
+ std::vector<std::string> Ids() const;
+ Result<std::string> ToJson() const;
+
+ private:
+ std::vector<Entry> entries_;
+ std::map<std::string, size_t, std::less<>> index_;
+};
+
+// json.dumps of a string, ensure_ascii=False.
+std::string JsonString(std::string_view s);
+
+struct Probability {
+ Label label;
+ double p = 0;
+};
+
+struct Window {
+ int64_t index = 0, token_start = 0, token_end = 0, count = 0;
+};
+
+// One answer; the fields that apply depend on type.
+struct Answer {
+ Type type = Type::Invalid;
+ Label choice; // choice
+ double score = 0; // score
+ double noul = 0; // noul
+ std::vector<std::string> legend; // score
+ std::vector<Probability> probabilities; // choice, score
+ double confidence = 0, answer_confidence = 0, act_probability = 0;
+ std::optional<Window> window;
+ bool low_confidence = false;
+ std::string abstention; // "", "passed", "abstained" or "unevaluated"
+ std::optional<double> abstention_threshold;
+
+ Result<std::string> ToJson() const;
+ static Result<Answer> FromJson(const Value& v);
+};
+
+struct Answered {
+ std::string id;
+ Answer answer;
+};
+
+struct CollapsedOptions {
+ int64_t total = 0, distinct = 0;
+ std::optional<int64_t> tokens_per_option;
+};
+struct CollapsedQuestion {
+ std::string id;
+ CollapsedOptions options;
+};
+
+struct Usage {
+ int64_t input_tokens = 0, output_tokens = 0, state_tokens = 0, state_tokens_dropped = 0;
+ bool truncated = false;
+ std::vector<std::string> truncated_questions;
+ std::vector<CollapsedQuestion> collapsed_options;
+ int64_t windows = 0;
+ int64_t truncated_windows = 0; // with windows > 1, the "truncated" key is this count
+ Result<std::string> ToJson() const;
+};
+
+constexpr const char* kModel = "laya-rl-agent-onnx";
+
+struct Prediction {
+ std::string model = kModel;
+ std::vector<Answered> answers;
+ Usage usage;
+ Result<std::string> ToJson() const;
+};
+
+} // namespace schema
+} // namespace laya
diff --git a/laya-cpp/include/laya/sequence.hpp b/laya-cpp/include/laya/sequence.hpp
new file mode 100644
index 00000000..66e0eb5f
--- /dev/null
+++ b/laya-cpp/include/laya/sequence.hpp
@@ -0,0 +1,104 @@
+#pragma once
+// The model's input sequence (laya-go/sequence): head, state, markers, and the window plan.
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "laya/pyvalue.hpp"
+#include "laya/result.hpp"
+#include "laya/schema.hpp"
+#include "laya/tokenizer.hpp"
+
+namespace laya {
+namespace sequence {
+
+constexpr int64_t kOptionTokenCap = 48;
+constexpr int64_t kMinOptionBudget = 16;
+constexpr int64_t kMinTokensPerOption = 4;
+constexpr int64_t kMinInstructionTokens = 8;
+
+struct Budget {
+ int64_t max_len = 0;
+ int64_t head_max_len = 0;
+ int64_t DefaultWindow() const; // max(64, max_len - head_max_len - 8)
+};
+
+enum class Keep { Start, End };
+
+struct State {
+ std::vector<int32_t> ids;
+ Keep keep = Keep::Start;
+};
+
+// serialize_state: a string as is, anything else as json.dumps(ensure_ascii=False).
+Result<std::string> SerializeState(const pyvalue::Value& state);
+// A list keeps its end when it is too long, anything else its start.
+Result<State> EncodeState(const tokenizer::Tokenizer& tok, const pyvalue::Value& state);
+
+struct Stats {
+ int64_t options = 0;
+ int64_t options_distinct = 0;
+ std::optional<int64_t> tokens_per_option;
+ bool Collapsed() const { return options_distinct < options; }
+};
+
+struct Truncation {
+ int64_t state_tokens = 0, state_tokens_used = 0, state_tokens_dropped = 0;
+ bool truncated = false;
+ bool operator==(const Truncation& o) const {
+ return state_tokens == o.state_tokens && state_tokens_used == o.state_tokens_used &&
+ state_tokens_dropped == o.state_tokens_dropped && truncated == o.truncated;
+ }
+};
+
+struct Head {
+ std::vector<int32_t> ids;
+ std::vector<int64_t> markers;
+ Stats stats;
+};
+
+struct Sequence {
+ std::vector<int32_t> ids;
+ std::vector<int64_t> markers;
+ Stats stats;
+ Truncation truncation;
+};
+
+// Some option markers fell beyond max_len. Message(id) is the agent's refusal text.
+struct MarkersError {
+ int64_t kept = 0, total = 0, max_len = 0, head_max_len = 0;
+ std::string Message(const std::string& id = "") const;
+};
+
+Result<Head> BuildHead(const tokenizer::Tokenizer& tok, const schema::Question& q, int64_t head_max_len);
+// markers_error, when not null, is filled iff the failure is the marker refusal.
+Result<Sequence> Build(const tokenizer::Tokenizer& tok, const schema::Question& q, const State& state,
+ const Budget& b, MarkersError* markers_error = nullptr);
+Result<int64_t> StateRoom(const tokenizer::Tokenizer& tok, const schema::Question& q, const Budget& b);
+
+struct WindowRequest {
+ int64_t window = 0; // <= 0: the default
+ int64_t stride = 0; // <= 0: half a window
+};
+
+struct WindowPlan {
+ int64_t window = 0, stride = 0, room = 0;
+ std::vector<std::string> warnings;
+ bool Clamped() const { return !warnings.empty(); }
+};
+
+enum class PlanError { None, NoRoom, StrideGap, Other };
+
+Result<WindowPlan> PlanWindows(const tokenizer::Tokenizer& tok, const std::vector<schema::Question>& questions,
+ const Budget& b, const WindowRequest& req, PlanError* kind = nullptr);
+
+// window_batch_cap: ok=false is Python's None (do not cap). batch_size <= 0 is unset.
+struct Cap {
+ int64_t cap = 0;
+ bool ok = false;
+};
+Cap BatchCap(int64_t n_windows, int64_t window, int64_t config_window, int64_t batch_size);
+
+} // namespace sequence
+} // namespace laya
diff --git a/laya-cpp/src/schema.cpp b/laya-cpp/src/schema.cpp
new file mode 100644
index 00000000..9e5a225d
--- /dev/null
+++ b/laya-cpp/src/schema.cpp
@@ -0,0 +1,938 @@
+#include "laya/schema.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+
+#include "laya/pyjson.hpp"
+#include "laya/unicodedata.hpp"
+
+namespace laya {
+namespace schema {
+
+namespace ud = unicodedata;
+using Kind = Value::Kind;
+
+std::string_view Name(Type t) {
+ switch (t) {
+ case Type::Choice: return "choice";
+ case Type::Score: return "score";
+ case Type::Noul: return "noul";
+ case Type::Invalid: break;
+ }
+ return "";
+}
+
+int Code(Type t) {
+ switch (t) {
+ case Type::Choice: return 0;
+ case Type::Score: return 1;
+ case Type::Noul: return 2;
+ case Type::Invalid: break;
+ }
+ return -1;
+}
+
+std::string JsonString(std::string_view s) {
+ auto r = pyvalue::Dumps(Value::String(ud::ValidUtf8(s) ? std::string(s) : ud::Lossy(s)));
+ return r.ok() ? r.value() : "\"\"";
+}
+
+namespace {
+
+// Reasons laya's Agent._check_question gives, word for word.
+constexpr const char* kNoInstructions = "no 'instructions'; add the text the model should answer";
+constexpr const char* kNilInstructions = "'instructions' must not be None; add the text the model should answer";
+constexpr const char* kBlankIns = "'instructions' must not be empty; add the text the model should answer";
+constexpr const char* kChoiceShape =
+ "a choice question takes 'criteria' as a dict of label -> description, or a list of labels";
+constexpr const char* kChoiceEmpty = "a choice question needs at least one criterion";
+constexpr const char* kScoreShape = "a score question takes 'criteria' as a list of level descriptions, index 0 first";
+constexpr const char* kScoreEmpty = "a score question needs at least one level";
+constexpr const char* kNoulShape =
+ "a noul question takes 'criteria' as a dict with optional 'true'/'false' descriptions, or omits it";
+constexpr const char* kLabelsOnlyNoul = "'labels' is only supported for noul questions";
+constexpr const char* kNoulLabels = "noul labels must map exactly 'false' and 'true' to distinct non-empty strings";
+constexpr const char* kDefaultFalse = "no, the statement does not hold";
+constexpr const char* kDefaultTrue = "yes, the statement holds";
+
+std::string N(int64_t i) { return std::to_string(i); }
+
+std::string NoulKeysReason(const std::string& got) {
+ return "a noul question takes 'criteria' keyed only 'true'/'false' (either or both, and omitted is fine), got " + got +
+ ". Those keys are the option texts the model reads; any other key was silently dropped and replaced with "
+ "the defaults. If you want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "
+ "'labels' instead.";
+}
+
+std::string OrderReason(size_t n, const std::string& got) {
+ return "'option_order' must be a permutation of range(" + N(int64_t(n)) +
+ ") -- one slot per option, each option once -- got " + got;
+}
+
+std::string LabelRepeatReason(size_t i, const std::string& repr, size_t j) {
+ return "choice label " + N(int64_t(i)) + " (" + repr + ") repeats label " + N(int64_t(j)) +
+ "; the labels are the answer keys, so every option needs its own (1, 1.0 and True are one key)";
+}
+
+std::string ScoreNullReason(size_t i) {
+ return "score level " + N(int64_t(i)) + " is null; give every level a description, index 0 first";
+}
+
+// An integer-valued double as exact decimal digits (no sign for zero).
+std::string ExactInteger(double d) {
+ if (d == 0) return "0";
+ if (std::fabs(d) < 9.0e18) return N(int64_t(d));
+ int exp = 0;
+ double m = std::frexp(std::fabs(d), &exp); // d = m * 2^exp, 0.5 <= m < 1
+ uint64_t mant = uint64_t(std::ldexp(m, 53)); // 53-bit integer
+ exp -= 53; // d = mant * 2^exp, exp >= 0 here
+ std::vector<int> digits; // little endian, base 10
+ for (uint64_t v = mant; v > 0; v /= 10) digits.push_back(int(v % 10));
+ for (int i = 0; i < exp; i++) {
+ int carry = 0;
+ for (int& x : digits) {
+ int v = x * 2 + carry;
+ x = v % 10;
+ carry = v / 10;
+ }
+ if (carry) digits.push_back(carry);
+ }
+ std::string out = d < 0 ? "-" : "";
+ for (size_t i = digits.size(); i-- > 0;) out.push_back(char('0' + digits[i]));
+ return out;
+}
+
+Criterion NewCriterion(const Value& v, std::string* err) {
+ if (v.is_null()) return {};
+ if (v.kind() == Kind::String) {
+ if (v.as_string().empty()) return {};
+ return {true, v.as_string(), JsonString(v.as_string())};
+ }
+ auto text = pyvalue::Dumps(v);
+ if (!text.ok()) {
+ *err = text.error();
+ return {};
+ }
+ return {true, text.value(), text.value()};
+}
+
+std::string CheckInstructions(const std::string& ins) {
+ if (ud::IsBlank(ins)) return kBlankIns;
+ if (!ud::ValidUtf8(ins)) return "'instructions' is not valid UTF-8; a Python str always is";
+ return "";
+}
+
+std::string CheckDistinct(const std::vector<Label>& labels) {
+ std::map<std::string, size_t> first;
+ for (size_t i = 0; i < labels.size(); i++) {
+ auto ins = first.emplace(labels[i].EqKey(), i);
+ if (!ins.second) return LabelRepeatReason(i, labels[i].Repr(), ins.first->second);
+ }
+ return "";
+}
+
+template <class T>
+Result<T> Refuse(const std::string& reason) {
+ return Result<T>::Fail(reason);
+}
+
+} // namespace
+
+// ---- Label -------------------------------------------------------------------------------------
+
+Label Label::Text(std::string s) {
+ Label l;
+ l.text_ = std::move(s);
+ return l;
+}
+Label Label::OfBool(bool b) {
+ Label l;
+ l.kind_ = Kind::Bool;
+ l.text_ = b ? "true" : "false";
+ return l;
+}
+Label Label::OfInt(int64_t i) {
+ Label l;
+ l.kind_ = Kind::Int;
+ l.text_ = std::to_string(i);
+ return l;
+}
+Label Label::OfFloat(double d) {
+ Label l;
+ l.kind_ = Kind::Float;
+ l.text_ = pyjson::ReprFloat(d);
+ return l;
+}
+
+Result<Label> Label::Of(const Value& v) {
+ switch (v.kind()) {
+ case pyvalue::Value::Kind::String: return Result<Label>::Ok(Text(v.as_string()));
+ case pyvalue::Value::Kind::Bool: return Result<Label>::Ok(OfBool(v.as_bool()));
+ case pyvalue::Value::Kind::Int: return Result<Label>::Ok(OfInt(v.as_int()));
+ case pyvalue::Value::Kind::BigInt: {
+ Label l;
+ l.kind_ = Kind::Int;
+ l.text_ = v.big_int();
+ return Result<Label>::Ok(l);
+ }
+ case pyvalue::Value::Kind::Float: return Result<Label>::Ok(OfFloat(v.as_float()));
+ default: break;
+ }
+ return Result<Label>::Fail("schema: " + pyvalue::TypeName(v) + " is not a label; use a string, bool or number");
+}
+
+std::string Label::String() const {
+ if (kind_ == Kind::Bool) return text_ == "true" ? "True" : "False";
+ if (kind_ == Kind::Float) {
+ if (text_ == "NaN") return "nan";
+ if (text_ == "Infinity") return "inf";
+ if (text_ == "-Infinity") return "-inf";
+ }
+ return text_;
+}
+
+std::string Label::Repr() const { return kind_ == Kind::Text ? pyjson::Repr(text_) : String(); }
+
+std::string Label::EqKey() const {
+ switch (kind_) {
+ case Kind::Text: return "s:" + text_;
+ case Kind::Bool: return text_ == "true" ? "n:1" : "n:0";
+ case Kind::Int: return "n:" + text_;
+ case Kind::Float: break;
+ }
+ if (text_ == "NaN" || text_ == "Infinity" || text_ == "-Infinity") return "n:" + text_;
+ double d = std::strtod(text_.c_str(), nullptr); // the C locale: the library never changes it
+ if (d == std::trunc(d)) return "n:" + ExactInteger(d);
+ return "f:" + text_; // shortest repr is unique per double
+}
+
+std::string Label::Json() const { return kind_ == Kind::Text ? JsonString(text_) : text_; }
+
+// ---- Question ----------------------------------------------------------------------------------
+
+Result<Question> Question::Choice(std::string instructions, const std::vector<Option>& options) {
+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);
+ if (options.empty()) return Refuse<Question>(kChoiceEmpty);
+ std::vector<Label> labels;
+ for (size_t i = 0; i < options.size(); i++) {
+ const Label& l = options[i].label;
+ if (l.kind() == Label::Kind::Text && !ud::ValidUtf8(l.Key())) {
+ return Refuse<Question>("choice label " + N(int64_t(i)) + " is not valid UTF-8; a Python str always is");
+ }
+ labels.push_back(l);
+ }
+ if (auto r = CheckDistinct(labels); !r.empty()) return Refuse<Question>(r);
+ Question q;
+ q.typ_ = Type::Choice;
+ q.instructions_ = std::move(instructions);
+ for (size_t i = 0; i < options.size(); i++) {
+ std::string err;
+ Criterion c = NewCriterion(options[i].criterion, &err);
+ if (!err.empty()) {
+ return Refuse<Question>("choice option " + labels[i].Repr() + ": criterion cannot be rendered: " + err);
+ }
+ q.options_.push_back({labels[i], std::move(c)});
+ }
+ return Result<Question>::Ok(std::move(q));
+}
+
+Result<Question> Question::ChoiceOf(std::string instructions, const std::vector<std::string>& labels) {
+ std::vector<Option> options;
+ for (const auto& l : labels) options.push_back({Label::Text(l), Value()});
+ return Choice(std::move(instructions), options);
+}
+
+Result<Question> Question::Score(std::string instructions, const std::vector<Value>& levels) {
+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);
+ if (levels.empty()) return Refuse<Question>(kScoreEmpty);
+ Question q;
+ q.typ_ = Type::Score;
+ q.instructions_ = std::move(instructions);
+ for (size_t i = 0; i < levels.size(); i++) {
+ if (levels[i].is_null()) return Refuse<Question>(ScoreNullReason(i));
+ std::string err;
+ Criterion c = NewCriterion(levels[i], &err);
+ if (!err.empty()) return Refuse<Question>("score level " + N(int64_t(i)) + " cannot be rendered: " + err);
+ if (!c.set) c = {true, "", "\"\""}; // the empty string is a level: it renders as "level i: "
+ q.levels_.push_back(std::move(c));
+ }
+ return Result<Question>::Ok(std::move(q));
+}
+
+Result<Question> Question::Noul(std::string instructions, const NoulSpec& spec) {
+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);
+ Question q;
+ q.typ_ = Type::Noul;
+ q.instructions_ = std::move(instructions);
+ q.false_label_ = "false";
+ q.true_label_ = "true";
+ const Value* sides[2] = {&spec.false_text, &spec.true_text};
+ const char* names[2] = {"false", "true"};
+ for (int i = 0; i < 2; i++) {
+ std::string err;
+ q.noul_[i] = NewCriterion(*sides[i], &err);
+ if (!err.empty()) return Refuse<Question>(std::string("noul ") + names[i] + " description cannot be rendered: " + err);
+ }
+ if (!spec.true_label.empty() || !spec.false_label.empty()) {
+ std::string f = ud::Strip(spec.false_label), t = ud::Strip(spec.true_label);
+ if (f.empty() || t.empty() || f == t || !ud::ValidUtf8(f) || !ud::ValidUtf8(t)) return Refuse<Question>(kNoulLabels);
+ q.false_label_ = f;
+ q.true_label_ = t;
+ q.custom_labels_ = true;
+ }
+ return Result<Question>::Ok(std::move(q));
+}
+
+size_t Question::NumOptions() const {
+ switch (typ_) {
+ case Type::Choice: return options_.size();
+ case Type::Score: return levels_.size();
+ case Type::Noul: return 2;
+ case Type::Invalid: break;
+ }
+ return 0;
+}
+
+Result<Question> Question::WithOptionOrder(const std::vector<int>& order) const {
+ if (typ_ == Type::Invalid) {
+ return Refuse<Question>("the zero Question has no options to order; build one with Choice, ChoiceOf, Score, Noul");
+ }
+ Question q = *this;
+ size_t n = NumOptions();
+ std::vector<bool> seen(n, false);
+ bool ok = order.size() == n;
+ for (size_t i = 0; ok && i < order.size(); i++) {
+ int v = order[i];
+ if (v < 0 || size_t(v) >= n || seen[size_t(v)]) ok = false;
+ else seen[size_t(v)] = true;
+ }
+ if (!ok) {
+ std::string s = "[";
+ for (size_t i = 0; i < order.size(); i++) s += (i ? ", " : "") + N(order[i]);
+ return Refuse<Question>(OrderReason(n, s + "]"));
+ }
+ q.order_ = order;
+ return Result<Question>::Ok(std::move(q));
+}
+
+std::vector<std::string> Question::Labels() const {
+ std::vector<std::string> out;
+ switch (typ_) {
+ case Type::Choice: for (const auto& o : options_) out.push_back(o.label.String()); break;
+ case Type::Score: for (size_t i = 0; i < levels_.size(); i++) out.push_back(N(int64_t(i))); break;
+ case Type::Noul: out = {false_label_, true_label_}; break;
+ case Type::Invalid: break;
+ }
+ return out;
+}
+
+std::vector<Label> Question::AnswerLabels() const {
+ std::vector<Label> out;
+ switch (typ_) {
+ case Type::Choice: for (const auto& o : options_) out.push_back(o.label); break;
+ case Type::Score: for (size_t i = 0; i < levels_.size(); i++) out.push_back(Label::Text(N(int64_t(i)))); break;
+ case Type::Noul: out = {Label::Text(false_label_), Label::Text(true_label_)}; break;
+ case Type::Invalid: break;
+ }
+ return out;
+}
+
+std::vector<std::string> Question::RenderedOptions() const {
+ std::vector<std::string> out;
+ switch (typ_) {
+ case Type::Choice:
+ for (const auto& o : options_) out.push_back(o.crit.set ? o.label.String() + ": " + o.crit.text : o.label.String());
+ break;
+ case Type::Score:
+ for (size_t i = 0; i < levels_.size(); i++) out.push_back("level " + N(int64_t(i)) + ": " + levels_[i].text);
+ break;
+ case Type::Noul:
+ out.push_back(false_label_ + ": " + (noul_[0].set ? noul_[0].text : std::string(kDefaultFalse)));
+ out.push_back(true_label_ + ": " + (noul_[1].set ? noul_[1].text : std::string(kDefaultTrue)));
+ break;
+ case Type::Invalid: break;
+ }
+ return out;
+}
+
+std::vector<std::string> Question::Legend() const {
+ std::vector<std::string> out;
+ if (typ_ == Type::Score) for (const auto& l : levels_) out.push_back(l.text);
+ return out;
+}
+
+Result<std::string> Question::ToJson() const {
+ if (typ_ == Type::Invalid) return Refuse<std::string>("schema: cannot marshal the zero Question");
+ std::string b = "{\"type\":\"" + std::string(Name(typ_)) + "\",\"instructions\":" + JsonString(instructions_);
+ switch (typ_) {
+ case Type::Choice: {
+ bool described = false;
+ for (const auto& o : options_) described = described || o.crit.set;
+ if (!described) {
+ b += ",\"criteria\":[";
+ for (size_t i = 0; i < options_.size(); i++) b += (i ? "," : "") + options_[i].label.Json();
+ b += "]";
+ } else {
+ b += ",\"criteria\":{";
+ for (size_t i = 0; i < options_.size(); i++) {
+ b += (i ? "," : "") + JsonString(options_[i].label.Key()) + ":" + (options_[i].crit.set ? options_[i].crit.raw : "null");
+ }
+ b += "}";
+ }
+ break;
+ }
+ case Type::Score:
+ b += ",\"criteria\":[";
+ for (size_t i = 0; i < levels_.size(); i++) b += (i ? "," : "") + levels_[i].raw;
+ b += "]";
+ break;
+ case Type::Noul:
+ if (noul_[0].set || noul_[1].set) {
+ b += ",\"criteria\":{";
+ const char* sep = "";
+ const char* keys[2] = {"false", "true"};
+ for (int i = 0; i < 2; i++) {
+ if (!noul_[i].set) continue;
+ b += std::string(sep) + "\"" + keys[i] + "\":" + noul_[i].raw;
+ sep = ",";
+ }
+ b += "}";
+ }
+ if (custom_labels_) {
+ b += ",\"labels\":{\"false\":" + JsonString(false_label_) + ",\"true\":" + JsonString(true_label_) + "}";
+ }
+ break;
+ case Type::Invalid: break;
+ }
+ if (!order_.empty()) {
+ b += ",\"option_order\":[";
+ for (size_t i = 0; i < order_.size(); i++) b += (i ? "," : "") + N(order_[i]);
+ b += "]";
+ }
+ b += "}";
+ return Result<std::string>::Ok(std::move(b));
+}
+
+Question Question::WithoutOptionOrder() const {
+ Question q = *this;
+ q.order_.clear();
+ return q;
+}
+
+Result<Question> Question::Subset(const std::vector<int>& indices) const {
+ if (typ_ != Type::Choice) {
+ return Refuse<Question>("only a choice question has options to subset, not " + std::string(Name(typ_)));
+ }
+ if (indices.empty()) return Refuse<Question>(kChoiceEmpty);
+ std::vector<bool> seen(options_.size(), false);
+ Question q;
+ q.typ_ = Type::Choice;
+ q.instructions_ = instructions_;
+ for (size_t i = 0; i < indices.size(); i++) {
+ int ix = indices[i];
+ if (ix < 0 || size_t(ix) >= options_.size() || seen[size_t(ix)]) {
+ return Refuse<Question>("subset index " + N(ix) + " at position " + N(int64_t(i)) + " is out of range or repeated");
+ }
+ seen[size_t(ix)] = true;
+ q.options_.push_back(options_[size_t(ix)]);
+ }
+ return Result<Question>::Ok(std::move(q));
+}
+
+// ---- wire form ---------------------------------------------------------------------------------
+
+namespace {
+
+Result<std::string> WireInstructions(const Value& def) {
+ const Value* ins = def.Find("instructions");
+ if (ins == nullptr) return Refuse<std::string>(kNoInstructions);
+ switch (ins->kind()) {
+ case Kind::Null: return Refuse<std::string>(kNilInstructions);
+ case Kind::String: {
+ if (auto r = CheckInstructions(ins->as_string()); !r.empty()) return Refuse<std::string>(r);
+ return Result<std::string>::Ok(ins->as_string());
+ }
+ case Kind::Array:
+ case Kind::Object:
+ if (ins->size() == 0) return Refuse<std::string>(kBlankIns);
+ break;
+ default: break;
+ }
+ auto text = pyvalue::Dumps(*ins); // _to_internal: json.dumps(ins, ensure_ascii=False)
+ if (!text.ok()) return Refuse<std::string>("'instructions' cannot be written as JSON: " + text.error());
+ return text;
+}
+
+Result<std::vector<ChoiceOption>> WireChoice(const Value* crit) {
+ using R = Result<std::vector<ChoiceOption>>;
+ std::vector<Label> labels;
+ std::vector<Value> values;
+ if (crit != nullptr && crit->kind() == Kind::Object) {
+ for (size_t i = 0; i < crit->size(); i++) {
+ labels.push_back(Label::Text(crit->key(i)));
+ values.push_back(crit->value(i));
+ }
+ } else if (crit != nullptr && crit->kind() == Kind::Array) {
+ for (size_t i = 0; i < crit->size(); i++) {
+ const Value& l = crit->value(i);
+ switch (l.kind()) {
+ case Kind::String: case Kind::Bool: case Kind::Int: case Kind::BigInt: case Kind::Float: break;
+ case Kind::Null:
+ return R::Fail("choice label " + N(int64_t(i)) +
+ " is null; a label is rendered as option text and used as the answer key, so it must be a "
+ "string, number or bool -- a null label renders as the text \"None\" while its answer key is "
+ "\"null\"");
+ default:
+ return R::Fail("choice label " + N(int64_t(i)) + " is a " + pyvalue::TypeName(l) +
+ "; a label is rendered as option text and used as the answer key, so it must be a scalar (a "
+ "string, number or bool), got " + pyvalue::Repr(l));
+ }
+ labels.push_back(Label::Of(l).value());
+ values.push_back(Value());
+ }
+ if (auto r = CheckDistinct(labels); !r.empty()) return R::Fail(r);
+ } else {
+ return R::Fail(kChoiceShape);
+ }
+ if (labels.empty()) return R::Fail(kChoiceEmpty);
+ std::vector<ChoiceOption> opts;
+ for (size_t i = 0; i < labels.size(); i++) {
+ std::string err;
+ Criterion c = NewCriterion(values[i], &err);
+ if (!err.empty()) return R::Fail("choice option " + labels[i].Repr() + ": criterion cannot be rendered: " + err);
+ opts.push_back({labels[i], std::move(c)});
+ }
+ return R::Ok(std::move(opts));
+}
+
+Result<std::vector<Criterion>> WireScore(const Value* crit) {
+ using R = Result<std::vector<Criterion>>;
+ if (crit == nullptr || crit->kind() != Kind::Array) return R::Fail(kScoreShape);
+ if (crit->size() == 0) return R::Fail(kScoreEmpty);
+ std::vector<Criterion> out;
+ for (size_t i = 0; i < crit->size(); i++) {
+ const Value& v = crit->value(i);
+ if (v.is_null()) return R::Fail(ScoreNullReason(i));
+ std::string err;
+ Criterion c = NewCriterion(v, &err);
+ if (!err.empty()) return R::Fail("score level " + N(int64_t(i)) + " cannot be rendered: " + err);
+ if (!c.set) c = {true, "", "\"\""};
+ out.push_back(std::move(c));
+ }
+ return R::Ok(std::move(out));
+}
+
+bool WireNoul(const Value* crit, Criterion (&out)[2], std::string* reason) {
+ if (crit == nullptr || crit->is_null()) return true;
+ if (crit->kind() != Kind::Object) {
+ *reason = kNoulShape;
+ return false;
+ }
+ std::map<std::string, Value> sides; // _to_internal: {str(k).lower(): v}; a later key wins
+ for (size_t i = 0; i < crit->size(); i++) sides[ud::Lower(crit->key(i))] = crit->value(i);
+ bool unknown = false;
+ Value keys = Value::Array();
+ for (const auto& kv : sides) {
+ unknown = unknown || (kv.first != "true" && kv.first != "false");
+ keys.Push(Value::String(kv.first));
+ }
+ if (unknown) {
+ *reason = NoulKeysReason(pyvalue::Repr(keys));
+ return false;
+ }
+ const char* names[2] = {"false", "true"};
+ for (int i = 0; i < 2; i++) {
+ auto it = sides.find(names[i]);
+ std::string err;
+ out[i] = NewCriterion(it == sides.end() ? Value() : it->second, &err);
+ if (!err.empty()) {
+ *reason = std::string("noul ") + names[i] + " description cannot be rendered: " + err;
+ return false;
+ }
+ }
+ return true;
+}
+
+} // namespace
+
+Result<Question> Question::FromWire(const Value& def) {
+ if (def.kind() != Kind::Object) return Refuse<Question>("definition must be a dict, got " + pyvalue::TypeName(def));
+ const Value* tv = def.Find("type");
+ Type typ = Type::Invalid;
+ if (tv != nullptr && tv->kind() == Kind::String) {
+ const std::string& s = tv->as_string();
+ typ = s == "choice" ? Type::Choice : s == "score" ? Type::Score : s == "noul" ? Type::Noul : Type::Invalid;
+ }
+ if (typ == Type::Invalid) {
+ return Refuse<Question>("unknown type " + pyvalue::Repr(tv == nullptr ? Value() : *tv) +
+ "; use one of ['choice', 'noul', 'score']");
+ }
+ auto ins = WireInstructions(def);
+ if (!ins.ok()) return Refuse<Question>(ins.error());
+ Question q;
+ q.typ_ = typ;
+ q.instructions_ = ins.value();
+ const Value* crit = def.Find("criteria");
+ if (typ == Type::Choice) {
+ auto r = WireChoice(crit);
+ if (!r.ok()) return Refuse<Question>(r.error());
+ q.options_ = std::move(r).value();
+ } else if (typ == Type::Score) {
+ auto r = WireScore(crit);
+ if (!r.ok()) return Refuse<Question>(r.error());
+ q.levels_ = std::move(r).value();
+ } else {
+ std::string reason;
+ if (!WireNoul(crit, q.noul_, &reason)) return Refuse<Question>(reason);
+ q.false_label_ = "false";
+ q.true_label_ = "true";
+ }
+ if (const Value* order = def.Find("option_order")) {
+ size_t n = q.NumOptions();
+ bool good = order->kind() == Kind::Array && order->size() == n;
+ std::vector<int> ints;
+ std::vector<bool> seen(n, false);
+ for (size_t i = 0; good && i < n; i++) {
+ const Value& v = order->value(i); // only true ints (not bools or floats) count
+ if (v.kind() != Kind::Int || v.as_int() < 0 || uint64_t(v.as_int()) >= n || seen[size_t(v.as_int())]) {
+ good = false;
+ } else {
+ seen[size_t(v.as_int())] = true;
+ ints.push_back(int(v.as_int()));
+ }
+ }
+ if (!good) return Refuse<Question>(OrderReason(n, pyvalue::Repr(*order)));
+ q.order_ = std::move(ints);
+ }
+ if (const Value* labels = def.Find("labels")) {
+ if (typ != Type::Noul) return Refuse<Question>(kLabelsOnlyNoul);
+ if (!labels->is_null()) { // _resolve_noul_labels(None) is the default pair
+ if (labels->kind() != Kind::Object || labels->size() != 2) return Refuse<Question>(kNoulLabels);
+ std::string f, t;
+ bool have_f = false, have_t = false;
+ for (size_t i = 0; i < 2; i++) {
+ bool is_str = labels->value(i).kind() == Kind::String;
+ const std::string s = is_str ? labels->value(i).as_string() : "";
+ if (labels->key(i) == "false") { f = s; have_f = is_str; }
+ else if (labels->key(i) == "true") { t = s; have_t = is_str; }
+ else return Refuse<Question>(kNoulLabels);
+ }
+ f = ud::Strip(f);
+ t = ud::Strip(t);
+ if (!have_f || !have_t || f.empty() || t.empty() || f == t) return Refuse<Question>(kNoulLabels);
+ q.false_label_ = f;
+ q.true_label_ = t;
+ q.custom_labels_ = true;
+ }
+ }
+ return Result<Question>::Ok(std::move(q));
+}
+
+// ---- Questions ---------------------------------------------------------------------------------
+
+namespace {
+Result<Questions> IdRefusal(const std::string& id, const std::string& reason) {
+ return Result<Questions>::Fail(id.empty() ? reason : "question " + pyjson::Repr(id) + ": " + reason);
+}
+std::string CheckId(const std::string& id) {
+ if (ud::IsBlank(id)) return "question id must be a non-empty string, got " + pyjson::Repr(id);
+ return "";
+}
+} // namespace
+
+Result<Questions> Questions::Of(const std::vector<Entry>& entries) {
+ Questions qs;
+ for (const auto& e : entries) {
+ if (auto r = CheckId(e.id); !r.empty()) return Result<Questions>::Fail(r);
+ if (e.question.type() == Type::Invalid) {
+ return IdRefusal(e.id, "zero Question; build one with Choice, ChoiceOf, Score, Noul");
+ }
+ if (qs.index_.count(e.id)) {
+ return IdRefusal(e.id, "id is given twice; answers are keyed by id, so every question needs its own");
+ }
+ qs.index_[e.id] = qs.entries_.size();
+ qs.entries_.push_back(e);
+ }
+ return Result<Questions>::Ok(std::move(qs));
+}
+
+Result<Questions> Questions::Parse(std::string_view json) {
+ auto v = pyvalue::Parse(json);
+ if (!v.ok()) return Result<Questions>::Fail("schema: questions: " + v.error());
+ const Value& root = v.value();
+ if (root.kind() != Kind::Object) {
+ return Result<Questions>::Fail("schema: questions must be a JSON object, got " + pyvalue::TypeName(root));
+ }
+ Questions qs;
+ for (size_t i = 0; i < root.size(); i++) {
+ const std::string& id = root.key(i);
+ if (auto r = CheckId(id); !r.empty()) return Result<Questions>::Fail(r);
+ auto q = Question::FromWire(root.value(i));
+ if (!q.ok()) return IdRefusal(id, q.error());
+ qs.index_[id] = qs.entries_.size();
+ qs.entries_.push_back({id, std::move(q).value()});
+ }
+ return Result<Questions>::Ok(std::move(qs));
+}
+
+const Question* Questions::Get(std::string_view id) const {
+ auto it = index_.find(id);
+ return it == index_.end() ? nullptr : &entries_[it->second].question;
+}
+
+std::vector<std::string> Questions::Ids() const {
+ std::vector<std::string> out;
+ for (const auto& e : entries_) out.push_back(e.id);
+ return out;
+}
+
+Result<std::string> Questions::ToJson() const {
+ std::string b = "{";
+ for (size_t i = 0; i < entries_.size(); i++) {
+ auto q = entries_[i].question.ToJson();
+ if (!q.ok()) return q;
+ b += (i ? "," : "") + JsonString(entries_[i].id) + ":" + q.value();
+ }
+ return Result<std::string>::Ok(b + "}");
+}
+
+// ---- answers and prediction --------------------------------------------------------------------
+
+namespace {
+
+Result<std::string> Float(double f) {
+ if (std::isnan(f) || std::isinf(f)) return Refuse<std::string>("schema: cannot marshal " + pyjson::ReprFloat(f) + " as JSON");
+ return Result<std::string>::Ok(pyjson::ReprFloat(f));
+}
+
+#define LAYA_TRY_APPEND(out, expr) \
+ do { \
+ auto r_ = (expr); \
+ if (!r_.ok()) return r_; \
+ (out) += r_.value(); \
+ } while (0)
+
+Result<std::string> Probabilities(const std::vector<Probability>& ps) {
+ std::string b = "{";
+ for (size_t i = 0; i < ps.size(); i++) {
+ b += (i ? "," : "") + JsonString(ps[i].label.Key()) + ":";
+ LAYA_TRY_APPEND(b, Float(ps[i].p));
+ }
+ return Result<std::string>::Ok(b + "}");
+}
+
+} // namespace
+
+Result<std::string> Answer::ToJson() const {
+ std::string b;
+ switch (type) {
+ case Type::Choice:
+ b = "{\"type\":\"choice\",\"choice\":" + choice.Json() + ",\"probabilities\":";
+ LAYA_TRY_APPEND(b, Probabilities(probabilities));
+ break;
+ case Type::Score:
+ b = "{\"type\":\"score\",\"score\":";
+ LAYA_TRY_APPEND(b, Float(score));
+ b += ",\"legend\":{";
+ for (size_t i = 0; i < legend.size(); i++) b += (i ? ",\"" : "\"") + N(int64_t(i)) + "\":" + JsonString(legend[i]);
+ b += "},\"probabilities\":";
+ LAYA_TRY_APPEND(b, Probabilities(probabilities));
+ break;
+ case Type::Noul:
+ b = "{\"type\":\"noul\",\"noul\":";
+ LAYA_TRY_APPEND(b, Float(noul));
+ break;
+ case Type::Invalid: return Refuse<std::string>("schema: cannot marshal an answer of no type");
+ }
+ b += ",\"confidence\":";
+ LAYA_TRY_APPEND(b, Float(confidence));
+ b += ",\"answer_confidence\":";
+ LAYA_TRY_APPEND(b, Float(answer_confidence));
+ b += ",\"action\":{\"act_probability\":";
+ LAYA_TRY_APPEND(b, Float(act_probability));
+ b += "}";
+ if (window) {
+ b += ",\"window\":{\"index\":" + N(window->index) + ",\"token_start\":" + N(window->token_start) +
+ ",\"token_end\":" + N(window->token_end) + ",\"count\":" + N(window->count) + "}";
+ }
+ if (low_confidence) b += ",\"low_confidence\":true";
+ if (!abstention.empty()) b += ",\"abstention\":" + JsonString(abstention);
+ if (abstention_threshold) {
+ b += ",\"abstention_threshold\":";
+ LAYA_TRY_APPEND(b, Float(*abstention_threshold));
+ }
+ return Result<std::string>::Ok(b + "}");
+}
+
+namespace {
+
+Result<double> Number(const Value& v, const std::string& what) {
+ switch (v.kind()) {
+ case Kind::Int: return Result<double>::Ok(double(v.as_int()));
+ case Kind::BigInt: return Result<double>::Ok(std::strtod(v.big_int().c_str(), nullptr));
+ case Kind::Float: return Result<double>::Ok(v.as_float());
+ default: break;
+ }
+ return Result<double>::Fail("schema: \"" + what + "\" must be a number, got " + pyvalue::TypeName(v));
+}
+
+Result<const Value*> Field(const Value& obj, const char* key) {
+ const Value* v = obj.Find(key);
+ if (v == nullptr) return Result<const Value*>::Fail(std::string("schema: answer has no \"") + key + "\"");
+ return Result<const Value*>::Ok(v);
+}
+
+Result<double> FieldNumber(const Value& obj, const char* key) {
+ auto v = Field(obj, key);
+ if (!v.ok()) return Result<double>::Fail(v.error());
+ return Number(*v.value(), key);
+}
+
+Result<std::vector<Probability>> ProbsFrom(const Value& v) {
+ using R = Result<std::vector<Probability>>;
+ if (v.kind() != Kind::Object) return R::Fail("schema: probabilities must be an object, got " + pyvalue::TypeName(v));
+ std::vector<Probability> out;
+ for (size_t i = 0; i < v.size(); i++) {
+ auto p = Number(v.value(i), v.key(i));
+ if (!p.ok()) return R::Fail(p.error());
+ out.push_back({Label::Text(v.key(i)), p.value()});
+ }
+ return R::Ok(std::move(out));
+}
+
+} // namespace
+
+Result<Answer> Answer::FromJson(const Value& v) {
+ using R = Result<Answer>;
+ if (v.kind() != Kind::Object) return R::Fail("schema: an answer is a JSON object, got " + pyvalue::TypeName(v));
+ Answer a;
+ auto tv = Field(v, "type");
+ if (!tv.ok()) return R::Fail(tv.error());
+ if (tv.value()->kind() != Kind::String) {
+ return R::Fail("schema: answer \"type\" must be a string, got " + pyvalue::TypeName(*tv.value()));
+ }
+ const std::string& ts = tv.value()->as_string();
+ a.type = ts == "choice" ? Type::Choice : ts == "score" ? Type::Score : ts == "noul" ? Type::Noul : Type::Invalid;
+ auto conf = FieldNumber(v, "confidence");
+ if (!conf.ok()) return R::Fail(conf.error());
+ a.confidence = conf.value();
+ auto ac = FieldNumber(v, "answer_confidence");
+ if (!ac.ok()) return R::Fail(ac.error());
+ a.answer_confidence = ac.value();
+ auto act = Field(v, "action");
+ if (!act.ok()) return R::Fail(act.error());
+ if (act.value()->kind() != Kind::Object) return R::Fail("schema: answer \"action\" must be an object");
+ auto ap = FieldNumber(*act.value(), "act_probability");
+ if (!ap.ok()) return R::Fail(ap.error());
+ a.act_probability = ap.value();
+ if (const Value* w = v.Find("window")) {
+ if (w->kind() != Kind::Object) return R::Fail("schema: answer \"window\" must be an object");
+ int64_t f[4];
+ const char* keys[4] = {"index", "token_start", "token_end", "count"};
+ for (int i = 0; i < 4; i++) {
+ auto n = FieldNumber(*w, keys[i]);
+ if (!n.ok() || n.value() != std::trunc(n.value()) || std::fabs(n.value()) > 9.0e15) {
+ return R::Fail(std::string("schema: answer window \"") + keys[i] + "\" must be an integer");
+ }
+ f[i] = int64_t(n.value());
+ }
+ if (f[3] < 1 || f[0] < 0 || f[0] >= f[3] || f[1] < 0 || f[2] < f[1]) return R::Fail("schema: answer \"window\" is out of range");
+ a.window = Window{f[0], f[1], f[2], f[3]};
+ }
+ if (const Value* lc = v.Find("low_confidence")) {
+ if (lc->kind() != Kind::Bool || !lc->as_bool()) return R::Fail("schema: answer \"low_confidence\" must be true when present");
+ a.low_confidence = true;
+ }
+ if (const Value* ab = v.Find("abstention")) {
+ if (ab->kind() != Kind::String || (ab->as_string() != "passed" && ab->as_string() != "abstained" && ab->as_string() != "unevaluated")) {
+ return R::Fail("schema: answer \"abstention\" must be one of \"passed\", \"abstained\", \"unevaluated\", got " + pyvalue::Repr(*ab));
+ }
+ a.abstention = ab->as_string();
+ }
+ if (v.Find("abstention_threshold") != nullptr) {
+ auto t = FieldNumber(v, "abstention_threshold");
+ if (!t.ok()) return R::Fail(t.error());
+ a.abstention_threshold = t.value();
+ }
+ if (a.abstention.empty() != !a.abstention_threshold.has_value()) {
+ return R::Fail("schema: answer \"abstention\" and \"abstention_threshold\" come together");
+ }
+ if (a.type == Type::Choice) {
+ auto c = Field(v, "choice");
+ if (!c.ok()) return R::Fail(c.error());
+ auto l = Label::Of(*c.value());
+ if (!l.ok()) return R::Fail("schema: answer \"choice\" must be a string, bool or number, got " + pyvalue::TypeName(*c.value()));
+ a.choice = l.value();
+ } else if (a.type == Type::Score) {
+ auto s = FieldNumber(v, "score");
+ if (!s.ok()) return R::Fail(s.error());
+ a.score = s.value();
+ auto lg = Field(v, "legend");
+ if (!lg.ok()) return R::Fail(lg.error());
+ const Value& lo = *lg.value();
+ if (lo.kind() != Kind::Object) return R::Fail("schema: answer \"legend\" must be an object");
+ for (size_t i = 0; i < lo.size(); i++) {
+ if (lo.value(i).kind() != Kind::String || lo.key(i) != N(int64_t(i))) {
+ return R::Fail("schema: answer \"legend\" must map \"0\", \"1\", ... to text, got key " + lo.key(i));
+ }
+ a.legend.push_back(lo.value(i).as_string());
+ }
+ } else if (a.type == Type::Noul) {
+ auto n = FieldNumber(v, "noul");
+ if (!n.ok()) return R::Fail(n.error());
+ a.noul = n.value();
+ } else {
+ return R::Fail("schema: unknown answer type " + pyjson::Repr(ts));
+ }
+ if (a.type != Type::Noul) {
+ auto p = Field(v, "probabilities");
+ if (!p.ok()) return R::Fail(p.error());
+ auto ps = ProbsFrom(*p.value());
+ if (!ps.ok()) return R::Fail(ps.error());
+ a.probabilities = std::move(ps).value();
+ }
+ return R::Ok(std::move(a));
+}
+
+Result<std::string> Usage::ToJson() const {
+ bool scanned = windows > 1;
+ if (scanned && state_tokens == 0 && state_tokens_dropped == 0 && truncated_windows == 0 &&
+ truncated_questions.empty() && collapsed_options.empty()) {
+ return Result<std::string>::Ok("{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) +
+ ",\"windows\":" + N(windows) + "}");
+ }
+ std::string b = "{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) +
+ ",\"state_tokens\":" + N(state_tokens) + ",\"state_tokens_dropped\":" + N(state_tokens_dropped) +
+ ",\"truncated\":" + (scanned ? N(truncated_windows) : std::string(truncated ? "true" : "false")) +
+ ",\"truncated_questions\":[";
+ for (size_t i = 0; i < truncated_questions.size(); i++) b += (i ? "," : "") + JsonString(truncated_questions[i]);
+ b += "]";
+ if (!collapsed_options.empty()) {
+ b += ",\"options\":{";
+ for (size_t i = 0; i < collapsed_options.size(); i++) {
+ const auto& c = collapsed_options[i];
+ b += (i ? "," : "") + JsonString(c.id) + ":{\"total\":" + N(c.options.total) + ",\"distinct\":" +
+ N(c.options.distinct) + ",\"tokens_per_option\":" +
+ (c.options.tokens_per_option ? N(*c.options.tokens_per_option) : std::string("null")) + "}";
+ }
+ b += "}";
+ }
+ if (windows > 0) b += ",\"windows\":" + N(windows);
+ return Result<std::string>::Ok(b + "}");
+}
+
+Result<std::string> Prediction::ToJson() const {
+ std::string b = "{\"model\":" + JsonString(model) + ",\"answers\":{";
+ for (size_t i = 0; i < answers.size(); i++) {
+ auto a = answers[i].answer.ToJson();
+ if (!a.ok()) return a;
+ b += (i ? "," : "") + JsonString(answers[i].id) + ":" + a.value();
+ }
+ b += "},\"usage\":";
+ LAYA_TRY_APPEND(b, usage.ToJson());
+ return Result<std::string>::Ok(b + "}");
+}
+
+} // namespace schema
+} // namespace laya
diff --git a/laya-cpp/src/sequence.cpp b/laya-cpp/src/sequence.cpp
new file mode 100644
index 00000000..ae10bc95
--- /dev/null
+++ b/laya-cpp/src/sequence.cpp
@@ -0,0 +1,279 @@
+#include "laya/sequence.hpp"
+
+#include <algorithm>
+#include <cstdio>
+#include <set>
+
+#include "laya/pyjson.hpp"
+
+namespace laya {
+namespace sequence {
+
+using schema::Question;
+using tokenizer::Tokenizer;
+
+namespace {
+
+std::string N(int64_t i) { return std::to_string(i); }
+
+std::string ReplaceAll(std::string s, const std::string& from, const std::string& to) {
+ if (from.empty()) return s;
+ std::string out;
+ for (size_t at = 0;;) {
+ size_t hit = s.find(from, at);
+ if (hit == std::string::npos) {
+ out.append(s, at, std::string::npos);
+ return out;
+ }
+ out.append(s, at, hit - at);
+ out += to;
+ at = hit + from.size();
+ }
+}
+
+struct Specials {
+ int32_t cls = 0, sep = 0, mask = 0;
+ std::string mask_token;
+};
+
+Result<Specials> GetSpecials(const Tokenizer& tok) {
+ Specials sp;
+ auto cls = tok.Cls(), sep = tok.Sep(), mask = tok.Mask();
+ sp.mask_token = tok.MaskToken();
+ const char* what = "tokenizer names no required special token";
+ if (!cls) return Result<Specials>::Fail(std::string(what) + ": no CLS token");
+ if (!sep) return Result<Specials>::Fail(std::string(what) + ": no SEP token");
+ if (!mask || sp.mask_token.empty()) return Result<Specials>::Fail(std::string(what) + ": no mask token");
+ sp.cls = *cls;
+ sp.sep = *sep;
+ sp.mask = *mask;
+ return Result<Specials>::Ok(std::move(sp));
+}
+
+int64_t FloorDiv(int64_t a, int64_t b) {
+ int64_t q = a / b;
+ if (a % b != 0 && a < 0) q--;
+ return q;
+}
+
+bool CheckBudget(const Budget& b, std::string* err) {
+ if (b.max_len <= 0 || b.head_max_len <= 0) {
+ *err = "sequence: budget needs a positive MaxLen and HeadMaxLen, got " + N(b.max_len) + " and " + N(b.head_max_len);
+ return false;
+ }
+ return true;
+}
+
+} // namespace
+
+int64_t Budget::DefaultWindow() const { return std::max<int64_t>(64, max_len - head_max_len - 8); }
+
+Result<std::string> SerializeState(const pyvalue::Value& state) {
+ if (state.kind() == pyvalue::Value::Kind::String) return Result<std::string>::Ok(state.as_string());
+ auto text = pyvalue::Dumps(state);
+ if (!text.ok()) return Result<std::string>::Fail("state cannot be serialized as laya serializes it: " + text.error());
+ return text;
+}
+
+Result<State> EncodeState(const Tokenizer& tok, const pyvalue::Value& state) {
+ auto sp = GetSpecials(tok);
+ if (!sp.ok()) return Result<State>::Fail(sp.error());
+ auto text = SerializeState(state);
+ if (!text.ok()) return Result<State>::Fail(text.error());
+ State out;
+ out.keep = state.kind() == pyvalue::Value::Kind::Array ? Keep::End : Keep::Start;
+ out.ids = tok.Encode(ReplaceAll(text.value(), sp.value().mask_token, " "));
+ return Result<State>::Ok(std::move(out));
+}
+
+std::string MarkersError::Message(const std::string& id) const {
+ std::string prefix = id.empty() ? "" : "question " + pyjson::Repr(id) + ": ";
+ return prefix + "only " + N(kept) + " of its " + N(total) + " option markers fit in max_len=" + N(max_len) +
+ " with head_max_len=" + N(head_max_len) +
+ " spent on the question; lower head_max_len, raise max_len, or use fewer options";
+}
+
+Result<Head> BuildHead(const Tokenizer& tok, const Question& q, int64_t head_max_len) {
+ auto spr = GetSpecials(tok);
+ if (!spr.ok()) return Result<Head>::Fail(spr.error());
+ const Specials& sp = spr.value();
+ if (q.type() == schema::Type::Invalid || q.NumOptions() == 0) {
+ return Result<Head>::Fail(
+ "sequence: the zero Question has no head; build one with Question::Choice, ChoiceOf, Score or Noul");
+ }
+ std::vector<std::string> opts = q.RenderedOptions();
+ std::vector<int> order = q.OptionOrder();
+ if (order.empty()) {
+ for (size_t i = 0; i < opts.size(); i++) order.push_back(int(i));
+ }
+ std::string ins = ReplaceAll(q.instructions(), sp.mask_token, " ");
+ std::vector<int32_t> head_ids = tok.Encode(std::string(schema::Name(q.type())) + " question: " + ins);
+ std::vector<std::vector<int32_t>> spans(order.size());
+ int64_t total = 0;
+ for (size_t slot = 0; slot < order.size(); slot++) {
+ std::string text = " " + ReplaceAll(opts[size_t(order[slot])], sp.mask_token, " ");
+ spans[slot].push_back(sp.mask);
+ std::vector<int32_t> body = tok.Encode(text, kOptionTokenCap);
+ spans[slot].insert(spans[slot].end(), body.begin(), body.end());
+ total += int64_t(spans[slot].size());
+ }
+ int64_t budget = head_max_len - total;
+ std::optional<int64_t> per_option;
+ if (budget < kMinOptionBudget) {
+ int64_t per = std::max(kMinTokensPerOption,
+ FloorDiv(head_max_len - kMinOptionBudget, std::max<int64_t>(1, int64_t(spans.size()))));
+ per_option = per;
+ total = 0;
+ for (auto& span : spans) {
+ if (int64_t(span.size()) > per) span.resize(size_t(per));
+ total += int64_t(span.size());
+ }
+ budget = head_max_len - total;
+ }
+ int64_t keep = std::max(kMinInstructionTokens, budget);
+ if (keep < int64_t(head_ids.size())) head_ids.resize(size_t(keep));
+
+ Head h;
+ h.ids.push_back(sp.cls);
+ h.ids.insert(h.ids.end(), head_ids.begin(), head_ids.end());
+ h.ids.push_back(sp.sep);
+ std::set<std::vector<int32_t>> distinct;
+ for (const auto& span : spans) {
+ h.markers.push_back(int64_t(h.ids.size()));
+ h.ids.insert(h.ids.end(), span.begin(), span.end());
+ distinct.insert(span);
+ }
+ h.ids.push_back(sp.sep);
+ h.stats = {int64_t(spans.size()), int64_t(distinct.size()), per_option};
+ return Result<Head>::Ok(std::move(h));
+}
+
+Result<Sequence> Build(const Tokenizer& tok, const Question& q, const State& state, const Budget& b,
+ MarkersError* markers_error) {
+ std::string err;
+ if (!CheckBudget(b, &err)) return Result<Sequence>::Fail(err);
+ auto head = BuildHead(tok, q, b.head_max_len);
+ if (!head.ok()) return Result<Sequence>::Fail(head.error());
+ const Head& h = head.value();
+ int32_t sep = *tok.Sep(); // BuildHead found it
+ int64_t room = std::max<int64_t>(0, b.max_len - int64_t(h.ids.size()) - 1);
+ int64_t have = int64_t(state.ids.size());
+ int64_t used = std::min(have, room);
+ Sequence s;
+ s.ids = h.ids;
+ size_t from = state.keep == Keep::End ? size_t(have - used) : 0;
+ s.ids.insert(s.ids.end(), state.ids.begin() + std::ptrdiff_t(from), state.ids.begin() + std::ptrdiff_t(from + size_t(used)));
+ s.ids.push_back(sep);
+ if (int64_t(s.ids.size()) > b.max_len) s.ids.resize(size_t(b.max_len));
+ for (int64_t m : h.markers) {
+ if (m < b.max_len) s.markers.push_back(m);
+ }
+ if (s.markers.size() != h.markers.size()) {
+ MarkersError e{int64_t(s.markers.size()), int64_t(h.markers.size()), b.max_len, b.head_max_len};
+ if (markers_error != nullptr) *markers_error = e;
+ return Result<Sequence>::Fail(e.Message());
+ }
+ s.stats = h.stats;
+ s.truncation = {have, used, have - used, used < have};
+ return Result<Sequence>::Ok(std::move(s));
+}
+
+Result<int64_t> StateRoom(const Tokenizer& tok, const Question& q, const Budget& b) {
+ std::string err;
+ if (!CheckBudget(b, &err)) return Result<int64_t>::Fail(err);
+ auto head = BuildHead(tok, q, b.head_max_len);
+ if (!head.ok()) return Result<int64_t>::Fail(head.error());
+ return Result<int64_t>::Ok(std::max<int64_t>(0, b.max_len - int64_t(head.value().ids.size()) - 1)); // -1: closing [SEP]
+}
+
+namespace {
+constexpr int64_t kClampWarnRatio = 2;
+constexpr int64_t kBatchBlowup = 2;
+
+std::string Ratio1(double x) {
+ char buf[48];
+ std::snprintf(buf, sizeof buf, "%.1f", x);
+ return buf;
+}
+} // namespace
+
+Result<WindowPlan> PlanWindows(const Tokenizer& tok, const std::vector<Question>& questions, const Budget& b,
+ const WindowRequest& req, PlanError* kind) {
+ if (kind != nullptr) *kind = PlanError::Other;
+ std::string err;
+ if (!CheckBudget(b, &err)) return Result<WindowPlan>::Fail(err);
+ std::vector<int64_t> rooms;
+ for (const auto& q : questions) {
+ auto r = StateRoom(tok, q, b);
+ if (!r.ok()) return Result<WindowPlan>::Fail(r.error());
+ rooms.push_back(r.value());
+ }
+ int64_t requested = req.window > 0 ? req.window : b.DefaultWindow();
+ int64_t size = requested;
+ int64_t room = rooms.empty() ? size : *std::min_element(rooms.begin(), rooms.end());
+ if (room <= 0) {
+ if (kind != nullptr) *kind = PlanError::NoRoom;
+ return Result<WindowPlan>::Fail(
+ "predict_long: the questions' options fill the whole sequence (max_len=" + N(b.max_len) + ", head_max_len=" +
+ N(b.head_max_len) + "), leaving no room for the state; no window can carry any of it. A label set this large "
+ "is what laya.shortlist.predict_shortlist is for");
+ }
+ WindowPlan plan;
+ if (size > room) {
+ if (req.window > 0) {
+ plan.warnings.push_back("laya: predict_long: window=" + N(size) + " is wider than the " + N(room) +
+ " state tokens these questions leave inside max_len=" + N(b.max_len) +
+ ", so every window would be truncated to " + N(room) +
+ " on the way to the model; scanning with window=" + N(room) + " instead");
+ } else if (size / kClampWarnRatio >= room) { // size >= room * kClampWarnRatio, without the overflow
+ plan.warnings.push_back("laya: predict_long: these questions leave only " + N(room) + " of max_len=" +
+ N(b.max_len) + " for the state (their heads take the rest), so the scan window is "
+ "capped " + N(size) + " -> " + N(room) + " and roughly " +
+ Ratio1(double(size) / double(room)) + "x as many windows -- each a full forward pass -- "
+ "are needed to read the document. Fewer or shorter options, a larger max_len, or "
+ "laya.shortlist.predict_shortlist for a large label set will all cost less than scanning "
+ "at this width");
+ }
+ size = room;
+ }
+ int64_t step = req.stride > 0 ? req.stride : std::max<int64_t>(1, size / 2);
+ if (step > size) {
+ if (size < requested && req.stride > 0 && req.stride <= requested) {
+ plan.warnings.push_back("laya: predict_long: stride=" + N(step) + " was a 50% step for the window=" +
+ N(requested) + " you asked for, but the window was reduced to " + N(size) +
+ " to fit the room these questions leave; scanning with stride=" +
+ N(std::max<int64_t>(1, size / 2)) + " instead");
+ step = std::max<int64_t>(1, size / 2);
+ } else {
+ if (kind != nullptr) *kind = PlanError::StrideGap;
+ return Result<WindowPlan>::Fail(
+ "predict_long: stride=" + N(step) + " steps past the " + N(size) + "-token window" +
+ (size < requested ? " these questions leave room for" : "") + ", so " + N(step - size) +
+ " tokens between every pair of windows would be read by no window at all; pass stride <= " + N(size));
+ }
+ }
+ plan.window = size;
+ plan.stride = step;
+ plan.room = room;
+ if (kind != nullptr) *kind = PlanError::None;
+ return Result<WindowPlan>::Ok(std::move(plan));
+}
+
+Cap BatchCap(int64_t n_windows, int64_t window, int64_t config_window, int64_t batch_size) {
+ if (batch_size > 0) return {batch_size, true};
+ if (window >= config_window) return {0, false}; // not capped: one pass, exactly as before
+#if defined(__SIZEOF_INT128__)
+ __extension__ typedef __int128 Wide;
+#else
+ typedef long long Wide; // no 128-bit integer: callers beyond 2^31 windows are out of contract
+#endif
+ Wide num = Wide(n_windows) * Wide(std::max<int64_t>(1, FloorDiv(window, 2)));
+ Wide den = Wide(std::max<int64_t>(1, FloorDiv(config_window, 2)));
+ Wide unclamped = num > 0 ? (num + den - 1) / den : 1; // ceil(num / den), at least 1
+ if (unclamped < 1) unclamped = 1;
+ if (Wide(n_windows) <= unclamped * kBatchBlowup) return {0, false}; // a mild cap: keep the single shared pass
+ return {int64_t(unclamped), true}; // below n_windows, so it fits
+}
+
+} // namespace sequence
+} // namespace laya
diff --git a/laya-cpp/tests/checkpoint.hpp b/laya-cpp/tests/checkpoint.hpp
new file mode 100644
index 00000000..66048cf1
--- /dev/null
+++ b/laya-cpp/tests/checkpoint.hpp
@@ -0,0 +1,43 @@
+#pragma once
+// A shipped checkpoint's tokenizer, loaded once. Without LAYA_CHECKPOINTS the test is skipped
+// locally and FAILS under LAYA_REQUIRE_GOLDENS=1 (CI), so a missing checkpoint cannot pass as green.
+#include <gtest/gtest.h>
+
+#include <cstdlib>
+#include <map>
+#include <mutex>
+#include <string>
+
+#include "laya/tokenizer.hpp"
+
+namespace laya_test {
+
+inline const laya::tokenizer::Tokenizer* LoadCheckpoint(const std::string& name) {
+ static std::mutex mu;
+ static std::map<std::string, laya::tokenizer::Tokenizer> loaded;
+ std::lock_guard<std::mutex> lock(mu);
+ auto it = loaded.find(name);
+ if (it != loaded.end()) return &it->second;
+ const char* root = std::getenv("LAYA_CHECKPOINTS");
+ if (root == nullptr || *root == 0) return nullptr;
+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(root) + "/" + name);
+ if (!t.ok()) {
+ ADD_FAILURE() << "load the " << name << " tokenizer: " << t.error();
+ return nullptr;
+ }
+ return &loaded.emplace(name, std::move(t).value()).first->second;
+}
+
+#define LAYA_CHECKPOINT_TOK(var, name) \
+ const laya::tokenizer::Tokenizer* var = laya_test::LoadCheckpoint(name); \
+ if (var == nullptr) { \
+ const char* req = std::getenv("LAYA_REQUIRE_GOLDENS"); \
+ if (req != nullptr && std::string(req) == "1" && !HasFailure()) { \
+ ADD_FAILURE() << "LAYA_CHECKPOINTS is unset but LAYA_REQUIRE_GOLDENS=1"; \
+ } else if (!HasFailure()) { \
+ GTEST_SKIP() << "LAYA_CHECKPOINTS unset (local run)"; \
+ } \
+ return; \
+ }
+
+} // namespace laya_test
diff --git a/laya-cpp/tests/schema_test.cpp b/laya-cpp/tests/schema_test.cpp
new file mode 100644
index 00000000..b6a5e6d0
--- /dev/null
+++ b/laya-cpp/tests/schema_test.cpp
@@ -0,0 +1,249 @@
+#include "laya/schema.hpp"
+
+#include <gtest/gtest.h>
+
+#include "golden.hpp"
+#include "laya/pyjson.hpp"
+
+namespace {
+
+using laya::pyvalue::Value;
+namespace sc = laya::schema;
+using laya_test::At;
+using Kind = Value::Kind;
+
+std::string Dump(const Value& v) {
+ auto r = laya::pyvalue::Dumps(v);
+ return r.ok() ? r.value() : "<" + r.error() + ">";
+}
+
+std::vector<std::string> Strings(const Value& list) {
+ std::vector<std::string> out;
+ for (const Value& v : list.items()) out.push_back(v.as_string());
+ return out;
+}
+
+void CheckQuestion(const sc::Question& q, const Value& rec, const std::string& what) {
+ EXPECT_EQ(std::string(sc::Name(q.type())), rec.Find("type")->as_string()) << what;
+ EXPECT_EQ(q.instructions(), rec.Find("instructions")->as_string()) << what;
+ EXPECT_EQ(q.RenderedOptions(), Strings(*rec.Find("options"))) << what;
+ EXPECT_EQ(q.Labels(), Strings(*rec.Find("labels"))) << what;
+ if (const Value* lg = rec.Find("legend"); lg != nullptr && !lg->is_null()) EXPECT_EQ(q.Legend(), Strings(*lg)) << what;
+ std::vector<int> want;
+ if (const Value* o = rec.Find("option_order"); o != nullptr && !o->is_null()) {
+ for (const Value& v : o->items()) want.push_back(int(v.as_int()));
+ }
+ EXPECT_EQ(q.OptionOrder(), want) << what;
+}
+
+class QuestionRow : public ::testing::TestWithParam<int> {};
+TEST_P(QuestionRow, MatchesAgentCheckQuestion) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");
+ LAYA_EXPECT_ROWS(rows, 140);
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {
+ const Value& rec = rows->value(i);
+ const std::string name = rec.Find("case")->as_string();
+ const std::string id = rec.Find("id")->as_string();
+ Value doc = Value::Object();
+ doc.Set(id, *rec.Find("definition"));
+ auto qs = sc::Questions::Parse(Dump(doc));
+ if (const Value* want = rec.Find("error")) {
+ ASSERT_FALSE(qs.ok()) << name << ": accepted; Python refuses with " << want->as_string();
+ EXPECT_EQ(qs.error(), want->as_string()) << name;
+ continue;
+ }
+ ASSERT_TRUE(qs.ok()) << name << ": " << qs.error();
+ const sc::Question* q = qs.value().Get(id);
+ ASSERT_NE(q, nullptr) << name;
+ ASSERT_EQ(qs.value().size(), 1u) << name;
+ CheckQuestion(*q, rec, name);
+ auto out = qs.value().ToJson();
+ ASSERT_TRUE(out.ok()) << name << ": " << out.error();
+ auto again = sc::Questions::Parse(out.value());
+ ASSERT_TRUE(again.ok()) << name << ": own output " << out.value() << ": " << again.error();
+ CheckQuestion(*again.value().Get(id), rec, name + " (round trip)");
+ EXPECT_EQ(again.value().ToJson().value(), out.value()) << name << ": marshal is not stable";
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, QuestionRow, ::testing::ValuesIn(laya_test::Chunks(140)));
+
+TEST(Questions, RefusalAndAcceptanceCounts) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");
+ int refused = 0, accepted = 0;
+ for (const Value& rec : rows->items()) (rec.Find("error") ? refused : accepted)++;
+ EXPECT_EQ(refused, 93);
+ EXPECT_EQ(accepted, 47);
+}
+
+// The typed constructors refuse with the wire reader's words.
+TEST(Builders, ReasonsMatchPython) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");
+ std::map<std::string, std::string> gold;
+ const std::string prefix = "question 'q': ";
+ for (const Value& rec : rows->items()) {
+ if (const Value* e = rec.Find("error")) gold[rec.Find("case")->as_string()] = e->as_string().substr(prefix.size());
+ }
+ auto two = sc::Question::ChoiceOf("Pick.", {"a", "b"}).value();
+ auto noul = sc::Question::Noul("Holds?").value();
+ auto spec = [](const char* f, const char* t) {
+ sc::NoulSpec s;
+ s.false_label = f;
+ s.true_label = t;
+ return s;
+ };
+ struct Row {
+ const char* gold;
+ std::string got;
+ };
+ auto err = [](const auto& r) { return r.ok() ? std::string("<accepted>") : r.error(); };
+ std::vector<Row> cases = {
+ {"ins-blank", err(sc::Question::ChoiceOf(" \t\n", {"a", "b"}))},
+ {"ins-nbsp", err(sc::Question::Score(" \xc2\x85", {Value::String("a")}))},
+ {"ins-ideographic-space", err(sc::Question::Noul("\xe3\x80\x80"))},
+ {"ins-empty", err(sc::Question::Choice("", {{sc::Label::Text("a"), Value()}}))},
+ {"choice-empty-list", err(sc::Question::Choice("Pick.", {}))},
+ {"choice-list-repeat", err(sc::Question::ChoiceOf("Pick.", {"a", "b", "a"}))},
+ {"score-empty", err(sc::Question::Score("Rate.", {}))},
+ {"score-null-second", err(sc::Question::Score("Rate.", {Value::String("a"), Value(), Value::String("c")}))},
+ {"labels-same", err(sc::Question::Noul("Holds?", spec("a", "a")))},
+ {"labels-same-after-strip", err(sc::Question::Noul("Holds?", spec("a", " a ")))},
+ {"labels-missing-true", err(sc::Question::Noul("Holds?", spec("a", "")))},
+ {"labels-blank-value", err(sc::Question::Noul("Holds?", spec(" ", "x")))},
+ {"order-short", err(two.WithOptionOrder({0}))},
+ {"order-long", err(two.WithOptionOrder({0, 1, 2}))},
+ {"order-repeat", err(two.WithOptionOrder({0, 0}))},
+ {"order-range", err(two.WithOptionOrder({0, 2}))},
+ {"order-negative", err(two.WithOptionOrder({-1, 0}))},
+ {"order-empty", err(two.WithOptionOrder({}))},
+ {"order-noul-wrong-size", err(noul.WithOptionOrder({0, 1, 2}))},
+ };
+ for (const Row& c : cases) {
+ auto it = gold.find(c.gold);
+ ASSERT_NE(it, gold.end()) << c.gold;
+ EXPECT_EQ(c.got, it->second) << c.gold;
+ }
+}
+
+TEST(Builders, OptionOrderAndSubset) {
+ auto q = sc::Question::ChoiceOf("Pick.", {"a", "b", "c"}).value();
+ auto o = q.WithOptionOrder({2, 0, 1});
+ ASSERT_TRUE(o.ok());
+ EXPECT_EQ(o.value().OptionOrder(), (std::vector<int>{2, 0, 1}));
+ EXPECT_EQ(o.value().RenderedOptions(), q.RenderedOptions()); // rendering is label order; the order applies later
+ EXPECT_TRUE(o.value().WithoutOptionOrder().OptionOrder().empty());
+ EXPECT_TRUE(q.OptionOrder().empty()); // a question is immutable
+ auto s = o.value().Subset({2, 0});
+ ASSERT_TRUE(s.ok());
+ EXPECT_EQ(s.value().Labels(), (std::vector<std::string>{"c", "a"}));
+ EXPECT_TRUE(s.value().OptionOrder().empty());
+ EXPECT_FALSE(q.Subset({}).ok());
+ EXPECT_FALSE(q.Subset({0, 0}).ok());
+ EXPECT_FALSE(q.Subset({3}).ok());
+ EXPECT_FALSE(sc::Question().WithOptionOrder({0}).ok());
+ EXPECT_FALSE(sc::Question().ToJson().ok());
+}
+
+TEST(Labels, PythonDictKeying) {
+ using L = sc::Label;
+ EXPECT_EQ(L::OfInt(1).EqKey(), L::OfFloat(1.0).EqKey());
+ EXPECT_EQ(L::OfInt(1).EqKey(), L::OfBool(true).EqKey());
+ EXPECT_NE(L::OfInt(1).EqKey(), L::Text("1").EqKey());
+ EXPECT_NE(L::OfFloat(0.5).EqKey(), L::OfInt(0).EqKey());
+ EXPECT_EQ(L::OfFloat(1e22).EqKey(), "n:10000000000000000000000");
+ EXPECT_EQ(L::OfFloat(-0.0).EqKey(), L::OfInt(0).EqKey());
+ EXPECT_EQ(L::OfBool(true).String(), "True");
+ EXPECT_EQ(L::OfFloat(1.0 / 0.0).String(), "inf");
+ EXPECT_EQ(L::OfFloat(1.0 / 0.0).Key(), "Infinity");
+ EXPECT_FALSE(L::Of(Value()).ok());
+ EXPECT_FALSE(L::Of(Value::Array()).ok());
+ // 1, 1.0 and true are one dict key, so a question may not hold two of them.
+ auto q = sc::Question::Choice("Pick.", {{L::OfInt(1), Value()}, {L::OfFloat(1.0), Value()}});
+ ASSERT_FALSE(q.ok());
+ EXPECT_EQ(q.error(),
+ "choice label 1 (1.0) repeats label 0; the labels are the answer keys, so every option needs its own (1, "
+ "1.0 and True are one key)");
+}
+
+TEST(Questions, Duplicates) {
+ auto q = sc::Question::Noul("Holds?").value();
+ EXPECT_FALSE(sc::Questions::Of({{"a", q}, {"a", q}}).ok());
+ EXPECT_FALSE(sc::Questions::Of({{" ", q}}).ok());
+ EXPECT_FALSE(sc::Questions::Of({{"a", sc::Question()}}).ok());
+ EXPECT_EQ(sc::Questions::Of({{"b", q}, {"a", q}}).value().Ids(), (std::vector<std::string>{"b", "a"}));
+}
+
+// Answer marshals as Python writes it: parse the Python text, write it back, and get it byte for byte.
+class AnswerRow : public ::testing::TestWithParam<int> {};
+TEST_P(AnswerRow, RoundTripsPythonText) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "label_answers", "cases");
+ LAYA_EXPECT_ROWS(rows, 500);
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {
+ const std::string text = rows->value(i).Find("answer_text")->as_string();
+ auto v = laya::pyvalue::Parse(text);
+ ASSERT_TRUE(v.ok()) << i;
+ auto a = sc::Answer::FromJson(v.value());
+ ASSERT_TRUE(a.ok()) << i << ": " << a.error();
+ auto out = a.value().ToJson();
+ ASSERT_TRUE(out.ok()) << i << ": " << out.error();
+ // Python writes 1 and "1" as two keys of one object; a parsed object keeps one. Those rows are
+ // the ones with fewer probabilities than options and are not byte-comparable after a parse.
+ const Value* opts = rows->value(i).Find("options");
+ if (a.value().type == sc::Type::Choice && a.value().probabilities.size() < opts->size()) continue;
+ EXPECT_EQ(out.value(), text) << "row " << i;
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, AnswerRow, ::testing::ValuesIn(laya_test::Chunks(500)));
+
+TEST(Answer, GateFieldsAndRefusals) {
+ sc::Answer a;
+ a.type = sc::Type::Noul;
+ a.noul = 0.5;
+ a.confidence = 0.25;
+ a.answer_confidence = 0.75;
+ a.act_probability = 1;
+ a.window = sc::Window{1, 10, 20, 3};
+ a.low_confidence = true;
+ a.abstention = "abstained";
+ a.abstention_threshold = 0.5;
+ const char* want =
+ "{\"type\":\"noul\",\"noul\":0.5,\"confidence\":0.25,\"answer_confidence\":0.75,\"action\":{\"act_probability\":1.0},"
+ "\"window\":{\"index\":1,\"token_start\":10,\"token_end\":20,\"count\":3},\"low_confidence\":true,"
+ "\"abstention\":\"abstained\",\"abstention_threshold\":0.5}";
+ EXPECT_EQ(a.ToJson().value(), want);
+ auto back = sc::Answer::FromJson(laya::pyvalue::Parse(want).value());
+ ASSERT_TRUE(back.ok()) << back.error();
+ EXPECT_EQ(back.value().ToJson().value(), want);
+ a.noul = 0.0 / 0.0; // NaN is not JSON
+ EXPECT_FALSE(a.ToJson().ok());
+ a.noul = 0.5;
+ a.abstention.clear(); // the gate fields come together
+ auto lone = a.ToJson();
+ ASSERT_TRUE(lone.ok());
+ EXPECT_FALSE(sc::Answer::FromJson(laya::pyvalue::Parse(lone.value()).value()).ok());
+ EXPECT_FALSE(sc::Answer().ToJson().ok());
+}
+
+TEST(Prediction, UsageShapes) {
+ sc::Prediction p;
+ p.usage.input_tokens = 10;
+ p.usage.output_tokens = 2;
+ p.usage.truncated_questions = {"q"};
+ p.usage.truncated = true;
+ p.usage.collapsed_options = {{"q", {3, 1, 4}}, {"r", {2, 2, std::nullopt}}};
+ EXPECT_EQ(p.ToJson().value(),
+ "{\"model\":\"laya-rl-agent-onnx\",\"answers\":{},\"usage\":{\"input_tokens\":10,\"output_tokens\":2,"
+ "\"state_tokens\":0,\"state_tokens_dropped\":0,\"truncated\":true,\"truncated_questions\":[\"q\"],"
+ "\"options\":{\"q\":{\"total\":3,\"distinct\":1,\"tokens_per_option\":4},\"r\":{\"total\":2,\"distinct\":2,"
+ "\"tokens_per_option\":null}}}}");
+ sc::Usage scan;
+ scan.input_tokens = 5;
+ scan.output_tokens = 1;
+ scan.windows = 3;
+ EXPECT_EQ(scan.ToJson().value(), "{\"input_tokens\":5,\"output_tokens\":1,\"windows\":3}");
+ scan.truncated_windows = 2;
+ EXPECT_EQ(scan.ToJson().value(),
+ "{\"input_tokens\":5,\"output_tokens\":1,\"state_tokens\":0,\"state_tokens_dropped\":0,\"truncated\":2,"
+ "\"truncated_questions\":[],\"windows\":3}");
+}
+
+} // namespace
diff --git a/laya-cpp/tests/sequence_test.cpp b/laya-cpp/tests/sequence_test.cpp
new file mode 100644
index 00000000..0a6afab5
--- /dev/null
+++ b/laya-cpp/tests/sequence_test.cpp
@@ -0,0 +1,342 @@
+#include "laya/sequence.hpp"
+
+#include <gtest/gtest.h>
+
+#include "checkpoint.hpp"
+#include "golden.hpp"
+
+namespace {
+
+using laya::pyvalue::Value;
+namespace sc = laya::schema;
+namespace sq = laya::sequence;
+using laya_test::At;
+using Kind = Value::Kind;
+
+const char* const kNames[] = {"english", "multilingual"};
+
+sc::Question GoldenQuestion(const Value& g, const Value& order) {
+ const std::string kind = g.Find("t")->as_string();
+ const std::string ins = g.Find("ins")->as_string();
+ const Value* crit = g.Find("crit");
+ laya::Result<sc::Question> q = laya::Result<sc::Question>::Fail("unset");
+ if (kind == "choice") {
+ std::vector<sc::Option> opts;
+ for (size_t i = 0; i < crit->size(); i++) opts.push_back({sc::Label::Text(crit->key(i)), crit->value(i)});
+ q = sc::Question::Choice(ins, opts);
+ } else if (kind == "score") {
+ q = sc::Question::Score(ins, crit->items());
+ } else {
+ sc::NoulSpec spec;
+ if (crit != nullptr && crit->kind() == Kind::Object) {
+ if (const Value* t = crit->Find("true")) spec.true_text = *t;
+ if (const Value* f = crit->Find("false")) spec.false_text = *f;
+ }
+ if (const Value* l = g.Find("labels"); l != nullptr && !l->is_null()) {
+ spec.false_label = l->Find("false")->as_string();
+ spec.true_label = l->Find("true")->as_string();
+ }
+ q = sc::Question::Noul(ins, spec);
+ }
+ EXPECT_TRUE(q.ok()) << (q.ok() ? "" : q.error());
+ if (!q.ok()) std::abort();
+ if (!order.is_null()) {
+ std::vector<int> o;
+ for (const Value& v : order.items()) o.push_back(int(v.as_int()));
+ q = q.value().WithOptionOrder(o);
+ EXPECT_TRUE(q.ok());
+ if (!q.ok()) std::abort();
+ }
+ return q.value();
+}
+
+std::vector<int32_t> Ids32(const Value& list) {
+ std::vector<int32_t> out;
+ for (const Value& v : list.items()) out.push_back(int32_t(v.as_int()));
+ return out;
+}
+std::vector<int64_t> Ids64(const Value& list) {
+ std::vector<int64_t> out;
+ for (const Value& v : list.items()) out.push_back(v.as_int());
+ return out;
+}
+
+// The generator's _window_question: options of four words each.
+sc::Question WindowQuestion(int options) {
+ std::vector<sc::Option> opts;
+ for (int i = 0; i < options; i++) {
+ std::string text;
+ for (int w = 0; w < 4; w++) text += (w ? " " : "") + std::string("criterion") + std::to_string(i) + " word" + std::to_string(w);
+ char label[16];
+ std::snprintf(label, sizeof label, "opt%03d", i);
+ opts.push_back({sc::Label::Text(label), Value::String(text)});
+ }
+ return sc::Question::Choice("Pick one.", opts).value();
+}
+
+class SequenceRow : public ::testing::TestWithParam<int> {};
+TEST_P(SequenceRow, MatchesBuildSequence) {
+ const char* name = kNames[GetParam() / 26];
+ LAYA_CHECKPOINT_TOK(tok, name);
+ LAYA_GOLDEN_LIST(cases, "sequences.json", name, "cases");
+ LAYA_EXPECT_ROWS(cases, 26);
+ const size_t at = size_t(GetParam() % 26);
+ const std::string label = std::string(name) + "/" + cases->key(at);
+ const Value& rec = cases->value(at);
+ sc::Question q = GoldenQuestion(*rec.Find("question"), *rec.Find("option_order"));
+ auto state = sq::EncodeState(*tok, *rec.Find("state"));
+ ASSERT_TRUE(state.ok()) << label << ": " << state.error();
+ if (rec.Find("truncate_left")->as_bool()) state.value().keep = sq::Keep::End;
+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};
+ sq::MarkersError dropped;
+ auto seq = sq::Build(*tok, q, state.value(), b, &dropped);
+ const auto want_markers = Ids64(*rec.Find("markers"));
+ const int64_t option_count = rec.Find("stats")->Find("options")->as_int();
+ if (!seq.ok() && dropped.total != 0) {
+ // build_sequence returns the markers that fit and the agent refuses; Build does both.
+ EXPECT_EQ(dropped.kept, int64_t(want_markers.size())) << label;
+ EXPECT_EQ(dropped.total, option_count) << label;
+ EXPECT_LT(int64_t(want_markers.size()), option_count) << label;
+ return;
+ }
+ ASSERT_TRUE(seq.ok()) << label << ": " << seq.error();
+ ASSERT_EQ(int64_t(want_markers.size()), option_count) << label << ": Python dropped markers and Build did not refuse";
+ EXPECT_EQ(seq.value().ids, Ids32(*rec.Find("ids"))) << label;
+ EXPECT_EQ(seq.value().markers, want_markers) << label;
+ const Value& stats = *rec.Find("stats");
+ EXPECT_EQ(seq.value().stats.options, stats.Find("options")->as_int()) << label;
+ EXPECT_EQ(seq.value().stats.options_distinct, stats.Find("options_distinct")->as_int()) << label;
+ const Value* tpo = stats.Find("tokens_per_option");
+ EXPECT_EQ(seq.value().stats.tokens_per_option.has_value(), !tpo->is_null()) << label;
+ if (!tpo->is_null() && seq.value().stats.tokens_per_option) EXPECT_EQ(*seq.value().stats.tokens_per_option, tpo->as_int()) << label;
+ const Value& tr = *rec.Find("truncation");
+ sq::Truncation want{tr.Find("state_tokens")->as_int(), tr.Find("state_tokens_used")->as_int(),
+ tr.Find("state_tokens_dropped")->as_int(), tr.Find("truncated")->as_bool()};
+ EXPECT_TRUE(seq.value().truncation == want) << label;
+}
+INSTANTIATE_TEST_SUITE_P(Golden, SequenceRow, ::testing::ValuesIn(laya_test::Rows(52)));
+
+class Checkpoint : public ::testing::TestWithParam<int> {};
+
+TEST_P(Checkpoint, SpecialIdsMatchPython) {
+ const char* name = kNames[GetParam()];
+ LAYA_CHECKPOINT_TOK(tok, name);
+ LAYA_GOLDEN_LIST(ids, "sequences.json", name, "special_ids");
+ EXPECT_EQ(int64_t(*tok->Cls()), ids->Find("cls")->as_int());
+ EXPECT_EQ(int64_t(*tok->Sep()), ids->Find("sep")->as_int());
+ EXPECT_EQ(int64_t(*tok->Mask()), ids->Find("mask")->as_int());
+ EXPECT_EQ(int64_t(*tok->Pad()), ids->Find("pad")->as_int());
+ EXPECT_EQ(tok->MaskToken(), ids->Find("mask_token")->as_string());
+}
+
+// The same questions and budgets succeed or fail together with Agent._encode_state, with the same text.
+TEST_P(Checkpoint, MarkerRefusalsMatchPython) {
+ const char* name = kNames[GetParam()];
+ LAYA_CHECKPOINT_TOK(tok, name);
+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "marker_refusals", name, "cases");
+ LAYA_EXPECT_ROWS(cases, 42);
+ int refused = 0;
+ for (const Value& rec : cases->items()) {
+ const int options = int(rec.Find("options")->as_int()), words = int(rec.Find("words")->as_int());
+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};
+ const std::string label = std::string(name) + "/" + std::to_string(options) + " options x " + std::to_string(words) +
+ " words/" + std::to_string(b.max_len) + "," + std::to_string(b.head_max_len);
+ std::vector<sc::Option> opts;
+ for (int i = 0; i < options; i++) {
+ std::string text;
+ for (int j = 0; j < words; j++) text += (j ? " " : "") + std::string("w") + std::to_string(i) + "_" + std::to_string(j);
+ char l[16];
+ std::snprintf(l, sizeof l, "opt%03d", i);
+ opts.push_back({sc::Label::Text(l), Value::String(text)});
+ }
+ auto q = sc::Question::Choice("Pick one.", opts);
+ ASSERT_TRUE(q.ok()) << label;
+ auto st = sq::EncodeState(*tok, Value::String("state text"));
+ ASSERT_TRUE(st.ok()) << label;
+ sq::MarkersError me;
+ auto seq = sq::Build(*tok, q.value(), st.value(), b, &me);
+ const Value* want = rec.Find("error");
+ if (want->is_null()) {
+ EXPECT_TRUE(seq.ok()) << label << ": refused: " << seq.error();
+ } else {
+ refused++;
+ ASSERT_FALSE(seq.ok()) << label << ": not refused; Python says " << want->as_string();
+ ASSERT_NE(me.total, 0) << label << ": " << seq.error();
+ EXPECT_EQ(me.Message("it's"), want->as_string()) << label;
+ }
+ }
+ EXPECT_GE(refused, 5) << name;
+}
+
+TEST_P(Checkpoint, WindowPlanMatchesPython) {
+ const char* name = kNames[GetParam()];
+ LAYA_CHECKPOINT_TOK(tok, name);
+ LAYA_GOLDEN_LIST(entry, "window_plan.json", "by", name);
+ const int64_t max_len = entry->Find("default_max_len")->as_int(), head_max = entry->Find("default_head_max_len")->as_int();
+ const Value& rooms = *entry->Find("state_room");
+ ASSERT_EQ(rooms.size(), 8u);
+ for (const Value& rec : rooms.items()) {
+ const int options = int(rec.Find("options")->as_int());
+ sc::Question q = WindowQuestion(options);
+ auto room = sq::StateRoom(*tok, q, {max_len, head_max});
+ ASSERT_TRUE(room.ok());
+ EXPECT_EQ(room.value(), rec.Find("room")->as_int()) << name << " room " << options;
+ auto tight = sq::StateRoom(*tok, q, {256, 128});
+ ASSERT_TRUE(tight.ok());
+ EXPECT_EQ(tight.value(), rec.Find("room_tight")->as_int()) << name << " room_tight " << options;
+ }
+ const Value& cases = *entry->Find("cases");
+ ASSERT_EQ(cases.size(), 24u);
+ int refusals = 0;
+ for (const Value& rec : cases.items()) {
+ const std::string label = std::string(name) + "/" + rec.Find("case")->as_string();
+ std::vector<sc::Question> qs;
+ for (const Value& n : rec.Find("options")->items()) qs.push_back(WindowQuestion(int(n.as_int())));
+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};
+ sq::WindowRequest req;
+ if (const Value* w = rec.Find("window"); !w->is_null()) req.window = w->as_int();
+ if (const Value* s = rec.Find("stride"); !s->is_null()) req.stride = s->as_int();
+ sq::PlanError kind = sq::PlanError::None;
+ auto plan = sq::PlanWindows(*tok, qs, b, req, &kind);
+ if (const Value* e = rec.Find("error")) {
+ refusals++;
+ ASSERT_FALSE(plan.ok()) << label << ": Python refuses with " << e->as_string();
+ EXPECT_EQ(plan.error(), e->as_string()) << label;
+ const bool no_room = e->as_string().find("fill the whole sequence") != std::string::npos;
+ EXPECT_EQ(kind, no_room ? sq::PlanError::NoRoom : sq::PlanError::StrideGap) << label;
+ continue;
+ }
+ ASSERT_TRUE(plan.ok()) << label << ": " << plan.error();
+ EXPECT_EQ(kind, sq::PlanError::None) << label;
+ const Value& res = *rec.Find("result");
+ EXPECT_EQ(plan.value().window, res.Find("window")->as_int()) << label;
+ EXPECT_EQ(plan.value().stride, res.Find("stride")->as_int()) << label;
+ EXPECT_EQ(plan.value().room, res.Find("room")->as_int()) << label;
+ std::vector<std::string> warnings;
+ for (const Value& w : rec.Find("warnings")->items()) warnings.push_back(w.as_string());
+ EXPECT_EQ(plan.value().warnings, warnings) << label;
+ EXPECT_EQ(plan.value().Clamped(), !warnings.empty()) << label;
+ }
+ EXPECT_GE(refusals, 1) << name;
+}
+INSTANTIATE_TEST_SUITE_P(Shipped, Checkpoint, ::testing::Values(0, 1));
+
+// A budget of exactly kMinOptionBudget left for the instruction leaves the option spans whole;
+// one token less cuts them.
+TEST(BuildHead, OptionBudgetBoundary) {
+ LAYA_CHECKPOINT_TOK(tok, "english");
+ std::vector<sc::Option> opts;
+ const char* texts[] = {"a", "a much longer description of the second option that has many words in it", "b c"};
+ for (int i = 0; i < 3; i++) opts.push_back({sc::Label::Text("o" + std::to_string(i)), Value::String(texts[i])});
+ auto q = sc::Question::Choice("Pick.", opts).value();
+ auto whole = sq::BuildHead(*tok, q, 10000).value();
+ const int64_t spans = int64_t(whole.ids.size()) - 1 - whole.markers[0];
+ auto at = sq::BuildHead(*tok, q, spans + sq::kMinOptionBudget).value();
+ EXPECT_FALSE(at.stats.tokens_per_option.has_value());
+ EXPECT_EQ(at.ids.size(), whole.ids.size());
+ auto below = sq::BuildHead(*tok, q, spans + sq::kMinOptionBudget - 1).value();
+ EXPECT_TRUE(below.stats.tokens_per_option.has_value());
+}
+
+// A list keeps its end when it is too long; a string or object keeps its start.
+TEST(EncodeState, KeepSide) {
+ LAYA_CHECKPOINT_TOK(tok, "english");
+ Value list = Value::Array();
+ list.Push(Value::String("x"));
+ Value obj = Value::Object();
+ obj.Set("k", Value::String("v"));
+ EXPECT_EQ(sq::EncodeState(*tok, list).value().keep, sq::Keep::End);
+ EXPECT_EQ(sq::EncodeState(*tok, obj).value().keep, sq::Keep::Start);
+ EXPECT_EQ(sq::EncodeState(*tok, Value::String("x")).value().keep, sq::Keep::Start);
+ EXPECT_EQ(sq::EncodeState(*tok, Value::Int(5)).value().keep, sq::Keep::Start);
+}
+
+// A default window of exactly twice the room is the ratio branch of the clamp warning.
+TEST(PlanWindows, ClampRatioBoundary) {
+ LAYA_CHECKPOINT_TOK(tok, "english");
+ bool found = false;
+ for (int n = 1; n <= 40 && !found; n++) {
+ auto q = WindowQuestion(n);
+ for (int64_t head = 32; head <= 192 && !found; head += 8) {
+ // room = max_len - len(head) - 1 and the default window = max_len - head_max_len - 8, so
+ // default == 2 * room at max_len = 2 * len(head) - head_max_len - 6.
+ const int64_t len = int64_t(sq::BuildHead(*tok, q, head).value().ids.size());
+ sq::Budget b{2 * len - head - 6, head};
+ if (b.max_len < 1 || b.DefaultWindow() <= 64) continue;
+ auto room = sq::StateRoom(*tok, q, b);
+ if (!room.ok() || room.value() <= 0 || b.DefaultWindow() != 2 * room.value()) continue;
+ found = true;
+ auto plan = sq::PlanWindows(*tok, {q}, b, {});
+ ASSERT_TRUE(plan.ok());
+ ASSERT_EQ(plan.value().warnings.size(), 1u);
+ EXPECT_NE(plan.value().warnings[0].find("is capped"), std::string::npos) << plan.value().warnings[0];
+ EXPECT_EQ(plan.value().window, room.value());
+ }
+ }
+ EXPECT_TRUE(found) << "no budget with a default window of twice the room";
+}
+
+TEST(BatchCap, MatchesPythonGrid) {
+ LAYA_GOLDEN_LIST(grid, "window_plan.json", "batch_cap");
+ LAYA_EXPECT_ROWS(grid, 810);
+ int capped = 0;
+ for (size_t i = 0; i < grid->size(); i++) {
+ const Value& rec = grid->value(i);
+ const int64_t n = rec.Find("n_windows")->as_int(), w = rec.Find("window")->as_int(), cfg = rec.Find("config_budget")->as_int();
+ const Value* bs = rec.Find("batch_size");
+ const int64_t batch = bs->is_null() ? 0 : bs->as_int();
+ const Value* want = rec.Find("cap");
+ sq::Cap got = sq::BatchCap(n, w, cfg, batch);
+ EXPECT_EQ(got.ok, !want->is_null()) << "row " << i;
+ if (!want->is_null()) EXPECT_EQ(got.cap, want->as_int()) << "row " << i;
+ if (!want->is_null() && batch <= 0) capped++;
+ }
+ EXPECT_GT(capped, 0);
+}
+
+TEST(BatchCap, HugeInputsDoNotOverflow) {
+ const int64_t big = INT64_MAX;
+ sq::Cap c = sq::BatchCap(big, big - 1, big, 0); // a one-token-narrower window: a mild cap, so none
+ EXPECT_FALSE(c.ok);
+ sq::Cap d = sq::BatchCap(big, 2, big, 0);
+ EXPECT_TRUE(d.ok);
+ EXPECT_LT(d.cap, big);
+}
+
+TEST(Budget, RefusesNonPositive) {
+ LAYA_CHECKPOINT_TOK(tok, "english");
+ auto q = sc::Question::Noul("Holds?").value();
+ EXPECT_FALSE(sq::StateRoom(*tok, q, {0, 8}).ok());
+ EXPECT_FALSE(sq::StateRoom(*tok, q, {512, -1}).ok());
+ EXPECT_FALSE(sq::Build(*tok, q, {}, {0, 0}).ok());
+ EXPECT_FALSE(sq::BuildHead(*tok, sc::Question(), 64).ok());
+ EXPECT_EQ((sq::Budget{512, 192}).DefaultWindow(), 312);
+ EXPECT_EQ((sq::Budget{100, 192}).DefaultWindow(), 64);
+}
+
+// States: json.loads then serialize_state, on 2,000 random JSON texts.
+class StateRow : public ::testing::TestWithParam<int> {};
+TEST_P(StateRow, SerializeMatchesPython) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "states", "cases");
+ LAYA_EXPECT_ROWS(rows, 2000);
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {
+ const Value& rec = rows->value(i);
+ const std::string text = rec.Find("text")->as_string();
+ auto v = laya::pyvalue::Parse(text);
+ ASSERT_TRUE(v.ok()) << "state " << i << ": " << v.error();
+ auto s = sq::SerializeState(v.value());
+ ASSERT_TRUE(s.ok()) << "state " << i << ": " << s.error();
+ EXPECT_EQ(s.value(), rec.Find("serialized")->as_string()) << "state " << i;
+ EXPECT_EQ(v.value().kind() == Kind::Array, rec.Find("list")->as_bool()) << "state " << i;
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, StateRow, ::testing::ValuesIn(laya_test::Chunks(2000)));
+
+TEST(States, PythonRejects) {
+ LAYA_GOLDEN_LIST(rejects, "go_extra.json", "states", "python_rejects");
+ for (const Value& t : rejects->items()) {
+ EXPECT_FALSE(laya::pyvalue::Parse(t.as_string()).ok()) << "accepted " << t.as_string().substr(0, 60);
+ }
+}
+
+} // namespace
diff --git a/laya-cpp/tests/tools/laya_c3_cli.cpp b/laya-cpp/tests/tools/laya_c3_cli.cpp
new file mode 100644
index 00000000..549a4d71
--- /dev/null
+++ b/laya-cpp/tests/tools/laya_c3_cli.cpp
@@ -0,0 +1,150 @@
+// A tiny line protocol over laya::schema and laya::sequence for the differential fuzz
+// (laya-e2e/cpp-lane/fuzz-c3): argv[1] is the checkpoints directory; bytes travel as hex ("-" is empty).
+// q <hex def> -> "err <hex>" | "ok <hex json {type, ins, opts, labels}>"
+// seq <name> <max> <head> <left 0|1> <order|-> <hex def> <hex state json>
+// -> "err <hex>" | "mk <kept> <total>" | "ok ids|markers|opts|distinct|tpo|st|su|sd|trunc"
+// plan <name> <max> <head> <window|-> <stride|-> <n,n,...|-> -> "err <hex>" | "ok w s r <hex warning>..."
+// cap <n> <window> <config> <batch|-> -> "none" | cap
+#include <iostream>
+#include <map>
+#include <sstream>
+
+#include "laya/pyjson.hpp"
+#include "laya/pyvalue.hpp"
+#include "laya/schema.hpp"
+#include "laya/sequence.hpp"
+
+namespace {
+using laya::pyvalue::Value;
+namespace sc = laya::schema;
+namespace sq = laya::sequence;
+
+std::string FromHex(const std::string& h) {
+ std::string out;
+ if (h == "-") return out;
+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));
+ return out;
+}
+std::string Hex(const std::string& s) {
+ static const char* d = "0123456789abcdef";
+ std::string o;
+ for (unsigned char c : s) {
+ o.push_back(d[c >> 4]);
+ o.push_back(d[c & 15]);
+ }
+ return o.empty() ? "-" : o;
+}
+template <class T>
+std::string Join(const std::vector<T>& v) {
+ std::string o;
+ for (size_t i = 0; i < v.size(); i++) o += (i ? "," : "") + std::to_string(v[i]);
+ return o.empty() ? "-" : o;
+}
+std::vector<int> Ints(const std::string& s) {
+ std::vector<int> out;
+ if (s == "-") return out;
+ std::stringstream ss(s);
+ for (std::string part; std::getline(ss, part, ',');) out.push_back(std::stoi(part));
+ return out;
+}
+int64_t Opt(const std::string& s) { return s == "-" ? 0 : std::stoll(s); }
+
+laya::Result<sc::Question> Make(const std::string& def_json, const std::string& id = "q") {
+ auto v = laya::pyvalue::Parse(def_json);
+ if (!v.ok()) return laya::Result<sc::Question>::Fail("parse: " + v.error());
+ auto q = sc::Question::FromWire(v.value());
+ if (!q.ok()) return laya::Result<sc::Question>::Fail("question " + laya::pyjson::Repr(id) + ": " + q.error());
+ return q;
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ if (argc < 2) return 2;
+ std::map<std::string, laya::tokenizer::Tokenizer> toks;
+ for (const char* n : {"english", "multilingual"}) {
+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(argv[1]) + "/" + n);
+ if (!t.ok()) { std::cerr << t.error() << "\n"; return 2; }
+ toks.emplace(n, std::move(t).value());
+ }
+ std::string line;
+ while (std::getline(std::cin, line)) {
+ std::stringstream ss(line);
+ std::string kind;
+ ss >> kind;
+ if (kind == "q") {
+ std::string h;
+ ss >> h;
+ auto q = Make(FromHex(h));
+ if (!q.ok()) { std::cout << "err " << Hex(q.error()) << "\n"; continue; }
+ Value o = Value::Object();
+ o.Set("type", Value::String(std::string(sc::Name(q.value().type()))));
+ o.Set("ins", Value::String(q.value().instructions()));
+ Value opts = Value::Array(), labels = Value::Array();
+ for (const auto& s : q.value().RenderedOptions()) opts.Push(Value::String(s));
+ for (const auto& s : q.value().Labels()) labels.Push(Value::String(s));
+ o.Set("opts", opts);
+ o.Set("labels", labels);
+ std::cout << "ok " << Hex(laya::pyvalue::Dumps(o).value()) << "\n";
+ } else if (kind == "seq") {
+ std::string name, order, def, state;
+ int64_t mx, hd;
+ int left;
+ ss >> name >> mx >> hd >> left >> order >> def >> state;
+ auto q = Make(FromHex(def));
+ if (!q.ok()) { std::cout << "err " << Hex(q.error()) << "\n"; continue; }
+ sc::Question question = q.value();
+ if (order != "-") {
+ auto w = question.WithOptionOrder(Ints(order));
+ if (!w.ok()) { std::cout << "err " << Hex(w.error()) << "\n"; continue; }
+ question = w.value();
+ }
+ auto sv = laya::pyvalue::Parse(FromHex(state));
+ if (!sv.ok()) { std::cout << "err " << Hex("state: " + sv.error()) << "\n"; continue; }
+ const auto& tok = toks.at(name);
+ auto st = sq::EncodeState(tok, sv.value());
+ if (!st.ok()) { std::cout << "err " << Hex(st.error()) << "\n"; continue; }
+ if (left) st.value().keep = sq::Keep::End;
+ sq::MarkersError me;
+ auto s = sq::Build(tok, question, st.value(), {mx, hd}, &me);
+ if (!s.ok()) {
+ if (me.total != 0) std::cout << "mk " << me.kept << " " << me.total << "\n";
+ else std::cout << "err " << Hex(s.error()) << "\n";
+ continue;
+ }
+ const auto& r = s.value();
+ std::cout << "ok " << Join(r.ids) << "|" << Join(r.markers) << "|" << r.stats.options << "|" << r.stats.options_distinct
+ << "|" << (r.stats.tokens_per_option ? std::to_string(*r.stats.tokens_per_option) : "N") << "|"
+ << r.truncation.state_tokens << "|" << r.truncation.state_tokens_used << "|" << r.truncation.state_tokens_dropped
+ << "|" << (r.truncation.truncated ? 1 : 0) << "\n";
+ } else if (kind == "plan") {
+ std::string name, w, st, counts;
+ int64_t mx, hd;
+ ss >> name >> mx >> hd >> w >> st >> counts;
+ std::vector<sc::Question> qs;
+ for (int n : Ints(counts)) {
+ std::vector<sc::Option> opts;
+ for (int i = 0; i < n; i++) {
+ std::string t;
+ for (int k = 0; k < 4; k++) t += (k ? " " : "") + std::string("criterion") + std::to_string(i) + " word" + std::to_string(k);
+ char l[16];
+ std::snprintf(l, sizeof l, "opt%03d", i);
+ opts.push_back({sc::Label::Text(l), Value::String(t)});
+ }
+ qs.push_back(sc::Question::Choice("Pick one.", opts).value());
+ }
+ auto p = sq::PlanWindows(toks.at(name), qs, {mx, hd}, {Opt(w), Opt(st)});
+ if (!p.ok()) { std::cout << "err " << Hex(p.error()) << "\n"; continue; }
+ std::cout << "ok " << p.value().window << " " << p.value().stride << " " << p.value().room;
+ for (const auto& x : p.value().warnings) std::cout << " " << Hex(x);
+ std::cout << "\n";
+ } else if (kind == "cap") {
+ std::string n, w, c, b;
+ ss >> n >> w >> c >> b;
+ auto r = sq::BatchCap(std::stoll(n), std::stoll(w), std::stoll(c), Opt(b));
+ if (r.ok) std::cout << r.cap << "\n"; else std::cout << "none\n";
+ } else {
+ std::cout << "bad\n";
+ }
+ }
+ return 0;
+}