Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(cpp): schema and sequence
Adeshina Lasisid7491a3ae238
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex cb24e6b3..076166ff 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -42,7 +42,7 @@ concurrency:env:# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus# compile_flags). Raise it with every test added.- MIN_TESTS: "253"+ MIN_TESTS: "352"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints@@ -93,7 +93,7 @@ jobs:LAYA_FIXTURE_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpointsPYTHONPATH: ${{ github.workspace }}run: |- python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json+ python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json --only tokenizer_ids.json --only decode_text.json --only sequences.json --only window_plan.jsonpython laya-go/scripts/gen_go_fixtures.py --out "$GOLDENS"- name: Compiled C++ tables match CPythonenv:diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex 77981fe5..f76be6e7 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -26,7 +26,7 @@ function(laya_target_flags target)endfunction()add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp- src/tokenizer.cpp)+ src/tokenizer.cpp src/schema.cpp src/sequence.cpp)target_include_directories(laya PUBLIC include)laya_target_flags(laya)@@ -46,7 +46,7 @@ if(LAYA_BUILD_TESTS)enable_testing()add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp- tests/tokenizer_test.cpp)+ tests/tokenizer_test.cpp tests/schema_test.cpp tests/sequence_test.cpp)target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)laya_target_flags(laya_tests)@@ -56,6 +56,9 @@ if(LAYA_BUILD_TESTS)add_executable(laya_c2_cli tests/tools/laya_c2_cli.cpp)target_link_libraries(laya_c2_cli PRIVATE laya)laya_target_flags(laya_c2_cli)+ add_executable(laya_c3_cli tests/tools/laya_c3_cli.cpp)+ target_link_libraries(laya_c3_cli PRIVATE laya)+ laya_target_flags(laya_c3_cli)include(GoogleTest)# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.diff --git a/laya-cpp/include/laya/schema.hpp b/laya-cpp/include/laya/schema.hppnew file mode 100644index 00000000..bb5a3fe9--- /dev/null+++ b/laya-cpp/include/laya/schema.hpp@@ -0,0 +1,188 @@+#pragma once+// Question, answer and prediction types (laya-go/schema). Wording of every refusal is laya's own+// Agent._check_question; JSON is spelled as Python's json.dumps spells it, keys in Python's order.+#include <cstdint>+#include <map>+#include <optional>+#include <string>+#include <string_view>+#include <vector>++#include "laya/pyvalue.hpp"+#include "laya/result.hpp"++namespace laya {+namespace schema {++using pyvalue::Value;++enum class Type { Invalid, Choice, Score, Noul };+std::string_view Name(Type t); // "choice", "score", "noul", "" for Invalid+int Code(Type t); // the head's integer: 0, 1, 2; -1 for Invalid++// A choice label: a string, or the bool, int or float Python also allows.+class Label {+ public:+ enum class Kind { Text, Bool, Int, Float };+ Label() = default;+ static Label Text(std::string s);+ static Label OfBool(bool b);+ static Label OfInt(int64_t i);+ static Label OfFloat(double d);+ static Result<Label> Of(const Value& v); // string, bool, int or float; null and containers refused++ Kind kind() const { return kind_; }+ std::string String() const; // str(label): what the prompt shows+ const std::string& Key() const { return text_; } // json.dumps spelling: the answer key+ std::string Repr() const; // repr(label)+ std::string EqKey() const; // equal for the labels Python's dict keys as one (1, 1.0, True)+ std::string Json() const; // the label as a JSON value+ bool operator==(const Label& o) const { return kind_ == o.kind_ && text_ == o.text_; }++ private:+ Kind kind_ = Kind::Text;+ std::string text_;+};++struct Option {+ Label label;+ Value criterion; // Null and "" mean no description+};++struct NoulSpec {+ Value true_text, false_text; // Null or "" keep the default texts+ std::string true_label, false_label; // both empty keeps "true" and "false"+};++// A description snapshotted when the question is built.+struct Criterion {+ bool set = false;+ std::string text; // render_criterion of the value+ std::string raw; // the value as JSON, for the wire form+};++struct ChoiceOption {+ Label label;+ Criterion crit;+};++// The reason alone is the error of a constructor; ParseQuestions prefixes "question 'id': ".+class Question {+ public:+ static Result<Question> Choice(std::string instructions, const std::vector<Option>& options);+ static Result<Question> ChoiceOf(std::string instructions, const std::vector<std::string>& labels);+ static Result<Question> Score(std::string instructions, const std::vector<Value>& levels);+ static Result<Question> Noul(std::string instructions, const NoulSpec& spec = {});+ // One definition in the Python wire form ({"type", "instructions", "criteria", ...}).+ static Result<Question> FromWire(const Value& definition);++ Result<Question> WithOptionOrder(const std::vector<int>& order) const; // a permutation of 0..n-1, else refused+ Question WithoutOptionOrder() const; // restores label order+ Result<Question> Subset(const std::vector<int>& indices) const;++ Type type() const { return typ_; }+ const std::string& instructions() const { return instructions_; }+ size_t NumOptions() const;+ const std::vector<int>& OptionOrder() const { return order_; } // empty: label order+ std::vector<std::string> Labels() const; // str() of each label, label-index order+ std::vector<Label> AnswerLabels() const;+ std::vector<std::string> RenderedOptions() const; // render_options: the text the model reads+ std::vector<std::string> Legend() const; // score only+ Result<std::string> ToJson() const;++ private:+ Type typ_ = Type::Invalid;+ std::string instructions_;+ std::vector<ChoiceOption> options_;+ std::vector<Criterion> levels_;+ Criterion noul_[2]; // false side, true side+ std::string false_label_, true_label_;+ bool custom_labels_ = false;+ std::vector<int> order_;+};++struct Entry {+ std::string id;+ Question question;+};++class Questions {+ public:+ static Result<Questions> Of(const std::vector<Entry>& entries);+ static Result<Questions> Parse(std::string_view json);+ size_t size() const { return entries_.size(); }+ const Entry& at(size_t i) const { return entries_[i]; }+ const Question* Get(std::string_view id) const;+ std::vector<std::string> Ids() const;+ Result<std::string> ToJson() const;++ private:+ std::vector<Entry> entries_;+ std::map<std::string, size_t, std::less<>> index_;+};++// json.dumps of a string, ensure_ascii=False.+std::string JsonString(std::string_view s);++struct Probability {+ Label label;+ double p = 0;+};++struct Window {+ int64_t index = 0, token_start = 0, token_end = 0, count = 0;+};++// One answer; the fields that apply depend on type.+struct Answer {+ Type type = Type::Invalid;+ Label choice; // choice+ double score = 0; // score+ double noul = 0; // noul+ std::vector<std::string> legend; // score+ std::vector<Probability> probabilities; // choice, score+ double confidence = 0, answer_confidence = 0, act_probability = 0;+ std::optional<Window> window;+ bool low_confidence = false;+ std::string abstention; // "", "passed", "abstained" or "unevaluated"+ std::optional<double> abstention_threshold;++ Result<std::string> ToJson() const;+ static Result<Answer> FromJson(const Value& v);+};++struct Answered {+ std::string id;+ Answer answer;+};++struct CollapsedOptions {+ int64_t total = 0, distinct = 0;+ std::optional<int64_t> tokens_per_option;+};+struct CollapsedQuestion {+ std::string id;+ CollapsedOptions options;+};++struct Usage {+ int64_t input_tokens = 0, output_tokens = 0, state_tokens = 0, state_tokens_dropped = 0;+ bool truncated = false;+ std::vector<std::string> truncated_questions;+ std::vector<CollapsedQuestion> collapsed_options;+ int64_t windows = 0;+ int64_t truncated_windows = 0; // with windows > 1, the "truncated" key is this count+ Result<std::string> ToJson() const;+};++constexpr const char* kModel = "laya-rl-agent-onnx";++struct Prediction {+ std::string model = kModel;+ std::vector<Answered> answers;+ Usage usage;+ Result<std::string> ToJson() const;+};++} // namespace schema+} // namespace layadiff --git a/laya-cpp/include/laya/sequence.hpp b/laya-cpp/include/laya/sequence.hppnew file mode 100644index 00000000..66e0eb5f--- /dev/null+++ b/laya-cpp/include/laya/sequence.hpp@@ -0,0 +1,104 @@+#pragma once+// The model's input sequence (laya-go/sequence): head, state, markers, and the window plan.+#include <cstdint>+#include <optional>+#include <string>+#include <vector>++#include "laya/pyvalue.hpp"+#include "laya/result.hpp"+#include "laya/schema.hpp"+#include "laya/tokenizer.hpp"++namespace laya {+namespace sequence {++constexpr int64_t kOptionTokenCap = 48;+constexpr int64_t kMinOptionBudget = 16;+constexpr int64_t kMinTokensPerOption = 4;+constexpr int64_t kMinInstructionTokens = 8;++struct Budget {+ int64_t max_len = 0;+ int64_t head_max_len = 0;+ int64_t DefaultWindow() const; // max(64, max_len - head_max_len - 8)+};++enum class Keep { Start, End };++struct State {+ std::vector<int32_t> ids;+ Keep keep = Keep::Start;+};++// serialize_state: a string as is, anything else as json.dumps(ensure_ascii=False).+Result<std::string> SerializeState(const pyvalue::Value& state);+// A list keeps its end when it is too long, anything else its start.+Result<State> EncodeState(const tokenizer::Tokenizer& tok, const pyvalue::Value& state);++struct Stats {+ int64_t options = 0;+ int64_t options_distinct = 0;+ std::optional<int64_t> tokens_per_option;+ bool Collapsed() const { return options_distinct < options; }+};++struct Truncation {+ int64_t state_tokens = 0, state_tokens_used = 0, state_tokens_dropped = 0;+ bool truncated = false;+ bool operator==(const Truncation& o) const {+ return state_tokens == o.state_tokens && state_tokens_used == o.state_tokens_used &&+ state_tokens_dropped == o.state_tokens_dropped && truncated == o.truncated;+ }+};++struct Head {+ std::vector<int32_t> ids;+ std::vector<int64_t> markers;+ Stats stats;+};++struct Sequence {+ std::vector<int32_t> ids;+ std::vector<int64_t> markers;+ Stats stats;+ Truncation truncation;+};++// Some option markers fell beyond max_len. Message(id) is the agent's refusal text.+struct MarkersError {+ int64_t kept = 0, total = 0, max_len = 0, head_max_len = 0;+ std::string Message(const std::string& id = "") const;+};++Result<Head> BuildHead(const tokenizer::Tokenizer& tok, const schema::Question& q, int64_t head_max_len);+// markers_error, when not null, is filled iff the failure is the marker refusal.+Result<Sequence> Build(const tokenizer::Tokenizer& tok, const schema::Question& q, const State& state,+ const Budget& b, MarkersError* markers_error = nullptr);+Result<int64_t> StateRoom(const tokenizer::Tokenizer& tok, const schema::Question& q, const Budget& b);++struct WindowRequest {+ int64_t window = 0; // <= 0: the default+ int64_t stride = 0; // <= 0: half a window+};++struct WindowPlan {+ int64_t window = 0, stride = 0, room = 0;+ std::vector<std::string> warnings;+ bool Clamped() const { return !warnings.empty(); }+};++enum class PlanError { None, NoRoom, StrideGap, Other };++Result<WindowPlan> PlanWindows(const tokenizer::Tokenizer& tok, const std::vector<schema::Question>& questions,+ const Budget& b, const WindowRequest& req, PlanError* kind = nullptr);++// window_batch_cap: ok=false is Python's None (do not cap). batch_size <= 0 is unset.+struct Cap {+ int64_t cap = 0;+ bool ok = false;+};+Cap BatchCap(int64_t n_windows, int64_t window, int64_t config_window, int64_t batch_size);++} // namespace sequence+} // namespace layadiff --git a/laya-cpp/src/schema.cpp b/laya-cpp/src/schema.cppnew file mode 100644index 00000000..9e5a225d--- /dev/null+++ b/laya-cpp/src/schema.cpp@@ -0,0 +1,938 @@+#include "laya/schema.hpp"++#include <algorithm>+#include <cmath>+#include <cstdlib>++#include "laya/pyjson.hpp"+#include "laya/unicodedata.hpp"++namespace laya {+namespace schema {++namespace ud = unicodedata;+using Kind = Value::Kind;++std::string_view Name(Type t) {+ switch (t) {+ case Type::Choice: return "choice";+ case Type::Score: return "score";+ case Type::Noul: return "noul";+ case Type::Invalid: break;+ }+ return "";+}++int Code(Type t) {+ switch (t) {+ case Type::Choice: return 0;+ case Type::Score: return 1;+ case Type::Noul: return 2;+ case Type::Invalid: break;+ }+ return -1;+}++std::string JsonString(std::string_view s) {+ auto r = pyvalue::Dumps(Value::String(ud::ValidUtf8(s) ? std::string(s) : ud::Lossy(s)));+ return r.ok() ? r.value() : "\"\"";+}++namespace {++// Reasons laya's Agent._check_question gives, word for word.+constexpr const char* kNoInstructions = "no 'instructions'; add the text the model should answer";+constexpr const char* kNilInstructions = "'instructions' must not be None; add the text the model should answer";+constexpr const char* kBlankIns = "'instructions' must not be empty; add the text the model should answer";+constexpr const char* kChoiceShape =+ "a choice question takes 'criteria' as a dict of label -> description, or a list of labels";+constexpr const char* kChoiceEmpty = "a choice question needs at least one criterion";+constexpr const char* kScoreShape = "a score question takes 'criteria' as a list of level descriptions, index 0 first";+constexpr const char* kScoreEmpty = "a score question needs at least one level";+constexpr const char* kNoulShape =+ "a noul question takes 'criteria' as a dict with optional 'true'/'false' descriptions, or omits it";+constexpr const char* kLabelsOnlyNoul = "'labels' is only supported for noul questions";+constexpr const char* kNoulLabels = "noul labels must map exactly 'false' and 'true' to distinct non-empty strings";+constexpr const char* kDefaultFalse = "no, the statement does not hold";+constexpr const char* kDefaultTrue = "yes, the statement holds";++std::string N(int64_t i) { return std::to_string(i); }++std::string NoulKeysReason(const std::string& got) {+ return "a noul question takes 'criteria' keyed only 'true'/'false' (either or both, and omitted is fine), got " + got ++ ". Those keys are the option texts the model reads; any other key was silently dropped and replaced with "+ "the defaults. If you want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "+ "'labels' instead.";+}++std::string OrderReason(size_t n, const std::string& got) {+ return "'option_order' must be a permutation of range(" + N(int64_t(n)) ++ ") -- one slot per option, each option once -- got " + got;+}++std::string LabelRepeatReason(size_t i, const std::string& repr, size_t j) {+ return "choice label " + N(int64_t(i)) + " (" + repr + ") repeats label " + N(int64_t(j)) ++ "; the labels are the answer keys, so every option needs its own (1, 1.0 and True are one key)";+}++std::string ScoreNullReason(size_t i) {+ return "score level " + N(int64_t(i)) + " is null; give every level a description, index 0 first";+}++// An integer-valued double as exact decimal digits (no sign for zero).+std::string ExactInteger(double d) {+ if (d == 0) return "0";+ if (std::fabs(d) < 9.0e18) return N(int64_t(d));+ int exp = 0;+ double m = std::frexp(std::fabs(d), &exp); // d = m * 2^exp, 0.5 <= m < 1+ uint64_t mant = uint64_t(std::ldexp(m, 53)); // 53-bit integer+ exp -= 53; // d = mant * 2^exp, exp >= 0 here+ std::vector<int> digits; // little endian, base 10+ for (uint64_t v = mant; v > 0; v /= 10) digits.push_back(int(v % 10));+ for (int i = 0; i < exp; i++) {+ int carry = 0;+ for (int& x : digits) {+ int v = x * 2 + carry;+ x = v % 10;+ carry = v / 10;+ }+ if (carry) digits.push_back(carry);+ }+ std::string out = d < 0 ? "-" : "";+ for (size_t i = digits.size(); i-- > 0;) out.push_back(char('0' + digits[i]));+ return out;+}++Criterion NewCriterion(const Value& v, std::string* err) {+ if (v.is_null()) return {};+ if (v.kind() == Kind::String) {+ if (v.as_string().empty()) return {};+ return {true, v.as_string(), JsonString(v.as_string())};+ }+ auto text = pyvalue::Dumps(v);+ if (!text.ok()) {+ *err = text.error();+ return {};+ }+ return {true, text.value(), text.value()};+}++std::string CheckInstructions(const std::string& ins) {+ if (ud::IsBlank(ins)) return kBlankIns;+ if (!ud::ValidUtf8(ins)) return "'instructions' is not valid UTF-8; a Python str always is";+ return "";+}++std::string CheckDistinct(const std::vector<Label>& labels) {+ std::map<std::string, size_t> first;+ for (size_t i = 0; i < labels.size(); i++) {+ auto ins = first.emplace(labels[i].EqKey(), i);+ if (!ins.second) return LabelRepeatReason(i, labels[i].Repr(), ins.first->second);+ }+ return "";+}++template <class T>+Result<T> Refuse(const std::string& reason) {+ return Result<T>::Fail(reason);+}++} // namespace++// ---- Label -------------------------------------------------------------------------------------++Label Label::Text(std::string s) {+ Label l;+ l.text_ = std::move(s);+ return l;+}+Label Label::OfBool(bool b) {+ Label l;+ l.kind_ = Kind::Bool;+ l.text_ = b ? "true" : "false";+ return l;+}+Label Label::OfInt(int64_t i) {+ Label l;+ l.kind_ = Kind::Int;+ l.text_ = std::to_string(i);+ return l;+}+Label Label::OfFloat(double d) {+ Label l;+ l.kind_ = Kind::Float;+ l.text_ = pyjson::ReprFloat(d);+ return l;+}++Result<Label> Label::Of(const Value& v) {+ switch (v.kind()) {+ case pyvalue::Value::Kind::String: return Result<Label>::Ok(Text(v.as_string()));+ case pyvalue::Value::Kind::Bool: return Result<Label>::Ok(OfBool(v.as_bool()));+ case pyvalue::Value::Kind::Int: return Result<Label>::Ok(OfInt(v.as_int()));+ case pyvalue::Value::Kind::BigInt: {+ Label l;+ l.kind_ = Kind::Int;+ l.text_ = v.big_int();+ return Result<Label>::Ok(l);+ }+ case pyvalue::Value::Kind::Float: return Result<Label>::Ok(OfFloat(v.as_float()));+ default: break;+ }+ return Result<Label>::Fail("schema: " + pyvalue::TypeName(v) + " is not a label; use a string, bool or number");+}++std::string Label::String() const {+ if (kind_ == Kind::Bool) return text_ == "true" ? "True" : "False";+ if (kind_ == Kind::Float) {+ if (text_ == "NaN") return "nan";+ if (text_ == "Infinity") return "inf";+ if (text_ == "-Infinity") return "-inf";+ }+ return text_;+}++std::string Label::Repr() const { return kind_ == Kind::Text ? pyjson::Repr(text_) : String(); }++std::string Label::EqKey() const {+ switch (kind_) {+ case Kind::Text: return "s:" + text_;+ case Kind::Bool: return text_ == "true" ? "n:1" : "n:0";+ case Kind::Int: return "n:" + text_;+ case Kind::Float: break;+ }+ if (text_ == "NaN" || text_ == "Infinity" || text_ == "-Infinity") return "n:" + text_;+ double d = std::strtod(text_.c_str(), nullptr); // the C locale: the library never changes it+ if (d == std::trunc(d)) return "n:" + ExactInteger(d);+ return "f:" + text_; // shortest repr is unique per double+}++std::string Label::Json() const { return kind_ == Kind::Text ? JsonString(text_) : text_; }++// ---- Question ----------------------------------------------------------------------------------++Result<Question> Question::Choice(std::string instructions, const std::vector<Option>& options) {+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);+ if (options.empty()) return Refuse<Question>(kChoiceEmpty);+ std::vector<Label> labels;+ for (size_t i = 0; i < options.size(); i++) {+ const Label& l = options[i].label;+ if (l.kind() == Label::Kind::Text && !ud::ValidUtf8(l.Key())) {+ return Refuse<Question>("choice label " + N(int64_t(i)) + " is not valid UTF-8; a Python str always is");+ }+ labels.push_back(l);+ }+ if (auto r = CheckDistinct(labels); !r.empty()) return Refuse<Question>(r);+ Question q;+ q.typ_ = Type::Choice;+ q.instructions_ = std::move(instructions);+ for (size_t i = 0; i < options.size(); i++) {+ std::string err;+ Criterion c = NewCriterion(options[i].criterion, &err);+ if (!err.empty()) {+ return Refuse<Question>("choice option " + labels[i].Repr() + ": criterion cannot be rendered: " + err);+ }+ q.options_.push_back({labels[i], std::move(c)});+ }+ return Result<Question>::Ok(std::move(q));+}++Result<Question> Question::ChoiceOf(std::string instructions, const std::vector<std::string>& labels) {+ std::vector<Option> options;+ for (const auto& l : labels) options.push_back({Label::Text(l), Value()});+ return Choice(std::move(instructions), options);+}++Result<Question> Question::Score(std::string instructions, const std::vector<Value>& levels) {+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);+ if (levels.empty()) return Refuse<Question>(kScoreEmpty);+ Question q;+ q.typ_ = Type::Score;+ q.instructions_ = std::move(instructions);+ for (size_t i = 0; i < levels.size(); i++) {+ if (levels[i].is_null()) return Refuse<Question>(ScoreNullReason(i));+ std::string err;+ Criterion c = NewCriterion(levels[i], &err);+ if (!err.empty()) return Refuse<Question>("score level " + N(int64_t(i)) + " cannot be rendered: " + err);+ if (!c.set) c = {true, "", "\"\""}; // the empty string is a level: it renders as "level i: "+ q.levels_.push_back(std::move(c));+ }+ return Result<Question>::Ok(std::move(q));+}++Result<Question> Question::Noul(std::string instructions, const NoulSpec& spec) {+ if (auto r = CheckInstructions(instructions); !r.empty()) return Refuse<Question>(r);+ Question q;+ q.typ_ = Type::Noul;+ q.instructions_ = std::move(instructions);+ q.false_label_ = "false";+ q.true_label_ = "true";+ const Value* sides[2] = {&spec.false_text, &spec.true_text};+ const char* names[2] = {"false", "true"};+ for (int i = 0; i < 2; i++) {+ std::string err;+ q.noul_[i] = NewCriterion(*sides[i], &err);+ if (!err.empty()) return Refuse<Question>(std::string("noul ") + names[i] + " description cannot be rendered: " + err);+ }+ if (!spec.true_label.empty() || !spec.false_label.empty()) {+ std::string f = ud::Strip(spec.false_label), t = ud::Strip(spec.true_label);+ if (f.empty() || t.empty() || f == t || !ud::ValidUtf8(f) || !ud::ValidUtf8(t)) return Refuse<Question>(kNoulLabels);+ q.false_label_ = f;+ q.true_label_ = t;+ q.custom_labels_ = true;+ }+ return Result<Question>::Ok(std::move(q));+}++size_t Question::NumOptions() const {+ switch (typ_) {+ case Type::Choice: return options_.size();+ case Type::Score: return levels_.size();+ case Type::Noul: return 2;+ case Type::Invalid: break;+ }+ return 0;+}++Result<Question> Question::WithOptionOrder(const std::vector<int>& order) const {+ if (typ_ == Type::Invalid) {+ return Refuse<Question>("the zero Question has no options to order; build one with Choice, ChoiceOf, Score, Noul");+ }+ Question q = *this;+ size_t n = NumOptions();+ std::vector<bool> seen(n, false);+ bool ok = order.size() == n;+ for (size_t i = 0; ok && i < order.size(); i++) {+ int v = order[i];+ if (v < 0 || size_t(v) >= n || seen[size_t(v)]) ok = false;+ else seen[size_t(v)] = true;+ }+ if (!ok) {+ std::string s = "[";+ for (size_t i = 0; i < order.size(); i++) s += (i ? ", " : "") + N(order[i]);+ return Refuse<Question>(OrderReason(n, s + "]"));+ }+ q.order_ = order;+ return Result<Question>::Ok(std::move(q));+}++std::vector<std::string> Question::Labels() const {+ std::vector<std::string> out;+ switch (typ_) {+ case Type::Choice: for (const auto& o : options_) out.push_back(o.label.String()); break;+ case Type::Score: for (size_t i = 0; i < levels_.size(); i++) out.push_back(N(int64_t(i))); break;+ case Type::Noul: out = {false_label_, true_label_}; break;+ case Type::Invalid: break;+ }+ return out;+}++std::vector<Label> Question::AnswerLabels() const {+ std::vector<Label> out;+ switch (typ_) {+ case Type::Choice: for (const auto& o : options_) out.push_back(o.label); break;+ case Type::Score: for (size_t i = 0; i < levels_.size(); i++) out.push_back(Label::Text(N(int64_t(i)))); break;+ case Type::Noul: out = {Label::Text(false_label_), Label::Text(true_label_)}; break;+ case Type::Invalid: break;+ }+ return out;+}++std::vector<std::string> Question::RenderedOptions() const {+ std::vector<std::string> out;+ switch (typ_) {+ case Type::Choice:+ for (const auto& o : options_) out.push_back(o.crit.set ? o.label.String() + ": " + o.crit.text : o.label.String());+ break;+ case Type::Score:+ for (size_t i = 0; i < levels_.size(); i++) out.push_back("level " + N(int64_t(i)) + ": " + levels_[i].text);+ break;+ case Type::Noul:+ out.push_back(false_label_ + ": " + (noul_[0].set ? noul_[0].text : std::string(kDefaultFalse)));+ out.push_back(true_label_ + ": " + (noul_[1].set ? noul_[1].text : std::string(kDefaultTrue)));+ break;+ case Type::Invalid: break;+ }+ return out;+}++std::vector<std::string> Question::Legend() const {+ std::vector<std::string> out;+ if (typ_ == Type::Score) for (const auto& l : levels_) out.push_back(l.text);+ return out;+}++Result<std::string> Question::ToJson() const {+ if (typ_ == Type::Invalid) return Refuse<std::string>("schema: cannot marshal the zero Question");+ std::string b = "{\"type\":\"" + std::string(Name(typ_)) + "\",\"instructions\":" + JsonString(instructions_);+ switch (typ_) {+ case Type::Choice: {+ bool described = false;+ for (const auto& o : options_) described = described || o.crit.set;+ if (!described) {+ b += ",\"criteria\":[";+ for (size_t i = 0; i < options_.size(); i++) b += (i ? "," : "") + options_[i].label.Json();+ b += "]";+ } else {+ b += ",\"criteria\":{";+ for (size_t i = 0; i < options_.size(); i++) {+ b += (i ? "," : "") + JsonString(options_[i].label.Key()) + ":" + (options_[i].crit.set ? options_[i].crit.raw : "null");+ }+ b += "}";+ }+ break;+ }+ case Type::Score:+ b += ",\"criteria\":[";+ for (size_t i = 0; i < levels_.size(); i++) b += (i ? "," : "") + levels_[i].raw;+ b += "]";+ break;+ case Type::Noul:+ if (noul_[0].set || noul_[1].set) {+ b += ",\"criteria\":{";+ const char* sep = "";+ const char* keys[2] = {"false", "true"};+ for (int i = 0; i < 2; i++) {+ if (!noul_[i].set) continue;+ b += std::string(sep) + "\"" + keys[i] + "\":" + noul_[i].raw;+ sep = ",";+ }+ b += "}";+ }+ if (custom_labels_) {+ b += ",\"labels\":{\"false\":" + JsonString(false_label_) + ",\"true\":" + JsonString(true_label_) + "}";+ }+ break;+ case Type::Invalid: break;+ }+ if (!order_.empty()) {+ b += ",\"option_order\":[";+ for (size_t i = 0; i < order_.size(); i++) b += (i ? "," : "") + N(order_[i]);+ b += "]";+ }+ b += "}";+ return Result<std::string>::Ok(std::move(b));+}++Question Question::WithoutOptionOrder() const {+ Question q = *this;+ q.order_.clear();+ return q;+}++Result<Question> Question::Subset(const std::vector<int>& indices) const {+ if (typ_ != Type::Choice) {+ return Refuse<Question>("only a choice question has options to subset, not " + std::string(Name(typ_)));+ }+ if (indices.empty()) return Refuse<Question>(kChoiceEmpty);+ std::vector<bool> seen(options_.size(), false);+ Question q;+ q.typ_ = Type::Choice;+ q.instructions_ = instructions_;+ for (size_t i = 0; i < indices.size(); i++) {+ int ix = indices[i];+ if (ix < 0 || size_t(ix) >= options_.size() || seen[size_t(ix)]) {+ return Refuse<Question>("subset index " + N(ix) + " at position " + N(int64_t(i)) + " is out of range or repeated");+ }+ seen[size_t(ix)] = true;+ q.options_.push_back(options_[size_t(ix)]);+ }+ return Result<Question>::Ok(std::move(q));+}++// ---- wire form ---------------------------------------------------------------------------------++namespace {++Result<std::string> WireInstructions(const Value& def) {+ const Value* ins = def.Find("instructions");+ if (ins == nullptr) return Refuse<std::string>(kNoInstructions);+ switch (ins->kind()) {+ case Kind::Null: return Refuse<std::string>(kNilInstructions);+ case Kind::String: {+ if (auto r = CheckInstructions(ins->as_string()); !r.empty()) return Refuse<std::string>(r);+ return Result<std::string>::Ok(ins->as_string());+ }+ case Kind::Array:+ case Kind::Object:+ if (ins->size() == 0) return Refuse<std::string>(kBlankIns);+ break;+ default: break;+ }+ auto text = pyvalue::Dumps(*ins); // _to_internal: json.dumps(ins, ensure_ascii=False)+ if (!text.ok()) return Refuse<std::string>("'instructions' cannot be written as JSON: " + text.error());+ return text;+}++Result<std::vector<ChoiceOption>> WireChoice(const Value* crit) {+ using R = Result<std::vector<ChoiceOption>>;+ std::vector<Label> labels;+ std::vector<Value> values;+ if (crit != nullptr && crit->kind() == Kind::Object) {+ for (size_t i = 0; i < crit->size(); i++) {+ labels.push_back(Label::Text(crit->key(i)));+ values.push_back(crit->value(i));+ }+ } else if (crit != nullptr && crit->kind() == Kind::Array) {+ for (size_t i = 0; i < crit->size(); i++) {+ const Value& l = crit->value(i);+ switch (l.kind()) {+ case Kind::String: case Kind::Bool: case Kind::Int: case Kind::BigInt: case Kind::Float: break;+ case Kind::Null:+ return R::Fail("choice label " + N(int64_t(i)) ++ " is null; a label is rendered as option text and used as the answer key, so it must be a "+ "string, number or bool -- a null label renders as the text \"None\" while its answer key is "+ "\"null\"");+ default:+ return R::Fail("choice label " + N(int64_t(i)) + " is a " + pyvalue::TypeName(l) ++ "; a label is rendered as option text and used as the answer key, so it must be a scalar (a "+ "string, number or bool), got " + pyvalue::Repr(l));+ }+ labels.push_back(Label::Of(l).value());+ values.push_back(Value());+ }+ if (auto r = CheckDistinct(labels); !r.empty()) return R::Fail(r);+ } else {+ return R::Fail(kChoiceShape);+ }+ if (labels.empty()) return R::Fail(kChoiceEmpty);+ std::vector<ChoiceOption> opts;+ for (size_t i = 0; i < labels.size(); i++) {+ std::string err;+ Criterion c = NewCriterion(values[i], &err);+ if (!err.empty()) return R::Fail("choice option " + labels[i].Repr() + ": criterion cannot be rendered: " + err);+ opts.push_back({labels[i], std::move(c)});+ }+ return R::Ok(std::move(opts));+}++Result<std::vector<Criterion>> WireScore(const Value* crit) {+ using R = Result<std::vector<Criterion>>;+ if (crit == nullptr || crit->kind() != Kind::Array) return R::Fail(kScoreShape);+ if (crit->size() == 0) return R::Fail(kScoreEmpty);+ std::vector<Criterion> out;+ for (size_t i = 0; i < crit->size(); i++) {+ const Value& v = crit->value(i);+ if (v.is_null()) return R::Fail(ScoreNullReason(i));+ std::string err;+ Criterion c = NewCriterion(v, &err);+ if (!err.empty()) return R::Fail("score level " + N(int64_t(i)) + " cannot be rendered: " + err);+ if (!c.set) c = {true, "", "\"\""};+ out.push_back(std::move(c));+ }+ return R::Ok(std::move(out));+}++bool WireNoul(const Value* crit, Criterion (&out)[2], std::string* reason) {+ if (crit == nullptr || crit->is_null()) return true;+ if (crit->kind() != Kind::Object) {+ *reason = kNoulShape;+ return false;+ }+ std::map<std::string, Value> sides; // _to_internal: {str(k).lower(): v}; a later key wins+ for (size_t i = 0; i < crit->size(); i++) sides[ud::Lower(crit->key(i))] = crit->value(i);+ bool unknown = false;+ Value keys = Value::Array();+ for (const auto& kv : sides) {+ unknown = unknown || (kv.first != "true" && kv.first != "false");+ keys.Push(Value::String(kv.first));+ }+ if (unknown) {+ *reason = NoulKeysReason(pyvalue::Repr(keys));+ return false;+ }+ const char* names[2] = {"false", "true"};+ for (int i = 0; i < 2; i++) {+ auto it = sides.find(names[i]);+ std::string err;+ out[i] = NewCriterion(it == sides.end() ? Value() : it->second, &err);+ if (!err.empty()) {+ *reason = std::string("noul ") + names[i] + " description cannot be rendered: " + err;+ return false;+ }+ }+ return true;+}++} // namespace++Result<Question> Question::FromWire(const Value& def) {+ if (def.kind() != Kind::Object) return Refuse<Question>("definition must be a dict, got " + pyvalue::TypeName(def));+ const Value* tv = def.Find("type");+ Type typ = Type::Invalid;+ if (tv != nullptr && tv->kind() == Kind::String) {+ const std::string& s = tv->as_string();+ typ = s == "choice" ? Type::Choice : s == "score" ? Type::Score : s == "noul" ? Type::Noul : Type::Invalid;+ }+ if (typ == Type::Invalid) {+ return Refuse<Question>("unknown type " + pyvalue::Repr(tv == nullptr ? Value() : *tv) ++ "; use one of ['choice', 'noul', 'score']");+ }+ auto ins = WireInstructions(def);+ if (!ins.ok()) return Refuse<Question>(ins.error());+ Question q;+ q.typ_ = typ;+ q.instructions_ = ins.value();+ const Value* crit = def.Find("criteria");+ if (typ == Type::Choice) {+ auto r = WireChoice(crit);+ if (!r.ok()) return Refuse<Question>(r.error());+ q.options_ = std::move(r).value();+ } else if (typ == Type::Score) {+ auto r = WireScore(crit);+ if (!r.ok()) return Refuse<Question>(r.error());+ q.levels_ = std::move(r).value();+ } else {+ std::string reason;+ if (!WireNoul(crit, q.noul_, &reason)) return Refuse<Question>(reason);+ q.false_label_ = "false";+ q.true_label_ = "true";+ }+ if (const Value* order = def.Find("option_order")) {+ size_t n = q.NumOptions();+ bool good = order->kind() == Kind::Array && order->size() == n;+ std::vector<int> ints;+ std::vector<bool> seen(n, false);+ for (size_t i = 0; good && i < n; i++) {+ const Value& v = order->value(i); // only true ints (not bools or floats) count+ if (v.kind() != Kind::Int || v.as_int() < 0 || uint64_t(v.as_int()) >= n || seen[size_t(v.as_int())]) {+ good = false;+ } else {+ seen[size_t(v.as_int())] = true;+ ints.push_back(int(v.as_int()));+ }+ }+ if (!good) return Refuse<Question>(OrderReason(n, pyvalue::Repr(*order)));+ q.order_ = std::move(ints);+ }+ if (const Value* labels = def.Find("labels")) {+ if (typ != Type::Noul) return Refuse<Question>(kLabelsOnlyNoul);+ if (!labels->is_null()) { // _resolve_noul_labels(None) is the default pair+ if (labels->kind() != Kind::Object || labels->size() != 2) return Refuse<Question>(kNoulLabels);+ std::string f, t;+ bool have_f = false, have_t = false;+ for (size_t i = 0; i < 2; i++) {+ bool is_str = labels->value(i).kind() == Kind::String;+ const std::string s = is_str ? labels->value(i).as_string() : "";+ if (labels->key(i) == "false") { f = s; have_f = is_str; }+ else if (labels->key(i) == "true") { t = s; have_t = is_str; }+ else return Refuse<Question>(kNoulLabels);+ }+ f = ud::Strip(f);+ t = ud::Strip(t);+ if (!have_f || !have_t || f.empty() || t.empty() || f == t) return Refuse<Question>(kNoulLabels);+ q.false_label_ = f;+ q.true_label_ = t;+ q.custom_labels_ = true;+ }+ }+ return Result<Question>::Ok(std::move(q));+}++// ---- Questions ---------------------------------------------------------------------------------++namespace {+Result<Questions> IdRefusal(const std::string& id, const std::string& reason) {+ return Result<Questions>::Fail(id.empty() ? reason : "question " + pyjson::Repr(id) + ": " + reason);+}+std::string CheckId(const std::string& id) {+ if (ud::IsBlank(id)) return "question id must be a non-empty string, got " + pyjson::Repr(id);+ return "";+}+} // namespace++Result<Questions> Questions::Of(const std::vector<Entry>& entries) {+ Questions qs;+ for (const auto& e : entries) {+ if (auto r = CheckId(e.id); !r.empty()) return Result<Questions>::Fail(r);+ if (e.question.type() == Type::Invalid) {+ return IdRefusal(e.id, "zero Question; build one with Choice, ChoiceOf, Score, Noul");+ }+ if (qs.index_.count(e.id)) {+ return IdRefusal(e.id, "id is given twice; answers are keyed by id, so every question needs its own");+ }+ qs.index_[e.id] = qs.entries_.size();+ qs.entries_.push_back(e);+ }+ return Result<Questions>::Ok(std::move(qs));+}++Result<Questions> Questions::Parse(std::string_view json) {+ auto v = pyvalue::Parse(json);+ if (!v.ok()) return Result<Questions>::Fail("schema: questions: " + v.error());+ const Value& root = v.value();+ if (root.kind() != Kind::Object) {+ return Result<Questions>::Fail("schema: questions must be a JSON object, got " + pyvalue::TypeName(root));+ }+ Questions qs;+ for (size_t i = 0; i < root.size(); i++) {+ const std::string& id = root.key(i);+ if (auto r = CheckId(id); !r.empty()) return Result<Questions>::Fail(r);+ auto q = Question::FromWire(root.value(i));+ if (!q.ok()) return IdRefusal(id, q.error());+ qs.index_[id] = qs.entries_.size();+ qs.entries_.push_back({id, std::move(q).value()});+ }+ return Result<Questions>::Ok(std::move(qs));+}++const Question* Questions::Get(std::string_view id) const {+ auto it = index_.find(id);+ return it == index_.end() ? nullptr : &entries_[it->second].question;+}++std::vector<std::string> Questions::Ids() const {+ std::vector<std::string> out;+ for (const auto& e : entries_) out.push_back(e.id);+ return out;+}++Result<std::string> Questions::ToJson() const {+ std::string b = "{";+ for (size_t i = 0; i < entries_.size(); i++) {+ auto q = entries_[i].question.ToJson();+ if (!q.ok()) return q;+ b += (i ? "," : "") + JsonString(entries_[i].id) + ":" + q.value();+ }+ return Result<std::string>::Ok(b + "}");+}++// ---- answers and prediction --------------------------------------------------------------------++namespace {++Result<std::string> Float(double f) {+ if (std::isnan(f) || std::isinf(f)) return Refuse<std::string>("schema: cannot marshal " + pyjson::ReprFloat(f) + " as JSON");+ return Result<std::string>::Ok(pyjson::ReprFloat(f));+}++#define LAYA_TRY_APPEND(out, expr) \+ do { \+ auto r_ = (expr); \+ if (!r_.ok()) return r_; \+ (out) += r_.value(); \+ } while (0)++Result<std::string> Probabilities(const std::vector<Probability>& ps) {+ std::string b = "{";+ for (size_t i = 0; i < ps.size(); i++) {+ b += (i ? "," : "") + JsonString(ps[i].label.Key()) + ":";+ LAYA_TRY_APPEND(b, Float(ps[i].p));+ }+ return Result<std::string>::Ok(b + "}");+}++} // namespace++Result<std::string> Answer::ToJson() const {+ std::string b;+ switch (type) {+ case Type::Choice:+ b = "{\"type\":\"choice\",\"choice\":" + choice.Json() + ",\"probabilities\":";+ LAYA_TRY_APPEND(b, Probabilities(probabilities));+ break;+ case Type::Score:+ b = "{\"type\":\"score\",\"score\":";+ LAYA_TRY_APPEND(b, Float(score));+ b += ",\"legend\":{";+ for (size_t i = 0; i < legend.size(); i++) b += (i ? ",\"" : "\"") + N(int64_t(i)) + "\":" + JsonString(legend[i]);+ b += "},\"probabilities\":";+ LAYA_TRY_APPEND(b, Probabilities(probabilities));+ break;+ case Type::Noul:+ b = "{\"type\":\"noul\",\"noul\":";+ LAYA_TRY_APPEND(b, Float(noul));+ break;+ case Type::Invalid: return Refuse<std::string>("schema: cannot marshal an answer of no type");+ }+ b += ",\"confidence\":";+ LAYA_TRY_APPEND(b, Float(confidence));+ b += ",\"answer_confidence\":";+ LAYA_TRY_APPEND(b, Float(answer_confidence));+ b += ",\"action\":{\"act_probability\":";+ LAYA_TRY_APPEND(b, Float(act_probability));+ b += "}";+ if (window) {+ b += ",\"window\":{\"index\":" + N(window->index) + ",\"token_start\":" + N(window->token_start) ++ ",\"token_end\":" + N(window->token_end) + ",\"count\":" + N(window->count) + "}";+ }+ if (low_confidence) b += ",\"low_confidence\":true";+ if (!abstention.empty()) b += ",\"abstention\":" + JsonString(abstention);+ if (abstention_threshold) {+ b += ",\"abstention_threshold\":";+ LAYA_TRY_APPEND(b, Float(*abstention_threshold));+ }+ return Result<std::string>::Ok(b + "}");+}++namespace {++Result<double> Number(const Value& v, const std::string& what) {+ switch (v.kind()) {+ case Kind::Int: return Result<double>::Ok(double(v.as_int()));+ case Kind::BigInt: return Result<double>::Ok(std::strtod(v.big_int().c_str(), nullptr));+ case Kind::Float: return Result<double>::Ok(v.as_float());+ default: break;+ }+ return Result<double>::Fail("schema: \"" + what + "\" must be a number, got " + pyvalue::TypeName(v));+}++Result<const Value*> Field(const Value& obj, const char* key) {+ const Value* v = obj.Find(key);+ if (v == nullptr) return Result<const Value*>::Fail(std::string("schema: answer has no \"") + key + "\"");+ return Result<const Value*>::Ok(v);+}++Result<double> FieldNumber(const Value& obj, const char* key) {+ auto v = Field(obj, key);+ if (!v.ok()) return Result<double>::Fail(v.error());+ return Number(*v.value(), key);+}++Result<std::vector<Probability>> ProbsFrom(const Value& v) {+ using R = Result<std::vector<Probability>>;+ if (v.kind() != Kind::Object) return R::Fail("schema: probabilities must be an object, got " + pyvalue::TypeName(v));+ std::vector<Probability> out;+ for (size_t i = 0; i < v.size(); i++) {+ auto p = Number(v.value(i), v.key(i));+ if (!p.ok()) return R::Fail(p.error());+ out.push_back({Label::Text(v.key(i)), p.value()});+ }+ return R::Ok(std::move(out));+}++} // namespace++Result<Answer> Answer::FromJson(const Value& v) {+ using R = Result<Answer>;+ if (v.kind() != Kind::Object) return R::Fail("schema: an answer is a JSON object, got " + pyvalue::TypeName(v));+ Answer a;+ auto tv = Field(v, "type");+ if (!tv.ok()) return R::Fail(tv.error());+ if (tv.value()->kind() != Kind::String) {+ return R::Fail("schema: answer \"type\" must be a string, got " + pyvalue::TypeName(*tv.value()));+ }+ const std::string& ts = tv.value()->as_string();+ a.type = ts == "choice" ? Type::Choice : ts == "score" ? Type::Score : ts == "noul" ? Type::Noul : Type::Invalid;+ auto conf = FieldNumber(v, "confidence");+ if (!conf.ok()) return R::Fail(conf.error());+ a.confidence = conf.value();+ auto ac = FieldNumber(v, "answer_confidence");+ if (!ac.ok()) return R::Fail(ac.error());+ a.answer_confidence = ac.value();+ auto act = Field(v, "action");+ if (!act.ok()) return R::Fail(act.error());+ if (act.value()->kind() != Kind::Object) return R::Fail("schema: answer \"action\" must be an object");+ auto ap = FieldNumber(*act.value(), "act_probability");+ if (!ap.ok()) return R::Fail(ap.error());+ a.act_probability = ap.value();+ if (const Value* w = v.Find("window")) {+ if (w->kind() != Kind::Object) return R::Fail("schema: answer \"window\" must be an object");+ int64_t f[4];+ const char* keys[4] = {"index", "token_start", "token_end", "count"};+ for (int i = 0; i < 4; i++) {+ auto n = FieldNumber(*w, keys[i]);+ if (!n.ok() || n.value() != std::trunc(n.value()) || std::fabs(n.value()) > 9.0e15) {+ return R::Fail(std::string("schema: answer window \"") + keys[i] + "\" must be an integer");+ }+ f[i] = int64_t(n.value());+ }+ if (f[3] < 1 || f[0] < 0 || f[0] >= f[3] || f[1] < 0 || f[2] < f[1]) return R::Fail("schema: answer \"window\" is out of range");+ a.window = Window{f[0], f[1], f[2], f[3]};+ }+ if (const Value* lc = v.Find("low_confidence")) {+ if (lc->kind() != Kind::Bool || !lc->as_bool()) return R::Fail("schema: answer \"low_confidence\" must be true when present");+ a.low_confidence = true;+ }+ if (const Value* ab = v.Find("abstention")) {+ if (ab->kind() != Kind::String || (ab->as_string() != "passed" && ab->as_string() != "abstained" && ab->as_string() != "unevaluated")) {+ return R::Fail("schema: answer \"abstention\" must be one of \"passed\", \"abstained\", \"unevaluated\", got " + pyvalue::Repr(*ab));+ }+ a.abstention = ab->as_string();+ }+ if (v.Find("abstention_threshold") != nullptr) {+ auto t = FieldNumber(v, "abstention_threshold");+ if (!t.ok()) return R::Fail(t.error());+ a.abstention_threshold = t.value();+ }+ if (a.abstention.empty() != !a.abstention_threshold.has_value()) {+ return R::Fail("schema: answer \"abstention\" and \"abstention_threshold\" come together");+ }+ if (a.type == Type::Choice) {+ auto c = Field(v, "choice");+ if (!c.ok()) return R::Fail(c.error());+ auto l = Label::Of(*c.value());+ if (!l.ok()) return R::Fail("schema: answer \"choice\" must be a string, bool or number, got " + pyvalue::TypeName(*c.value()));+ a.choice = l.value();+ } else if (a.type == Type::Score) {+ auto s = FieldNumber(v, "score");+ if (!s.ok()) return R::Fail(s.error());+ a.score = s.value();+ auto lg = Field(v, "legend");+ if (!lg.ok()) return R::Fail(lg.error());+ const Value& lo = *lg.value();+ if (lo.kind() != Kind::Object) return R::Fail("schema: answer \"legend\" must be an object");+ for (size_t i = 0; i < lo.size(); i++) {+ if (lo.value(i).kind() != Kind::String || lo.key(i) != N(int64_t(i))) {+ return R::Fail("schema: answer \"legend\" must map \"0\", \"1\", ... to text, got key " + lo.key(i));+ }+ a.legend.push_back(lo.value(i).as_string());+ }+ } else if (a.type == Type::Noul) {+ auto n = FieldNumber(v, "noul");+ if (!n.ok()) return R::Fail(n.error());+ a.noul = n.value();+ } else {+ return R::Fail("schema: unknown answer type " + pyjson::Repr(ts));+ }+ if (a.type != Type::Noul) {+ auto p = Field(v, "probabilities");+ if (!p.ok()) return R::Fail(p.error());+ auto ps = ProbsFrom(*p.value());+ if (!ps.ok()) return R::Fail(ps.error());+ a.probabilities = std::move(ps).value();+ }+ return R::Ok(std::move(a));+}++Result<std::string> Usage::ToJson() const {+ bool scanned = windows > 1;+ if (scanned && state_tokens == 0 && state_tokens_dropped == 0 && truncated_windows == 0 &&+ truncated_questions.empty() && collapsed_options.empty()) {+ return Result<std::string>::Ok("{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) ++ ",\"windows\":" + N(windows) + "}");+ }+ std::string b = "{\"input_tokens\":" + N(input_tokens) + ",\"output_tokens\":" + N(output_tokens) ++ ",\"state_tokens\":" + N(state_tokens) + ",\"state_tokens_dropped\":" + N(state_tokens_dropped) ++ ",\"truncated\":" + (scanned ? N(truncated_windows) : std::string(truncated ? "true" : "false")) ++ ",\"truncated_questions\":[";+ for (size_t i = 0; i < truncated_questions.size(); i++) b += (i ? "," : "") + JsonString(truncated_questions[i]);+ b += "]";+ if (!collapsed_options.empty()) {+ b += ",\"options\":{";+ for (size_t i = 0; i < collapsed_options.size(); i++) {+ const auto& c = collapsed_options[i];+ b += (i ? "," : "") + JsonString(c.id) + ":{\"total\":" + N(c.options.total) + ",\"distinct\":" ++ N(c.options.distinct) + ",\"tokens_per_option\":" ++ (c.options.tokens_per_option ? N(*c.options.tokens_per_option) : std::string("null")) + "}";+ }+ b += "}";+ }+ if (windows > 0) b += ",\"windows\":" + N(windows);+ return Result<std::string>::Ok(b + "}");+}++Result<std::string> Prediction::ToJson() const {+ std::string b = "{\"model\":" + JsonString(model) + ",\"answers\":{";+ for (size_t i = 0; i < answers.size(); i++) {+ auto a = answers[i].answer.ToJson();+ if (!a.ok()) return a;+ b += (i ? "," : "") + JsonString(answers[i].id) + ":" + a.value();+ }+ b += "},\"usage\":";+ LAYA_TRY_APPEND(b, usage.ToJson());+ return Result<std::string>::Ok(b + "}");+}++} // namespace schema+} // namespace layadiff --git a/laya-cpp/src/sequence.cpp b/laya-cpp/src/sequence.cppnew file mode 100644index 00000000..ae10bc95--- /dev/null+++ b/laya-cpp/src/sequence.cpp@@ -0,0 +1,279 @@+#include "laya/sequence.hpp"++#include <algorithm>+#include <cstdio>+#include <set>++#include "laya/pyjson.hpp"++namespace laya {+namespace sequence {++using schema::Question;+using tokenizer::Tokenizer;++namespace {++std::string N(int64_t i) { return std::to_string(i); }++std::string ReplaceAll(std::string s, const std::string& from, const std::string& to) {+ if (from.empty()) return s;+ std::string out;+ for (size_t at = 0;;) {+ size_t hit = s.find(from, at);+ if (hit == std::string::npos) {+ out.append(s, at, std::string::npos);+ return out;+ }+ out.append(s, at, hit - at);+ out += to;+ at = hit + from.size();+ }+}++struct Specials {+ int32_t cls = 0, sep = 0, mask = 0;+ std::string mask_token;+};++Result<Specials> GetSpecials(const Tokenizer& tok) {+ Specials sp;+ auto cls = tok.Cls(), sep = tok.Sep(), mask = tok.Mask();+ sp.mask_token = tok.MaskToken();+ const char* what = "tokenizer names no required special token";+ if (!cls) return Result<Specials>::Fail(std::string(what) + ": no CLS token");+ if (!sep) return Result<Specials>::Fail(std::string(what) + ": no SEP token");+ if (!mask || sp.mask_token.empty()) return Result<Specials>::Fail(std::string(what) + ": no mask token");+ sp.cls = *cls;+ sp.sep = *sep;+ sp.mask = *mask;+ return Result<Specials>::Ok(std::move(sp));+}++int64_t FloorDiv(int64_t a, int64_t b) {+ int64_t q = a / b;+ if (a % b != 0 && a < 0) q--;+ return q;+}++bool CheckBudget(const Budget& b, std::string* err) {+ if (b.max_len <= 0 || b.head_max_len <= 0) {+ *err = "sequence: budget needs a positive MaxLen and HeadMaxLen, got " + N(b.max_len) + " and " + N(b.head_max_len);+ return false;+ }+ return true;+}++} // namespace++int64_t Budget::DefaultWindow() const { return std::max<int64_t>(64, max_len - head_max_len - 8); }++Result<std::string> SerializeState(const pyvalue::Value& state) {+ if (state.kind() == pyvalue::Value::Kind::String) return Result<std::string>::Ok(state.as_string());+ auto text = pyvalue::Dumps(state);+ if (!text.ok()) return Result<std::string>::Fail("state cannot be serialized as laya serializes it: " + text.error());+ return text;+}++Result<State> EncodeState(const Tokenizer& tok, const pyvalue::Value& state) {+ auto sp = GetSpecials(tok);+ if (!sp.ok()) return Result<State>::Fail(sp.error());+ auto text = SerializeState(state);+ if (!text.ok()) return Result<State>::Fail(text.error());+ State out;+ out.keep = state.kind() == pyvalue::Value::Kind::Array ? Keep::End : Keep::Start;+ out.ids = tok.Encode(ReplaceAll(text.value(), sp.value().mask_token, " "));+ return Result<State>::Ok(std::move(out));+}++std::string MarkersError::Message(const std::string& id) const {+ std::string prefix = id.empty() ? "" : "question " + pyjson::Repr(id) + ": ";+ return prefix + "only " + N(kept) + " of its " + N(total) + " option markers fit in max_len=" + N(max_len) ++ " with head_max_len=" + N(head_max_len) ++ " spent on the question; lower head_max_len, raise max_len, or use fewer options";+}++Result<Head> BuildHead(const Tokenizer& tok, const Question& q, int64_t head_max_len) {+ auto spr = GetSpecials(tok);+ if (!spr.ok()) return Result<Head>::Fail(spr.error());+ const Specials& sp = spr.value();+ if (q.type() == schema::Type::Invalid || q.NumOptions() == 0) {+ return Result<Head>::Fail(+ "sequence: the zero Question has no head; build one with Question::Choice, ChoiceOf, Score or Noul");+ }+ std::vector<std::string> opts = q.RenderedOptions();+ std::vector<int> order = q.OptionOrder();+ if (order.empty()) {+ for (size_t i = 0; i < opts.size(); i++) order.push_back(int(i));+ }+ std::string ins = ReplaceAll(q.instructions(), sp.mask_token, " ");+ std::vector<int32_t> head_ids = tok.Encode(std::string(schema::Name(q.type())) + " question: " + ins);+ std::vector<std::vector<int32_t>> spans(order.size());+ int64_t total = 0;+ for (size_t slot = 0; slot < order.size(); slot++) {+ std::string text = " " + ReplaceAll(opts[size_t(order[slot])], sp.mask_token, " ");+ spans[slot].push_back(sp.mask);+ std::vector<int32_t> body = tok.Encode(text, kOptionTokenCap);+ spans[slot].insert(spans[slot].end(), body.begin(), body.end());+ total += int64_t(spans[slot].size());+ }+ int64_t budget = head_max_len - total;+ std::optional<int64_t> per_option;+ if (budget < kMinOptionBudget) {+ int64_t per = std::max(kMinTokensPerOption,+ FloorDiv(head_max_len - kMinOptionBudget, std::max<int64_t>(1, int64_t(spans.size()))));+ per_option = per;+ total = 0;+ for (auto& span : spans) {+ if (int64_t(span.size()) > per) span.resize(size_t(per));+ total += int64_t(span.size());+ }+ budget = head_max_len - total;+ }+ int64_t keep = std::max(kMinInstructionTokens, budget);+ if (keep < int64_t(head_ids.size())) head_ids.resize(size_t(keep));++ Head h;+ h.ids.push_back(sp.cls);+ h.ids.insert(h.ids.end(), head_ids.begin(), head_ids.end());+ h.ids.push_back(sp.sep);+ std::set<std::vector<int32_t>> distinct;+ for (const auto& span : spans) {+ h.markers.push_back(int64_t(h.ids.size()));+ h.ids.insert(h.ids.end(), span.begin(), span.end());+ distinct.insert(span);+ }+ h.ids.push_back(sp.sep);+ h.stats = {int64_t(spans.size()), int64_t(distinct.size()), per_option};+ return Result<Head>::Ok(std::move(h));+}++Result<Sequence> Build(const Tokenizer& tok, const Question& q, const State& state, const Budget& b,+ MarkersError* markers_error) {+ std::string err;+ if (!CheckBudget(b, &err)) return Result<Sequence>::Fail(err);+ auto head = BuildHead(tok, q, b.head_max_len);+ if (!head.ok()) return Result<Sequence>::Fail(head.error());+ const Head& h = head.value();+ int32_t sep = *tok.Sep(); // BuildHead found it+ int64_t room = std::max<int64_t>(0, b.max_len - int64_t(h.ids.size()) - 1);+ int64_t have = int64_t(state.ids.size());+ int64_t used = std::min(have, room);+ Sequence s;+ s.ids = h.ids;+ size_t from = state.keep == Keep::End ? size_t(have - used) : 0;+ s.ids.insert(s.ids.end(), state.ids.begin() + std::ptrdiff_t(from), state.ids.begin() + std::ptrdiff_t(from + size_t(used)));+ s.ids.push_back(sep);+ if (int64_t(s.ids.size()) > b.max_len) s.ids.resize(size_t(b.max_len));+ for (int64_t m : h.markers) {+ if (m < b.max_len) s.markers.push_back(m);+ }+ if (s.markers.size() != h.markers.size()) {+ MarkersError e{int64_t(s.markers.size()), int64_t(h.markers.size()), b.max_len, b.head_max_len};+ if (markers_error != nullptr) *markers_error = e;+ return Result<Sequence>::Fail(e.Message());+ }+ s.stats = h.stats;+ s.truncation = {have, used, have - used, used < have};+ return Result<Sequence>::Ok(std::move(s));+}++Result<int64_t> StateRoom(const Tokenizer& tok, const Question& q, const Budget& b) {+ std::string err;+ if (!CheckBudget(b, &err)) return Result<int64_t>::Fail(err);+ auto head = BuildHead(tok, q, b.head_max_len);+ if (!head.ok()) return Result<int64_t>::Fail(head.error());+ return Result<int64_t>::Ok(std::max<int64_t>(0, b.max_len - int64_t(head.value().ids.size()) - 1)); // -1: closing [SEP]+}++namespace {+constexpr int64_t kClampWarnRatio = 2;+constexpr int64_t kBatchBlowup = 2;++std::string Ratio1(double x) {+ char buf[48];+ std::snprintf(buf, sizeof buf, "%.1f", x);+ return buf;+}+} // namespace++Result<WindowPlan> PlanWindows(const Tokenizer& tok, const std::vector<Question>& questions, const Budget& b,+ const WindowRequest& req, PlanError* kind) {+ if (kind != nullptr) *kind = PlanError::Other;+ std::string err;+ if (!CheckBudget(b, &err)) return Result<WindowPlan>::Fail(err);+ std::vector<int64_t> rooms;+ for (const auto& q : questions) {+ auto r = StateRoom(tok, q, b);+ if (!r.ok()) return Result<WindowPlan>::Fail(r.error());+ rooms.push_back(r.value());+ }+ int64_t requested = req.window > 0 ? req.window : b.DefaultWindow();+ int64_t size = requested;+ int64_t room = rooms.empty() ? size : *std::min_element(rooms.begin(), rooms.end());+ if (room <= 0) {+ if (kind != nullptr) *kind = PlanError::NoRoom;+ return Result<WindowPlan>::Fail(+ "predict_long: the questions' options fill the whole sequence (max_len=" + N(b.max_len) + ", head_max_len=" ++ N(b.head_max_len) + "), leaving no room for the state; no window can carry any of it. A label set this large "+ "is what laya.shortlist.predict_shortlist is for");+ }+ WindowPlan plan;+ if (size > room) {+ if (req.window > 0) {+ plan.warnings.push_back("laya: predict_long: window=" + N(size) + " is wider than the " + N(room) ++ " state tokens these questions leave inside max_len=" + N(b.max_len) ++ ", so every window would be truncated to " + N(room) ++ " on the way to the model; scanning with window=" + N(room) + " instead");+ } else if (size / kClampWarnRatio >= room) { // size >= room * kClampWarnRatio, without the overflow+ plan.warnings.push_back("laya: predict_long: these questions leave only " + N(room) + " of max_len=" ++ N(b.max_len) + " for the state (their heads take the rest), so the scan window is "+ "capped " + N(size) + " -> " + N(room) + " and roughly " ++ Ratio1(double(size) / double(room)) + "x as many windows -- each a full forward pass -- "+ "are needed to read the document. Fewer or shorter options, a larger max_len, or "+ "laya.shortlist.predict_shortlist for a large label set will all cost less than scanning "+ "at this width");+ }+ size = room;+ }+ int64_t step = req.stride > 0 ? req.stride : std::max<int64_t>(1, size / 2);+ if (step > size) {+ if (size < requested && req.stride > 0 && req.stride <= requested) {+ plan.warnings.push_back("laya: predict_long: stride=" + N(step) + " was a 50% step for the window=" ++ N(requested) + " you asked for, but the window was reduced to " + N(size) ++ " to fit the room these questions leave; scanning with stride=" ++ N(std::max<int64_t>(1, size / 2)) + " instead");+ step = std::max<int64_t>(1, size / 2);+ } else {+ if (kind != nullptr) *kind = PlanError::StrideGap;+ return Result<WindowPlan>::Fail(+ "predict_long: stride=" + N(step) + " steps past the " + N(size) + "-token window" ++ (size < requested ? " these questions leave room for" : "") + ", so " + N(step - size) ++ " tokens between every pair of windows would be read by no window at all; pass stride <= " + N(size));+ }+ }+ plan.window = size;+ plan.stride = step;+ plan.room = room;+ if (kind != nullptr) *kind = PlanError::None;+ return Result<WindowPlan>::Ok(std::move(plan));+}++Cap BatchCap(int64_t n_windows, int64_t window, int64_t config_window, int64_t batch_size) {+ if (batch_size > 0) return {batch_size, true};+ if (window >= config_window) return {0, false}; // not capped: one pass, exactly as before+#if defined(__SIZEOF_INT128__)+ __extension__ typedef __int128 Wide;+#else+ typedef long long Wide; // no 128-bit integer: callers beyond 2^31 windows are out of contract+#endif+ Wide num = Wide(n_windows) * Wide(std::max<int64_t>(1, FloorDiv(window, 2)));+ Wide den = Wide(std::max<int64_t>(1, FloorDiv(config_window, 2)));+ Wide unclamped = num > 0 ? (num + den - 1) / den : 1; // ceil(num / den), at least 1+ if (unclamped < 1) unclamped = 1;+ if (Wide(n_windows) <= unclamped * kBatchBlowup) return {0, false}; // a mild cap: keep the single shared pass+ return {int64_t(unclamped), true}; // below n_windows, so it fits+}++} // namespace sequence+} // namespace layadiff --git a/laya-cpp/tests/checkpoint.hpp b/laya-cpp/tests/checkpoint.hppnew file mode 100644index 00000000..66048cf1--- /dev/null+++ b/laya-cpp/tests/checkpoint.hpp@@ -0,0 +1,43 @@+#pragma once+// A shipped checkpoint's tokenizer, loaded once. Without LAYA_CHECKPOINTS the test is skipped+// locally and FAILS under LAYA_REQUIRE_GOLDENS=1 (CI), so a missing checkpoint cannot pass as green.+#include <gtest/gtest.h>++#include <cstdlib>+#include <map>+#include <mutex>+#include <string>++#include "laya/tokenizer.hpp"++namespace laya_test {++inline const laya::tokenizer::Tokenizer* LoadCheckpoint(const std::string& name) {+ static std::mutex mu;+ static std::map<std::string, laya::tokenizer::Tokenizer> loaded;+ std::lock_guard<std::mutex> lock(mu);+ auto it = loaded.find(name);+ if (it != loaded.end()) return &it->second;+ const char* root = std::getenv("LAYA_CHECKPOINTS");+ if (root == nullptr || *root == 0) return nullptr;+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(root) + "/" + name);+ if (!t.ok()) {+ ADD_FAILURE() << "load the " << name << " tokenizer: " << t.error();+ return nullptr;+ }+ return &loaded.emplace(name, std::move(t).value()).first->second;+}++#define LAYA_CHECKPOINT_TOK(var, name) \+ const laya::tokenizer::Tokenizer* var = laya_test::LoadCheckpoint(name); \+ if (var == nullptr) { \+ const char* req = std::getenv("LAYA_REQUIRE_GOLDENS"); \+ if (req != nullptr && std::string(req) == "1" && !HasFailure()) { \+ ADD_FAILURE() << "LAYA_CHECKPOINTS is unset but LAYA_REQUIRE_GOLDENS=1"; \+ } else if (!HasFailure()) { \+ GTEST_SKIP() << "LAYA_CHECKPOINTS unset (local run)"; \+ } \+ return; \+ }++} // namespace laya_testdiff --git a/laya-cpp/tests/schema_test.cpp b/laya-cpp/tests/schema_test.cppnew file mode 100644index 00000000..b6a5e6d0--- /dev/null+++ b/laya-cpp/tests/schema_test.cpp@@ -0,0 +1,249 @@+#include "laya/schema.hpp"++#include <gtest/gtest.h>++#include "golden.hpp"+#include "laya/pyjson.hpp"++namespace {++using laya::pyvalue::Value;+namespace sc = laya::schema;+using laya_test::At;+using Kind = Value::Kind;++std::string Dump(const Value& v) {+ auto r = laya::pyvalue::Dumps(v);+ return r.ok() ? r.value() : "<" + r.error() + ">";+}++std::vector<std::string> Strings(const Value& list) {+ std::vector<std::string> out;+ for (const Value& v : list.items()) out.push_back(v.as_string());+ return out;+}++void CheckQuestion(const sc::Question& q, const Value& rec, const std::string& what) {+ EXPECT_EQ(std::string(sc::Name(q.type())), rec.Find("type")->as_string()) << what;+ EXPECT_EQ(q.instructions(), rec.Find("instructions")->as_string()) << what;+ EXPECT_EQ(q.RenderedOptions(), Strings(*rec.Find("options"))) << what;+ EXPECT_EQ(q.Labels(), Strings(*rec.Find("labels"))) << what;+ if (const Value* lg = rec.Find("legend"); lg != nullptr && !lg->is_null()) EXPECT_EQ(q.Legend(), Strings(*lg)) << what;+ std::vector<int> want;+ if (const Value* o = rec.Find("option_order"); o != nullptr && !o->is_null()) {+ for (const Value& v : o->items()) want.push_back(int(v.as_int()));+ }+ EXPECT_EQ(q.OptionOrder(), want) << what;+}++class QuestionRow : public ::testing::TestWithParam<int> {};+TEST_P(QuestionRow, MatchesAgentCheckQuestion) {+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");+ LAYA_EXPECT_ROWS(rows, 140);+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {+ const Value& rec = rows->value(i);+ const std::string name = rec.Find("case")->as_string();+ const std::string id = rec.Find("id")->as_string();+ Value doc = Value::Object();+ doc.Set(id, *rec.Find("definition"));+ auto qs = sc::Questions::Parse(Dump(doc));+ if (const Value* want = rec.Find("error")) {+ ASSERT_FALSE(qs.ok()) << name << ": accepted; Python refuses with " << want->as_string();+ EXPECT_EQ(qs.error(), want->as_string()) << name;+ continue;+ }+ ASSERT_TRUE(qs.ok()) << name << ": " << qs.error();+ const sc::Question* q = qs.value().Get(id);+ ASSERT_NE(q, nullptr) << name;+ ASSERT_EQ(qs.value().size(), 1u) << name;+ CheckQuestion(*q, rec, name);+ auto out = qs.value().ToJson();+ ASSERT_TRUE(out.ok()) << name << ": " << out.error();+ auto again = sc::Questions::Parse(out.value());+ ASSERT_TRUE(again.ok()) << name << ": own output " << out.value() << ": " << again.error();+ CheckQuestion(*again.value().Get(id), rec, name + " (round trip)");+ EXPECT_EQ(again.value().ToJson().value(), out.value()) << name << ": marshal is not stable";+ }+}+INSTANTIATE_TEST_SUITE_P(Golden, QuestionRow, ::testing::ValuesIn(laya_test::Chunks(140)));++TEST(Questions, RefusalAndAcceptanceCounts) {+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");+ int refused = 0, accepted = 0;+ for (const Value& rec : rows->items()) (rec.Find("error") ? refused : accepted)++;+ EXPECT_EQ(refused, 93);+ EXPECT_EQ(accepted, 47);+}++// The typed constructors refuse with the wire reader's words.+TEST(Builders, ReasonsMatchPython) {+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "questions", "cases");+ std::map<std::string, std::string> gold;+ const std::string prefix = "question 'q': ";+ for (const Value& rec : rows->items()) {+ if (const Value* e = rec.Find("error")) gold[rec.Find("case")->as_string()] = e->as_string().substr(prefix.size());+ }+ auto two = sc::Question::ChoiceOf("Pick.", {"a", "b"}).value();+ auto noul = sc::Question::Noul("Holds?").value();+ auto spec = [](const char* f, const char* t) {+ sc::NoulSpec s;+ s.false_label = f;+ s.true_label = t;+ return s;+ };+ struct Row {+ const char* gold;+ std::string got;+ };+ auto err = [](const auto& r) { return r.ok() ? std::string("<accepted>") : r.error(); };+ std::vector<Row> cases = {+ {"ins-blank", err(sc::Question::ChoiceOf(" \t\n", {"a", "b"}))},+ {"ins-nbsp", err(sc::Question::Score(" \xc2\x85", {Value::String("a")}))},+ {"ins-ideographic-space", err(sc::Question::Noul("\xe3\x80\x80"))},+ {"ins-empty", err(sc::Question::Choice("", {{sc::Label::Text("a"), Value()}}))},+ {"choice-empty-list", err(sc::Question::Choice("Pick.", {}))},+ {"choice-list-repeat", err(sc::Question::ChoiceOf("Pick.", {"a", "b", "a"}))},+ {"score-empty", err(sc::Question::Score("Rate.", {}))},+ {"score-null-second", err(sc::Question::Score("Rate.", {Value::String("a"), Value(), Value::String("c")}))},+ {"labels-same", err(sc::Question::Noul("Holds?", spec("a", "a")))},+ {"labels-same-after-strip", err(sc::Question::Noul("Holds?", spec("a", " a ")))},+ {"labels-missing-true", err(sc::Question::Noul("Holds?", spec("a", "")))},+ {"labels-blank-value", err(sc::Question::Noul("Holds?", spec(" ", "x")))},+ {"order-short", err(two.WithOptionOrder({0}))},+ {"order-long", err(two.WithOptionOrder({0, 1, 2}))},+ {"order-repeat", err(two.WithOptionOrder({0, 0}))},+ {"order-range", err(two.WithOptionOrder({0, 2}))},+ {"order-negative", err(two.WithOptionOrder({-1, 0}))},+ {"order-empty", err(two.WithOptionOrder({}))},+ {"order-noul-wrong-size", err(noul.WithOptionOrder({0, 1, 2}))},+ };+ for (const Row& c : cases) {+ auto it = gold.find(c.gold);+ ASSERT_NE(it, gold.end()) << c.gold;+ EXPECT_EQ(c.got, it->second) << c.gold;+ }+}++TEST(Builders, OptionOrderAndSubset) {+ auto q = sc::Question::ChoiceOf("Pick.", {"a", "b", "c"}).value();+ auto o = q.WithOptionOrder({2, 0, 1});+ ASSERT_TRUE(o.ok());+ EXPECT_EQ(o.value().OptionOrder(), (std::vector<int>{2, 0, 1}));+ EXPECT_EQ(o.value().RenderedOptions(), q.RenderedOptions()); // rendering is label order; the order applies later+ EXPECT_TRUE(o.value().WithoutOptionOrder().OptionOrder().empty());+ EXPECT_TRUE(q.OptionOrder().empty()); // a question is immutable+ auto s = o.value().Subset({2, 0});+ ASSERT_TRUE(s.ok());+ EXPECT_EQ(s.value().Labels(), (std::vector<std::string>{"c", "a"}));+ EXPECT_TRUE(s.value().OptionOrder().empty());+ EXPECT_FALSE(q.Subset({}).ok());+ EXPECT_FALSE(q.Subset({0, 0}).ok());+ EXPECT_FALSE(q.Subset({3}).ok());+ EXPECT_FALSE(sc::Question().WithOptionOrder({0}).ok());+ EXPECT_FALSE(sc::Question().ToJson().ok());+}++TEST(Labels, PythonDictKeying) {+ using L = sc::Label;+ EXPECT_EQ(L::OfInt(1).EqKey(), L::OfFloat(1.0).EqKey());+ EXPECT_EQ(L::OfInt(1).EqKey(), L::OfBool(true).EqKey());+ EXPECT_NE(L::OfInt(1).EqKey(), L::Text("1").EqKey());+ EXPECT_NE(L::OfFloat(0.5).EqKey(), L::OfInt(0).EqKey());+ EXPECT_EQ(L::OfFloat(1e22).EqKey(), "n:10000000000000000000000");+ EXPECT_EQ(L::OfFloat(-0.0).EqKey(), L::OfInt(0).EqKey());+ EXPECT_EQ(L::OfBool(true).String(), "True");+ EXPECT_EQ(L::OfFloat(1.0 / 0.0).String(), "inf");+ EXPECT_EQ(L::OfFloat(1.0 / 0.0).Key(), "Infinity");+ EXPECT_FALSE(L::Of(Value()).ok());+ EXPECT_FALSE(L::Of(Value::Array()).ok());+ // 1, 1.0 and true are one dict key, so a question may not hold two of them.+ auto q = sc::Question::Choice("Pick.", {{L::OfInt(1), Value()}, {L::OfFloat(1.0), Value()}});+ ASSERT_FALSE(q.ok());+ EXPECT_EQ(q.error(),+ "choice label 1 (1.0) repeats label 0; the labels are the answer keys, so every option needs its own (1, "+ "1.0 and True are one key)");+}++TEST(Questions, Duplicates) {+ auto q = sc::Question::Noul("Holds?").value();+ EXPECT_FALSE(sc::Questions::Of({{"a", q}, {"a", q}}).ok());+ EXPECT_FALSE(sc::Questions::Of({{" ", q}}).ok());+ EXPECT_FALSE(sc::Questions::Of({{"a", sc::Question()}}).ok());+ EXPECT_EQ(sc::Questions::Of({{"b", q}, {"a", q}}).value().Ids(), (std::vector<std::string>{"b", "a"}));+}++// Answer marshals as Python writes it: parse the Python text, write it back, and get it byte for byte.+class AnswerRow : public ::testing::TestWithParam<int> {};+TEST_P(AnswerRow, RoundTripsPythonText) {+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "label_answers", "cases");+ LAYA_EXPECT_ROWS(rows, 500);+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {+ const std::string text = rows->value(i).Find("answer_text")->as_string();+ auto v = laya::pyvalue::Parse(text);+ ASSERT_TRUE(v.ok()) << i;+ auto a = sc::Answer::FromJson(v.value());+ ASSERT_TRUE(a.ok()) << i << ": " << a.error();+ auto out = a.value().ToJson();+ ASSERT_TRUE(out.ok()) << i << ": " << out.error();+ // Python writes 1 and "1" as two keys of one object; a parsed object keeps one. Those rows are+ // the ones with fewer probabilities than options and are not byte-comparable after a parse.+ const Value* opts = rows->value(i).Find("options");+ if (a.value().type == sc::Type::Choice && a.value().probabilities.size() < opts->size()) continue;+ EXPECT_EQ(out.value(), text) << "row " << i;+ }+}+INSTANTIATE_TEST_SUITE_P(Golden, AnswerRow, ::testing::ValuesIn(laya_test::Chunks(500)));++TEST(Answer, GateFieldsAndRefusals) {+ sc::Answer a;+ a.type = sc::Type::Noul;+ a.noul = 0.5;+ a.confidence = 0.25;+ a.answer_confidence = 0.75;+ a.act_probability = 1;+ a.window = sc::Window{1, 10, 20, 3};+ a.low_confidence = true;+ a.abstention = "abstained";+ a.abstention_threshold = 0.5;+ const char* want =+ "{\"type\":\"noul\",\"noul\":0.5,\"confidence\":0.25,\"answer_confidence\":0.75,\"action\":{\"act_probability\":1.0},"+ "\"window\":{\"index\":1,\"token_start\":10,\"token_end\":20,\"count\":3},\"low_confidence\":true,"+ "\"abstention\":\"abstained\",\"abstention_threshold\":0.5}";+ EXPECT_EQ(a.ToJson().value(), want);+ auto back = sc::Answer::FromJson(laya::pyvalue::Parse(want).value());+ ASSERT_TRUE(back.ok()) << back.error();+ EXPECT_EQ(back.value().ToJson().value(), want);+ a.noul = 0.0 / 0.0; // NaN is not JSON+ EXPECT_FALSE(a.ToJson().ok());+ a.noul = 0.5;+ a.abstention.clear(); // the gate fields come together+ auto lone = a.ToJson();+ ASSERT_TRUE(lone.ok());+ EXPECT_FALSE(sc::Answer::FromJson(laya::pyvalue::Parse(lone.value()).value()).ok());+ EXPECT_FALSE(sc::Answer().ToJson().ok());+}++TEST(Prediction, UsageShapes) {+ sc::Prediction p;+ p.usage.input_tokens = 10;+ p.usage.output_tokens = 2;+ p.usage.truncated_questions = {"q"};+ p.usage.truncated = true;+ p.usage.collapsed_options = {{"q", {3, 1, 4}}, {"r", {2, 2, std::nullopt}}};+ EXPECT_EQ(p.ToJson().value(),+ "{\"model\":\"laya-rl-agent-onnx\",\"answers\":{},\"usage\":{\"input_tokens\":10,\"output_tokens\":2,"+ "\"state_tokens\":0,\"state_tokens_dropped\":0,\"truncated\":true,\"truncated_questions\":[\"q\"],"+ "\"options\":{\"q\":{\"total\":3,\"distinct\":1,\"tokens_per_option\":4},\"r\":{\"total\":2,\"distinct\":2,"+ "\"tokens_per_option\":null}}}}");+ sc::Usage scan;+ scan.input_tokens = 5;+ scan.output_tokens = 1;+ scan.windows = 3;+ EXPECT_EQ(scan.ToJson().value(), "{\"input_tokens\":5,\"output_tokens\":1,\"windows\":3}");+ scan.truncated_windows = 2;+ EXPECT_EQ(scan.ToJson().value(),+ "{\"input_tokens\":5,\"output_tokens\":1,\"state_tokens\":0,\"state_tokens_dropped\":0,\"truncated\":2,"+ "\"truncated_questions\":[],\"windows\":3}");+}++} // namespacediff --git a/laya-cpp/tests/sequence_test.cpp b/laya-cpp/tests/sequence_test.cppnew file mode 100644index 00000000..0a6afab5--- /dev/null+++ b/laya-cpp/tests/sequence_test.cpp@@ -0,0 +1,342 @@+#include "laya/sequence.hpp"++#include <gtest/gtest.h>++#include "checkpoint.hpp"+#include "golden.hpp"++namespace {++using laya::pyvalue::Value;+namespace sc = laya::schema;+namespace sq = laya::sequence;+using laya_test::At;+using Kind = Value::Kind;++const char* const kNames[] = {"english", "multilingual"};++sc::Question GoldenQuestion(const Value& g, const Value& order) {+ const std::string kind = g.Find("t")->as_string();+ const std::string ins = g.Find("ins")->as_string();+ const Value* crit = g.Find("crit");+ laya::Result<sc::Question> q = laya::Result<sc::Question>::Fail("unset");+ if (kind == "choice") {+ std::vector<sc::Option> opts;+ for (size_t i = 0; i < crit->size(); i++) opts.push_back({sc::Label::Text(crit->key(i)), crit->value(i)});+ q = sc::Question::Choice(ins, opts);+ } else if (kind == "score") {+ q = sc::Question::Score(ins, crit->items());+ } else {+ sc::NoulSpec spec;+ if (crit != nullptr && crit->kind() == Kind::Object) {+ if (const Value* t = crit->Find("true")) spec.true_text = *t;+ if (const Value* f = crit->Find("false")) spec.false_text = *f;+ }+ if (const Value* l = g.Find("labels"); l != nullptr && !l->is_null()) {+ spec.false_label = l->Find("false")->as_string();+ spec.true_label = l->Find("true")->as_string();+ }+ q = sc::Question::Noul(ins, spec);+ }+ EXPECT_TRUE(q.ok()) << (q.ok() ? "" : q.error());+ if (!q.ok()) std::abort();+ if (!order.is_null()) {+ std::vector<int> o;+ for (const Value& v : order.items()) o.push_back(int(v.as_int()));+ q = q.value().WithOptionOrder(o);+ EXPECT_TRUE(q.ok());+ if (!q.ok()) std::abort();+ }+ return q.value();+}++std::vector<int32_t> Ids32(const Value& list) {+ std::vector<int32_t> out;+ for (const Value& v : list.items()) out.push_back(int32_t(v.as_int()));+ return out;+}+std::vector<int64_t> Ids64(const Value& list) {+ std::vector<int64_t> out;+ for (const Value& v : list.items()) out.push_back(v.as_int());+ return out;+}++// The generator's _window_question: options of four words each.+sc::Question WindowQuestion(int options) {+ std::vector<sc::Option> opts;+ for (int i = 0; i < options; i++) {+ std::string text;+ for (int w = 0; w < 4; w++) text += (w ? " " : "") + std::string("criterion") + std::to_string(i) + " word" + std::to_string(w);+ char label[16];+ std::snprintf(label, sizeof label, "opt%03d", i);+ opts.push_back({sc::Label::Text(label), Value::String(text)});+ }+ return sc::Question::Choice("Pick one.", opts).value();+}++class SequenceRow : public ::testing::TestWithParam<int> {};+TEST_P(SequenceRow, MatchesBuildSequence) {+ const char* name = kNames[GetParam() / 26];+ LAYA_CHECKPOINT_TOK(tok, name);+ LAYA_GOLDEN_LIST(cases, "sequences.json", name, "cases");+ LAYA_EXPECT_ROWS(cases, 26);+ const size_t at = size_t(GetParam() % 26);+ const std::string label = std::string(name) + "/" + cases->key(at);+ const Value& rec = cases->value(at);+ sc::Question q = GoldenQuestion(*rec.Find("question"), *rec.Find("option_order"));+ auto state = sq::EncodeState(*tok, *rec.Find("state"));+ ASSERT_TRUE(state.ok()) << label << ": " << state.error();+ if (rec.Find("truncate_left")->as_bool()) state.value().keep = sq::Keep::End;+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};+ sq::MarkersError dropped;+ auto seq = sq::Build(*tok, q, state.value(), b, &dropped);+ const auto want_markers = Ids64(*rec.Find("markers"));+ const int64_t option_count = rec.Find("stats")->Find("options")->as_int();+ if (!seq.ok() && dropped.total != 0) {+ // build_sequence returns the markers that fit and the agent refuses; Build does both.+ EXPECT_EQ(dropped.kept, int64_t(want_markers.size())) << label;+ EXPECT_EQ(dropped.total, option_count) << label;+ EXPECT_LT(int64_t(want_markers.size()), option_count) << label;+ return;+ }+ ASSERT_TRUE(seq.ok()) << label << ": " << seq.error();+ ASSERT_EQ(int64_t(want_markers.size()), option_count) << label << ": Python dropped markers and Build did not refuse";+ EXPECT_EQ(seq.value().ids, Ids32(*rec.Find("ids"))) << label;+ EXPECT_EQ(seq.value().markers, want_markers) << label;+ const Value& stats = *rec.Find("stats");+ EXPECT_EQ(seq.value().stats.options, stats.Find("options")->as_int()) << label;+ EXPECT_EQ(seq.value().stats.options_distinct, stats.Find("options_distinct")->as_int()) << label;+ const Value* tpo = stats.Find("tokens_per_option");+ EXPECT_EQ(seq.value().stats.tokens_per_option.has_value(), !tpo->is_null()) << label;+ if (!tpo->is_null() && seq.value().stats.tokens_per_option) EXPECT_EQ(*seq.value().stats.tokens_per_option, tpo->as_int()) << label;+ const Value& tr = *rec.Find("truncation");+ sq::Truncation want{tr.Find("state_tokens")->as_int(), tr.Find("state_tokens_used")->as_int(),+ tr.Find("state_tokens_dropped")->as_int(), tr.Find("truncated")->as_bool()};+ EXPECT_TRUE(seq.value().truncation == want) << label;+}+INSTANTIATE_TEST_SUITE_P(Golden, SequenceRow, ::testing::ValuesIn(laya_test::Rows(52)));++class Checkpoint : public ::testing::TestWithParam<int> {};++TEST_P(Checkpoint, SpecialIdsMatchPython) {+ const char* name = kNames[GetParam()];+ LAYA_CHECKPOINT_TOK(tok, name);+ LAYA_GOLDEN_LIST(ids, "sequences.json", name, "special_ids");+ EXPECT_EQ(int64_t(*tok->Cls()), ids->Find("cls")->as_int());+ EXPECT_EQ(int64_t(*tok->Sep()), ids->Find("sep")->as_int());+ EXPECT_EQ(int64_t(*tok->Mask()), ids->Find("mask")->as_int());+ EXPECT_EQ(int64_t(*tok->Pad()), ids->Find("pad")->as_int());+ EXPECT_EQ(tok->MaskToken(), ids->Find("mask_token")->as_string());+}++// The same questions and budgets succeed or fail together with Agent._encode_state, with the same text.+TEST_P(Checkpoint, MarkerRefusalsMatchPython) {+ const char* name = kNames[GetParam()];+ LAYA_CHECKPOINT_TOK(tok, name);+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "marker_refusals", name, "cases");+ LAYA_EXPECT_ROWS(cases, 42);+ int refused = 0;+ for (const Value& rec : cases->items()) {+ const int options = int(rec.Find("options")->as_int()), words = int(rec.Find("words")->as_int());+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};+ const std::string label = std::string(name) + "/" + std::to_string(options) + " options x " + std::to_string(words) ++ " words/" + std::to_string(b.max_len) + "," + std::to_string(b.head_max_len);+ std::vector<sc::Option> opts;+ for (int i = 0; i < options; i++) {+ std::string text;+ for (int j = 0; j < words; j++) text += (j ? " " : "") + std::string("w") + std::to_string(i) + "_" + std::to_string(j);+ char l[16];+ std::snprintf(l, sizeof l, "opt%03d", i);+ opts.push_back({sc::Label::Text(l), Value::String(text)});+ }+ auto q = sc::Question::Choice("Pick one.", opts);+ ASSERT_TRUE(q.ok()) << label;+ auto st = sq::EncodeState(*tok, Value::String("state text"));+ ASSERT_TRUE(st.ok()) << label;+ sq::MarkersError me;+ auto seq = sq::Build(*tok, q.value(), st.value(), b, &me);+ const Value* want = rec.Find("error");+ if (want->is_null()) {+ EXPECT_TRUE(seq.ok()) << label << ": refused: " << seq.error();+ } else {+ refused++;+ ASSERT_FALSE(seq.ok()) << label << ": not refused; Python says " << want->as_string();+ ASSERT_NE(me.total, 0) << label << ": " << seq.error();+ EXPECT_EQ(me.Message("it's"), want->as_string()) << label;+ }+ }+ EXPECT_GE(refused, 5) << name;+}++TEST_P(Checkpoint, WindowPlanMatchesPython) {+ const char* name = kNames[GetParam()];+ LAYA_CHECKPOINT_TOK(tok, name);+ LAYA_GOLDEN_LIST(entry, "window_plan.json", "by", name);+ const int64_t max_len = entry->Find("default_max_len")->as_int(), head_max = entry->Find("default_head_max_len")->as_int();+ const Value& rooms = *entry->Find("state_room");+ ASSERT_EQ(rooms.size(), 8u);+ for (const Value& rec : rooms.items()) {+ const int options = int(rec.Find("options")->as_int());+ sc::Question q = WindowQuestion(options);+ auto room = sq::StateRoom(*tok, q, {max_len, head_max});+ ASSERT_TRUE(room.ok());+ EXPECT_EQ(room.value(), rec.Find("room")->as_int()) << name << " room " << options;+ auto tight = sq::StateRoom(*tok, q, {256, 128});+ ASSERT_TRUE(tight.ok());+ EXPECT_EQ(tight.value(), rec.Find("room_tight")->as_int()) << name << " room_tight " << options;+ }+ const Value& cases = *entry->Find("cases");+ ASSERT_EQ(cases.size(), 24u);+ int refusals = 0;+ for (const Value& rec : cases.items()) {+ const std::string label = std::string(name) + "/" + rec.Find("case")->as_string();+ std::vector<sc::Question> qs;+ for (const Value& n : rec.Find("options")->items()) qs.push_back(WindowQuestion(int(n.as_int())));+ sq::Budget b{rec.Find("max_len")->as_int(), rec.Find("head_max_len")->as_int()};+ sq::WindowRequest req;+ if (const Value* w = rec.Find("window"); !w->is_null()) req.window = w->as_int();+ if (const Value* s = rec.Find("stride"); !s->is_null()) req.stride = s->as_int();+ sq::PlanError kind = sq::PlanError::None;+ auto plan = sq::PlanWindows(*tok, qs, b, req, &kind);+ if (const Value* e = rec.Find("error")) {+ refusals++;+ ASSERT_FALSE(plan.ok()) << label << ": Python refuses with " << e->as_string();+ EXPECT_EQ(plan.error(), e->as_string()) << label;+ const bool no_room = e->as_string().find("fill the whole sequence") != std::string::npos;+ EXPECT_EQ(kind, no_room ? sq::PlanError::NoRoom : sq::PlanError::StrideGap) << label;+ continue;+ }+ ASSERT_TRUE(plan.ok()) << label << ": " << plan.error();+ EXPECT_EQ(kind, sq::PlanError::None) << label;+ const Value& res = *rec.Find("result");+ EXPECT_EQ(plan.value().window, res.Find("window")->as_int()) << label;+ EXPECT_EQ(plan.value().stride, res.Find("stride")->as_int()) << label;+ EXPECT_EQ(plan.value().room, res.Find("room")->as_int()) << label;+ std::vector<std::string> warnings;+ for (const Value& w : rec.Find("warnings")->items()) warnings.push_back(w.as_string());+ EXPECT_EQ(plan.value().warnings, warnings) << label;+ EXPECT_EQ(plan.value().Clamped(), !warnings.empty()) << label;+ }+ EXPECT_GE(refusals, 1) << name;+}+INSTANTIATE_TEST_SUITE_P(Shipped, Checkpoint, ::testing::Values(0, 1));++// A budget of exactly kMinOptionBudget left for the instruction leaves the option spans whole;+// one token less cuts them.+TEST(BuildHead, OptionBudgetBoundary) {+ LAYA_CHECKPOINT_TOK(tok, "english");+ std::vector<sc::Option> opts;+ const char* texts[] = {"a", "a much longer description of the second option that has many words in it", "b c"};+ for (int i = 0; i < 3; i++) opts.push_back({sc::Label::Text("o" + std::to_string(i)), Value::String(texts[i])});+ auto q = sc::Question::Choice("Pick.", opts).value();+ auto whole = sq::BuildHead(*tok, q, 10000).value();+ const int64_t spans = int64_t(whole.ids.size()) - 1 - whole.markers[0];+ auto at = sq::BuildHead(*tok, q, spans + sq::kMinOptionBudget).value();+ EXPECT_FALSE(at.stats.tokens_per_option.has_value());+ EXPECT_EQ(at.ids.size(), whole.ids.size());+ auto below = sq::BuildHead(*tok, q, spans + sq::kMinOptionBudget - 1).value();+ EXPECT_TRUE(below.stats.tokens_per_option.has_value());+}++// A list keeps its end when it is too long; a string or object keeps its start.+TEST(EncodeState, KeepSide) {+ LAYA_CHECKPOINT_TOK(tok, "english");+ Value list = Value::Array();+ list.Push(Value::String("x"));+ Value obj = Value::Object();+ obj.Set("k", Value::String("v"));+ EXPECT_EQ(sq::EncodeState(*tok, list).value().keep, sq::Keep::End);+ EXPECT_EQ(sq::EncodeState(*tok, obj).value().keep, sq::Keep::Start);+ EXPECT_EQ(sq::EncodeState(*tok, Value::String("x")).value().keep, sq::Keep::Start);+ EXPECT_EQ(sq::EncodeState(*tok, Value::Int(5)).value().keep, sq::Keep::Start);+}++// A default window of exactly twice the room is the ratio branch of the clamp warning.+TEST(PlanWindows, ClampRatioBoundary) {+ LAYA_CHECKPOINT_TOK(tok, "english");+ bool found = false;+ for (int n = 1; n <= 40 && !found; n++) {+ auto q = WindowQuestion(n);+ for (int64_t head = 32; head <= 192 && !found; head += 8) {+ // room = max_len - len(head) - 1 and the default window = max_len - head_max_len - 8, so+ // default == 2 * room at max_len = 2 * len(head) - head_max_len - 6.+ const int64_t len = int64_t(sq::BuildHead(*tok, q, head).value().ids.size());+ sq::Budget b{2 * len - head - 6, head};+ if (b.max_len < 1 || b.DefaultWindow() <= 64) continue;+ auto room = sq::StateRoom(*tok, q, b);+ if (!room.ok() || room.value() <= 0 || b.DefaultWindow() != 2 * room.value()) continue;+ found = true;+ auto plan = sq::PlanWindows(*tok, {q}, b, {});+ ASSERT_TRUE(plan.ok());+ ASSERT_EQ(plan.value().warnings.size(), 1u);+ EXPECT_NE(plan.value().warnings[0].find("is capped"), std::string::npos) << plan.value().warnings[0];+ EXPECT_EQ(plan.value().window, room.value());+ }+ }+ EXPECT_TRUE(found) << "no budget with a default window of twice the room";+}++TEST(BatchCap, MatchesPythonGrid) {+ LAYA_GOLDEN_LIST(grid, "window_plan.json", "batch_cap");+ LAYA_EXPECT_ROWS(grid, 810);+ int capped = 0;+ for (size_t i = 0; i < grid->size(); i++) {+ const Value& rec = grid->value(i);+ const int64_t n = rec.Find("n_windows")->as_int(), w = rec.Find("window")->as_int(), cfg = rec.Find("config_budget")->as_int();+ const Value* bs = rec.Find("batch_size");+ const int64_t batch = bs->is_null() ? 0 : bs->as_int();+ const Value* want = rec.Find("cap");+ sq::Cap got = sq::BatchCap(n, w, cfg, batch);+ EXPECT_EQ(got.ok, !want->is_null()) << "row " << i;+ if (!want->is_null()) EXPECT_EQ(got.cap, want->as_int()) << "row " << i;+ if (!want->is_null() && batch <= 0) capped++;+ }+ EXPECT_GT(capped, 0);+}++TEST(BatchCap, HugeInputsDoNotOverflow) {+ const int64_t big = INT64_MAX;+ sq::Cap c = sq::BatchCap(big, big - 1, big, 0); // a one-token-narrower window: a mild cap, so none+ EXPECT_FALSE(c.ok);+ sq::Cap d = sq::BatchCap(big, 2, big, 0);+ EXPECT_TRUE(d.ok);+ EXPECT_LT(d.cap, big);+}++TEST(Budget, RefusesNonPositive) {+ LAYA_CHECKPOINT_TOK(tok, "english");+ auto q = sc::Question::Noul("Holds?").value();+ EXPECT_FALSE(sq::StateRoom(*tok, q, {0, 8}).ok());+ EXPECT_FALSE(sq::StateRoom(*tok, q, {512, -1}).ok());+ EXPECT_FALSE(sq::Build(*tok, q, {}, {0, 0}).ok());+ EXPECT_FALSE(sq::BuildHead(*tok, sc::Question(), 64).ok());+ EXPECT_EQ((sq::Budget{512, 192}).DefaultWindow(), 312);+ EXPECT_EQ((sq::Budget{100, 192}).DefaultWindow(), 64);+}++// States: json.loads then serialize_state, on 2,000 random JSON texts.+class StateRow : public ::testing::TestWithParam<int> {};+TEST_P(StateRow, SerializeMatchesPython) {+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "states", "cases");+ LAYA_EXPECT_ROWS(rows, 2000);+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < rows->size() && i < size_t(GetParam() + 1) * laya_test::kChunk; i++) {+ const Value& rec = rows->value(i);+ const std::string text = rec.Find("text")->as_string();+ auto v = laya::pyvalue::Parse(text);+ ASSERT_TRUE(v.ok()) << "state " << i << ": " << v.error();+ auto s = sq::SerializeState(v.value());+ ASSERT_TRUE(s.ok()) << "state " << i << ": " << s.error();+ EXPECT_EQ(s.value(), rec.Find("serialized")->as_string()) << "state " << i;+ EXPECT_EQ(v.value().kind() == Kind::Array, rec.Find("list")->as_bool()) << "state " << i;+ }+}+INSTANTIATE_TEST_SUITE_P(Golden, StateRow, ::testing::ValuesIn(laya_test::Chunks(2000)));++TEST(States, PythonRejects) {+ LAYA_GOLDEN_LIST(rejects, "go_extra.json", "states", "python_rejects");+ for (const Value& t : rejects->items()) {+ EXPECT_FALSE(laya::pyvalue::Parse(t.as_string()).ok()) << "accepted " << t.as_string().substr(0, 60);+ }+}++} // namespacediff --git a/laya-cpp/tests/tools/laya_c3_cli.cpp b/laya-cpp/tests/tools/laya_c3_cli.cppnew file mode 100644index 00000000..549a4d71--- /dev/null+++ b/laya-cpp/tests/tools/laya_c3_cli.cpp@@ -0,0 +1,150 @@+// A tiny line protocol over laya::schema and laya::sequence for the differential fuzz+// (laya-e2e/cpp-lane/fuzz-c3): argv[1] is the checkpoints directory; bytes travel as hex ("-" is empty).+// q <hex def> -> "err <hex>" | "ok <hex json {type, ins, opts, labels}>"+// seq <name> <max> <head> <left 0|1> <order|-> <hex def> <hex state json>+// -> "err <hex>" | "mk <kept> <total>" | "ok ids|markers|opts|distinct|tpo|st|su|sd|trunc"+// plan <name> <max> <head> <window|-> <stride|-> <n,n,...|-> -> "err <hex>" | "ok w s r <hex warning>..."+// cap <n> <window> <config> <batch|-> -> "none" | cap+#include <iostream>+#include <map>+#include <sstream>++#include "laya/pyjson.hpp"+#include "laya/pyvalue.hpp"+#include "laya/schema.hpp"+#include "laya/sequence.hpp"++namespace {+using laya::pyvalue::Value;+namespace sc = laya::schema;+namespace sq = laya::sequence;++std::string FromHex(const std::string& h) {+ std::string out;+ if (h == "-") return out;+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));+ return out;+}+std::string Hex(const std::string& s) {+ static const char* d = "0123456789abcdef";+ std::string o;+ for (unsigned char c : s) {+ o.push_back(d[c >> 4]);+ o.push_back(d[c & 15]);+ }+ return o.empty() ? "-" : o;+}+template <class T>+std::string Join(const std::vector<T>& v) {+ std::string o;+ for (size_t i = 0; i < v.size(); i++) o += (i ? "," : "") + std::to_string(v[i]);+ return o.empty() ? "-" : o;+}+std::vector<int> Ints(const std::string& s) {+ std::vector<int> out;+ if (s == "-") return out;+ std::stringstream ss(s);+ for (std::string part; std::getline(ss, part, ',');) out.push_back(std::stoi(part));+ return out;+}+int64_t Opt(const std::string& s) { return s == "-" ? 0 : std::stoll(s); }++laya::Result<sc::Question> Make(const std::string& def_json, const std::string& id = "q") {+ auto v = laya::pyvalue::Parse(def_json);+ if (!v.ok()) return laya::Result<sc::Question>::Fail("parse: " + v.error());+ auto q = sc::Question::FromWire(v.value());+ if (!q.ok()) return laya::Result<sc::Question>::Fail("question " + laya::pyjson::Repr(id) + ": " + q.error());+ return q;+}+} // namespace++int main(int argc, char** argv) {+ if (argc < 2) return 2;+ std::map<std::string, laya::tokenizer::Tokenizer> toks;+ for (const char* n : {"english", "multilingual"}) {+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(argv[1]) + "/" + n);+ if (!t.ok()) { std::cerr << t.error() << "\n"; return 2; }+ toks.emplace(n, std::move(t).value());+ }+ std::string line;+ while (std::getline(std::cin, line)) {+ std::stringstream ss(line);+ std::string kind;+ ss >> kind;+ if (kind == "q") {+ std::string h;+ ss >> h;+ auto q = Make(FromHex(h));+ if (!q.ok()) { std::cout << "err " << Hex(q.error()) << "\n"; continue; }+ Value o = Value::Object();+ o.Set("type", Value::String(std::string(sc::Name(q.value().type()))));+ o.Set("ins", Value::String(q.value().instructions()));+ Value opts = Value::Array(), labels = Value::Array();+ for (const auto& s : q.value().RenderedOptions()) opts.Push(Value::String(s));+ for (const auto& s : q.value().Labels()) labels.Push(Value::String(s));+ o.Set("opts", opts);+ o.Set("labels", labels);+ std::cout << "ok " << Hex(laya::pyvalue::Dumps(o).value()) << "\n";+ } else if (kind == "seq") {+ std::string name, order, def, state;+ int64_t mx, hd;+ int left;+ ss >> name >> mx >> hd >> left >> order >> def >> state;+ auto q = Make(FromHex(def));+ if (!q.ok()) { std::cout << "err " << Hex(q.error()) << "\n"; continue; }+ sc::Question question = q.value();+ if (order != "-") {+ auto w = question.WithOptionOrder(Ints(order));+ if (!w.ok()) { std::cout << "err " << Hex(w.error()) << "\n"; continue; }+ question = w.value();+ }+ auto sv = laya::pyvalue::Parse(FromHex(state));+ if (!sv.ok()) { std::cout << "err " << Hex("state: " + sv.error()) << "\n"; continue; }+ const auto& tok = toks.at(name);+ auto st = sq::EncodeState(tok, sv.value());+ if (!st.ok()) { std::cout << "err " << Hex(st.error()) << "\n"; continue; }+ if (left) st.value().keep = sq::Keep::End;+ sq::MarkersError me;+ auto s = sq::Build(tok, question, st.value(), {mx, hd}, &me);+ if (!s.ok()) {+ if (me.total != 0) std::cout << "mk " << me.kept << " " << me.total << "\n";+ else std::cout << "err " << Hex(s.error()) << "\n";+ continue;+ }+ const auto& r = s.value();+ std::cout << "ok " << Join(r.ids) << "|" << Join(r.markers) << "|" << r.stats.options << "|" << r.stats.options_distinct+ << "|" << (r.stats.tokens_per_option ? std::to_string(*r.stats.tokens_per_option) : "N") << "|"+ << r.truncation.state_tokens << "|" << r.truncation.state_tokens_used << "|" << r.truncation.state_tokens_dropped+ << "|" << (r.truncation.truncated ? 1 : 0) << "\n";+ } else if (kind == "plan") {+ std::string name, w, st, counts;+ int64_t mx, hd;+ ss >> name >> mx >> hd >> w >> st >> counts;+ std::vector<sc::Question> qs;+ for (int n : Ints(counts)) {+ std::vector<sc::Option> opts;+ for (int i = 0; i < n; i++) {+ std::string t;+ for (int k = 0; k < 4; k++) t += (k ? " " : "") + std::string("criterion") + std::to_string(i) + " word" + std::to_string(k);+ char l[16];+ std::snprintf(l, sizeof l, "opt%03d", i);+ opts.push_back({sc::Label::Text(l), Value::String(t)});+ }+ qs.push_back(sc::Question::Choice("Pick one.", opts).value());+ }+ auto p = sq::PlanWindows(toks.at(name), qs, {mx, hd}, {Opt(w), Opt(st)});+ if (!p.ok()) { std::cout << "err " << Hex(p.error()) << "\n"; continue; }+ std::cout << "ok " << p.value().window << " " << p.value().stride << " " << p.value().room;+ for (const auto& x : p.value().warnings) std::cout << " " << Hex(x);+ std::cout << "\n";+ } else if (kind == "cap") {+ std::string n, w, c, b;+ ss >> n >> w >> c >> b;+ auto r = sq::BatchCap(std::stoll(n), std::stoll(w), std::stoll(c), Opt(b));+ if (r.ok) std::cout << r.cap << "\n"; else std::cout << "none\n";+ } else {+ std::cout << "bad\n";+ }+ }+ return 0;+}