Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cpp): locale-independent numbers, indexed objects, stricter tests
Adeshina Lasisi9c94374fb1cf
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex c9daf87d..111b7b02 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -16,6 +16,11 @@ name: C++# * The flags are tested: the `compile_flags` ctest fails if -ffp-contract=off or# -fno-fast-math is missing from a laya-cpp translation unit.#+# Goldens need HF_TOKEN (the tokenizer fetch) and the Python toolchain. A pull request from a fork+# gets no secrets: the fetch then runs unauthenticated (rate-limited) and may fail, which marks this+# advisory job red without blocking the PR, as in go.yml. Each matrix leg regenerates everything+# because the job is the unit of isolation; the shared families are small, the cost is the torch install.+## The security job is NOT advisory, as in go.yml, and `build` needs it: the fetched GoogleTest# commit is verified against the upstream tag before anything is built. Be clear about what it is:# the pin check and the workflow-folding check only. There is NO static analyzer for C++ in this@@ -37,7 +42,7 @@ concurrency:env:# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus# compile_flags). Raise it with every test added.- MIN_TESTS: "206"+ MIN_TESTS: "212"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_REQUIRE_GOLDENS: "1" # a missing golden is a failure, never a skip@@ -65,6 +70,10 @@ jobs:- name: Checker testsrun: python laya-cpp/scripts/test_check_test_xml.py+ # The Locale test needs a comma-decimal locale; a missing one is a skip and the floor check fails.+ - name: Install a comma-decimal locale+ run: sudo locale-gen de_DE.UTF-8+# Goldens are recorded from the Python reference at this commit, never read from a committed copy.- name: Install the Python toolchain the goldens are recorded withrun: |diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex 94d1f233..18d66676 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -44,7 +44,7 @@ if(LAYA_BUILD_TESTS)FetchContent_MakeAvailable(googletest)enable_testing()- add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp)+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp)target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)laya_target_flags(laya_tests)diff --git a/laya-cpp/include/laya/pyvalue.hpp b/laya-cpp/include/laya/pyvalue.hppindex adf49c8e..7d1a2098 100644--- a/laya-cpp/include/laya/pyvalue.hpp+++ b/laya-cpp/include/laya/pyvalue.hpp@@ -5,6 +5,7 @@#include <cstdint>#include <string>#include <string_view>+#include <unordered_map>#include <vector>#include "laya/result.hpp"@@ -53,10 +54,15 @@ class Value {double d_ = 0;std::string s_;std::vector<std::string> keys_;+ std::unordered_map<std::string, size_t> index_; // key -> position, so Set and Find are O(1)std::vector<Value> vals_;};-// json.loads of one document of UTF-8 text. Refused: invalid UTF-8, a lone surrogate escape, a raw+// json.loads of one document of UTF-8 text, as json.loads(str) reads it: a leading byte order mark+// is refused (json.loads(bytes) would skip it; strip it first to get that behavior). The nesting+// bound kMaxDepth is laya-go's, not Python's: CPython's RecursionError limit depends on the+// build and on what is nested (a wrapped value costs more than an empty array), so the two agree+// on ordinary documents and may differ right around the bound. Refused: invalid UTF-8, a lone surrogate escape, a raw// control character in a string, an int literal over 4300 digits, nesting beyond kMaxDepth.Result<Value> Parse(std::string_view text);// json.dumps(v, ensure_ascii=False) with the default ", " and ": " separators. Fails on a stringdiff --git a/laya-cpp/include/laya/sha256.hpp b/laya-cpp/include/laya/sha256.hppindex 46ab13da..8c85c138 100644--- a/laya-cpp/include/laya/sha256.hpp+++ b/laya-cpp/include/laya/sha256.hpp@@ -11,9 +11,10 @@ namespace laya {class Sha256 {public:Sha256();- void Update(const void* data, size_t size);- void Update(std::string_view s) { Update(s.data(), s.size()); }- // Finishes the hash; the object must not be updated afterwards.+ // Returns false, and changes nothing, once Finish has been called.+ bool Update(const void* data, size_t size);+ bool Update(std::string_view s) { return Update(s.data(), s.size()); }+ // Finishes the hash. Calling it again returns the same digest.std::array<uint8_t, 32> Finish();private:@@ -22,6 +23,8 @@ class Sha256 {uint8_t buf_[64];size_t used_ = 0;uint64_t total_ = 0;+ bool finished_ = false;+ std::array<uint8_t, 32> digest_{};};std::array<uint8_t, 32> Sha256Of(std::string_view data);diff --git a/laya-cpp/src/cnum.hpp b/laya-cpp/src/cnum.hppnew file mode 100644index 00000000..58826d6c--- /dev/null+++ b/laya-cpp/src/cnum.hpp@@ -0,0 +1,39 @@+#pragma once+// Locale-independent number text. strtod and snprintf follow the process locale (a comma decimal+// point under de_DE), which silently truncates or corrupts what Python writes with a dot; these+// helpers translate the decimal point and never touch the global locale.+#include <clocale>+#include <cstdio>+#include <cstdlib>+#include <string>++namespace laya {+namespace cnum {++inline std::string DecimalPoint() {+ const lconv* lc = std::localeconv();+ return (lc && lc->decimal_point && *lc->decimal_point) ? lc->decimal_point : ".";+}+// strtod over text using '.', whatever the locale. `point` is the locale's decimal point.+inline double ToDouble(const std::string& text, const std::string& point) {+ std::string t;+ for (char c : text) {+ if (c == '.') t += point;+ else t.push_back(c);+ }+ return std::strtod(t.c_str(), nullptr);+}+inline double ToDouble(const std::string& text) { return ToDouble(text, DecimalPoint()); }+// "%.<digits>f" with a '.' decimal point. `point` is the locale's decimal point.+inline std::string Fixed(double v, int digits, const std::string& point) {+ char buf[400];+ std::snprintf(buf, sizeof buf, "%.*f", digits, v);+ std::string s = buf;+ size_t p = s.find(point);+ if (point != "." && p != std::string::npos) s.replace(p, point.size(), ".");+ return s;+}+inline std::string Fixed(double v, int digits) { return Fixed(v, digits, DecimalPoint()); }++} // namespace cnum+} // namespace layadiff --git a/laya-cpp/src/pyjson.cpp b/laya-cpp/src/pyjson.cppindex 1b26a369..f0802fdc 100644--- a/laya-cpp/src/pyjson.cpp+++ b/laya-cpp/src/pyjson.cpp@@ -5,6 +5,7 @@#include <cstdio>#include <cstdlib>+#include "cnum.hpp"#include "laya/unicodedata.hpp"namespace laya {@@ -97,16 +98,12 @@ std::string Percent0(double fraction) {double scaled = 100 * fraction;if (std::isnan(scaled)) return "nan";if (std::isinf(scaled)) return scaled > 0 ? "inf" : "-inf";- char buf[400];- std::snprintf(buf, sizeof buf, "%.0f", scaled);- return buf;+ return cnum::Fixed(scaled, 0);}double Round4(double v) {if (std::isnan(v) || std::isinf(v)) return v;- char buf[400];- std::snprintf(buf, sizeof buf, "%.4f", v);- return std::strtod(buf, nullptr);+ return cnum::ToDouble(cnum::Fixed(v, 4));}} // namespace pyjsondiff --git a/laya-cpp/src/pyvalue.cpp b/laya-cpp/src/pyvalue.cppindex b568fb19..18ee9174 100644--- a/laya-cpp/src/pyvalue.cpp+++ b/laya-cpp/src/pyvalue.cpp@@ -6,6 +6,7 @@#include <cstdlib>#include <unordered_map>+#include "cnum.hpp"#include "laya/pyjson.hpp"#include "laya/unicodedata.hpp"@@ -55,19 +56,17 @@ Value Value::Object() {}const Value* Value::Find(std::string_view k) const {- for (size_t i = 0; i < keys_.size(); i++) {- if (keys_[i] == k) return &vals_[i];- }- return nullptr;+ auto it = index_.find(std::string(k));+ return it == index_.end() ? nullptr : &vals_[it->second];}void Value::Set(std::string k, Value v) {- for (size_t i = 0; i < keys_.size(); i++) {- if (keys_[i] == k) {- vals_[i] = std::move(v);- return;- }+ auto it = index_.find(k);+ if (it != index_.end()) {+ vals_[it->second] = std::move(v);+ return;}+ index_.emplace(k, keys_.size());keys_.push_back(std::move(k));vals_.push_back(std::move(v));}@@ -163,7 +162,6 @@ class Parser {Result<Value> Obj(int depth) {Value obj = Value::Object();- std::unordered_map<std::string, size_t> at;Space();if (i_ < s_.size() && s_[i_] == '}') {i_++;@@ -301,7 +299,7 @@ class Parser {return Result<Value>::Ok(Value::BigInt(text));}// Out of range is +-Inf, as float("1e400") is in Python.- return Result<Value>::Ok(Value::Float(std::strtod(text.c_str(), nullptr)));+ return Result<Value>::Ok(Value::Float(cnum::ToDouble(text)));}public:diff --git a/laya-cpp/src/sha256.cpp b/laya-cpp/src/sha256.cppindex 97089cd8..88305f33 100644--- a/laya-cpp/src/sha256.cpp+++ b/laya-cpp/src/sha256.cpp@@ -57,7 +57,8 @@ void Sha256::Block(const uint8_t* p) {h_[7] += h;}-void Sha256::Update(const void* data, size_t size) {+bool Sha256::Update(const void* data, size_t size) {+ if (finished_) return false;const uint8_t* p = static_cast<const uint8_t*>(data);total_ += size;while (size > 0) {@@ -72,9 +73,11 @@ void Sha256::Update(const void* data, size_t size) {used_ = 0;}}+ return true;}std::array<uint8_t, 32> Sha256::Finish() {+ if (finished_) return digest_;const uint64_t bits = total_ * 8;uint8_t pad[72] = {0x80};// Pad with 0x80 then zeros to 56 mod 64, then the 64-bit big-endian bit length.@@ -83,7 +86,8 @@ std::array<uint8_t, 32> Sha256::Finish() {for (int i = 0; i < 8; i++) len[i] = uint8_t(bits >> (56 - 8 * i));Update(pad, padlen);Update(len, 8);- std::array<uint8_t, 32> out{};+ finished_ = true;+ std::array<uint8_t, 32>& out = digest_;for (int i = 0; i < 8; i++) {out[4 * i] = uint8_t(h_[i] >> 24);out[4 * i + 1] = uint8_t(h_[i] >> 16);diff --git a/laya-cpp/tests/golden.hpp b/laya-cpp/tests/golden.hppindex c0226d12..d18b5732 100644--- a/laya-cpp/tests/golden.hpp+++ b/laya-cpp/tests/golden.hpp@@ -54,12 +54,19 @@ inline const Value* At(const Value* v, std::initializer_list<const char*> path)return v;}-// A test body's view of one golden list: fails (or skips locally) when it cannot be had.-#define LAYA_GOLDEN_LIST(var, file, ...) \- const laya::pyvalue::Value* var = laya_test::At(laya_test::Load(file, true), {__VA_ARGS__}); \- if (var == nullptr) { \- if (!HasFailure()) GTEST_SKIP() << "LAYA_GOLDENS unset (local run)"; \- return; \+inline bool RequireGoldens() {+ const char* r = std::getenv("LAYA_REQUIRE_GOLDENS");+ return r != nullptr && std::string(r) == "1";+}++// A test body's view of one golden list. Absent goldens skip locally; with LAYA_REQUIRE_GOLDENS=1 a+// missing directory, file or key path FAILS, so a renamed key cannot turn a parity test into a skip.+#define LAYA_GOLDEN_LIST(var, file, ...) \+ const laya::pyvalue::Value* var = laya_test::At(laya_test::Load(file, true), {__VA_ARGS__}); \+ if (var == nullptr) { \+ if (laya_test::RequireGoldens()) GTEST_FAIL() << "golden key path missing in " << file; \+ if (!::testing::Test::HasFailure()) GTEST_SKIP() << "LAYA_GOLDENS unset (local run)"; \+ return; \}constexpr size_t kChunk = 100; // rows per test case in the large familiesdiff --git a/laya-cpp/tests/hardening_test.cpp b/laya-cpp/tests/hardening_test.cppnew file mode 100644index 00000000..351e65f5--- /dev/null+++ b/laya-cpp/tests/hardening_test.cpp@@ -0,0 +1,146 @@+#include <gtest/gtest-spi.h>++#include <chrono>+#include <clocale>+#include <cstdlib>++#include "golden.hpp"+#include "laya/pyjson.hpp"+#include "laya/pyvalue.hpp"+#include "laya/sha256.hpp"+#include "laya/unicodedata.hpp"++namespace ud = laya::unicodedata;+namespace pv = laya::pyvalue;++// ---- all-code-point digests recorded from CPython (fuzz-c1/gen_unicode_digests.py) -------------+namespace {+struct DigestRow { const char* name; const char* sha; };+const DigestRow kDigests[] = {+#include "unicode_digests.inc"+};+const char* Want(const std::string& name) {+ for (const auto& d : kDigests) {+ if (name == d.name) return d.sha;+ }+ return "";+}+template <class F>+std::string Digest(F fn) {+ laya::Sha256 h;+ for (char32_t cp = 0; cp < 0x110000; cp++) {+ if (cp >= 0xD800 && cp <= 0xDFFF) continue;+ h.Update(fn(cp));+ }+ return laya::ToHex(h.Finish());+}+std::string One(char32_t cp) {+ std::string s;+ ud::AppendRune(s, cp);+ return s;+}+} // namespace++TEST(UnicodeDigests, PrintableTitleSigmaFieldsDecimal) {+ EXPECT_EQ(Digest([](char32_t c) { return std::string(ud::IsPrintable(c) ? "1" : "0"); }), Want("printable"));+ // IsCased (upper, lower or titlecase) decides whether "A"+ch is all uppercase.+ EXPECT_EQ(Digest([](char32_t c) { return std::string(ud::IsUpperString("A" + One(c)) ? "1" : "0"); }), Want("title_cased"));+ // The final-sigma context sets (cased, case-ignorable), probed exactly as gen_tables.py does.+ EXPECT_EQ(Digest([](char32_t c) {+ std::string l = ud::Lower("A" + One(c) + "\xCE\xA3");+ return std::string(l.compare(l.size() - 2, 2, "\xCF\x82") == 0 ? "1" : "0");+ }), Want("sigma_final"));+ EXPECT_EQ(Digest([](char32_t c) {+ std::string l = ud::Lower("A\xCE\xA3" + One(c));+ return std::string(l.compare(1, 2, "\xCF\x82") == 0 ? "1" : "0");+ }), Want("sigma_next"));+ EXPECT_EQ(Digest([](char32_t c) { return std::to_string(ud::Fields("a" + One(c) + "b").size()) + ";"; }), Want("fields"));+ EXPECT_EQ(Digest([](char32_t c) {+ std::string r = ud::DecimalASCII(One(c));+ return ud::IsDigit(c) ? r : std::string("-"); // an Nd digit becomes its ASCII digit+ }), Want("decimal"));+}++TEST(UnicodeDigests, PythonBoundaryRows) {+ EXPECT_EQ(ud::Fields("\n\xF0\x9F\x98\x80\x1c").size(), 1u); // "\n😀\x1c".split(): U+001C is whitespace+ EXPECT_EQ(ud::Fields("a\xE2\x80\xA8" "b\xC2\x85" "c").size(), 3u);+ EXPECT_FALSE(ud::IsUpperString("A\xC7\x85")); // "ADž".isupper() is False (titlecase)+ EXPECT_EQ(ud::DecimalASCII("\xF0\x9D\x9F\x8E\xF0\x9D\x9F\xBF"), "09"); // U+1D7CE..U+1D7FF cycle 0-9+ EXPECT_EQ(ud::DecimalASCII("\xF0\x9D\x9F\x98\xF0\x9D\x9F\x9A"), "02"); // U+1D7D8 starts the next cycle+}++// ---- locale independence -------------------------------------------------------------------+TEST(Locale, NumbersIgnoreACommaDecimalLocale) {+ const char* candidates[] = {"de_DE.UTF-8", "de_DE.utf8", "fr_FR.UTF-8", "fr_FR.utf8", "de_DE", "de_DE.ISO8859-1"};+ std::string saved = std::setlocale(LC_ALL, nullptr);+ bool set = false;+ for (const char* c : candidates) {+ if (std::setlocale(LC_ALL, c) != nullptr && *std::localeconv()->decimal_point == ',') {+ set = true;+ break;+ }+ }+ if (!set) {+ std::setlocale(LC_ALL, saved.c_str());+ GTEST_SKIP() << "no comma-decimal locale installed";+ }+ auto v = pv::Parse("[1.5, 0.30000000000000004, 1e-7, 123456.789]");+ std::string dumped = v.ok() ? pv::Dumps(v.value()).value() : "";+ std::string r4 = laya::pyjson::ReprFloat(laya::pyjson::Round4(0.12345));+ std::string pct = laya::pyjson::Percent0(0.125);+ double r4b = laya::pyjson::Round4(2.00015);+ std::setlocale(LC_ALL, saved.c_str());+ ASSERT_TRUE(v.ok());+ EXPECT_EQ(dumped, "[1.5, 0.30000000000000004, 1e-07, 123456.789]");+ EXPECT_EQ(r4, "0.1235");+ EXPECT_EQ(pct, "12"); // 12.5 rounds half to even+ EXPECT_EQ(r4b, 2.0002);+}++// ---- sha256 after Finish -------------------------------------------------------------------+TEST(Sha256, FinishIsIdempotentAndUpdateAfterFinishIsRefused) {+ laya::Sha256 h;+ EXPECT_TRUE(h.Update("abc"));+ auto first = h.Finish();+ EXPECT_EQ(laya::ToHex(first), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");+ EXPECT_EQ(h.Finish(), first); // not a digest of the padding+ EXPECT_FALSE(h.Update("d"));+ EXPECT_EQ(h.Finish(), first);+}++// ---- Value::Set / Find scale ---------------------------------------------------------------+TEST(PyValue, ObjectsWithManyKeysParseInLinearTime) {+ const int n = 200000;+ std::string text = "{";+ for (int i = 0; i < n; i++) text += (i ? ", \"k" : "\"k") + std::to_string(i) + "\": " + std::to_string(i);+ for (int i = 0; i < n; i++) text += ", \"k" + std::to_string(i % 1000) + "\": 0"; // repeated keys+ text += "}";+ auto t0 = std::chrono::steady_clock::now();+ auto v = pv::Parse(text);+ double secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();+ ASSERT_TRUE(v.ok());+ EXPECT_EQ(v.value().size(), size_t(n));+ ASSERT_NE(v.value().Find("k199999"), nullptr);+ EXPECT_EQ(v.value().Find("k5")->as_int(), 0); // last value wins+ EXPECT_EQ(v.value().key(5), "k5"); // first position kept+ EXPECT_LT(secs, 10.0) << "quadratic key handling"; // linear takes well under a second+}++// ---- a missing golden key fails under LAYA_REQUIRE_GOLDENS=1 -----------------------------------+namespace {+void UsesMissingKey() { LAYA_GOLDEN_LIST(x, "lang_detect.json", "no_such_key"); (void)x; }+} // namespace++TEST(Golden, MissingKeyFailsWhenRequired) {+ const char* dir = std::getenv("LAYA_GOLDENS");+ if (dir == nullptr || !*dir) {+ if (laya_test::RequireGoldens()) FAIL() << "LAYA_GOLDENS unset";+ GTEST_SKIP() << "LAYA_GOLDENS unset (local run)";+ }+ const char* old = std::getenv("LAYA_REQUIRE_GOLDENS");+ std::string saved = old ? old : "";+ setenv("LAYA_REQUIRE_GOLDENS", "1", 1);+ EXPECT_FATAL_FAILURE(UsesMissingKey(), "golden key path missing");+ if (old) setenv("LAYA_REQUIRE_GOLDENS", saved.c_str(), 1);+ else unsetenv("LAYA_REQUIRE_GOLDENS");+}diff --git a/laya-cpp/tests/tools/laya_cli.cpp b/laya-cpp/tests/tools/laya_cli.cppindex 8b5160c9..7bc85872 100644--- a/laya-cpp/tests/tools/laya_cli.cpp+++ b/laya-cpp/tests/tools/laya_cli.cpp@@ -3,6 +3,8 @@// json -> "ok <hex of json.dumps(json.loads(x))>" or "err"// lower -> hex of str.lower() sha -> hex digest// lang -> "<script>|<language>|<english>|<diacritic>|<non_latin>|<hex mixed_segment>"+#include <cstdlib>+#include <cstring>#include <iostream>#include <sstream>@@ -47,6 +49,25 @@ int main() {laya::lang::Analysis a = laya::lang::Analyze(std::string_view(data));std::cout << a.script << "|" << a.language << "|" << (a.english ? 1 : 0) << "|" << laya::pyjson::ReprFloat(a.diacritic_rate)<< "|" << laya::pyjson::ReprFloat(a.non_latin_fraction) << "|" << Hex(a.mixed_segment) << "\n";+ } else if (kind == "strip") {+ std::cout << Hex(laya::unicodedata::Strip(data)) << "\n";+ } else if (kind == "stripn") {+ std::cout << Hex(laya::unicodedata::StripNumeric(data)) << "\n";+ } else if (kind == "fields") {+ for (const auto& f : laya::unicodedata::Fields(data)) std::cout << Hex(f) << ",";+ std::cout << "\n";+ } else if (kind == "dec") {+ std::cout << Hex(laya::unicodedata::DecimalASCII(data)) << "\n";+ } else if (kind == "isup") {+ std::cout << (laya::unicodedata::IsUpperString(data) ? "1" : "0") << "\n";+ } else if (kind == "repr") {+ std::cout << Hex(laya::pyjson::Repr(data)) << "\n";+ } else if (kind == "float") { // the payload here is decimal bits, not hex+ uint64_t bits = std::strtoull(line.substr(sp + 1).c_str(), nullptr, 10);+ double v;+ std::memcpy(&v, &bits, 8);+ std::cout << laya::pyjson::ReprFloat(v) << "|" << laya::pyjson::ReprFloat(laya::pyjson::Round4(v)) << "|"+ << laya::pyjson::Percent0(v) << "\n";} else {std::cout << "?\n";}diff --git a/laya-cpp/tests/unicode_digests.inc b/laya-cpp/tests/unicode_digests.incnew file mode 100644index 00000000..1ffc128f--- /dev/null+++ b/laya-cpp/tests/unicode_digests.inc@@ -0,0 +1,6 @@+{"printable", "d1979fbf099b8f9cb34aeb666db7f712f0b649da5519df52432b276334e5f17a"},+{"title_cased", "92afae3e4e67633491ade83a52608923e599551a238e8ad7a3554824e0645856"},+{"sigma_final", "c7f019085f57781c89efbcb751eac5572917fc842816eb7e238dc3fe74f8badc"},+{"sigma_next", "bde63418368aae461c52b156a54e23362fe57430745d3705e27f9fc4ac510a06"},+{"fields", "2ea7a5be2dd4c93c3c2e8ae0076a40213613bbafe158ba824177c29bff8b4bc6"},+{"decimal", "14d7a76a25031eeafa2dc0e217fbe75d5fb2d0a80cd292a9520d816740464136"},