Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(cpp): tokenizer gates, braces for gcc -Wdangling-else, range check before float-to-int cast
Adeshina Lasisi90f8cfb5e920
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.ymlindex e9aac0cc..cb24e6b3 100644--- a/.github/workflows/cpp.yml+++ b/.github/workflows/cpp.yml@@ -42,7 +42,7 @@ concurrency:env:# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus# compile_flags). Raise it with every test added.- MIN_TESTS: "213"+ MIN_TESTS: "253"GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldensLAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpointsdiff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cppindex 710b0ddf..94a00c3a 100644--- a/laya-cpp/src/tokenizer.cpp+++ b/laya-cpp/src/tokenizer.cpp@@ -659,7 +659,9 @@ int Int(const Value& n, const char* key, int def) {const Value* v = Member(n, key);if (v == nullptr) return def;if (v->kind() == Value::Kind::Int && v->as_int() >= -1000000000 && v->as_int() <= 1000000000) return int(v->as_int());- if (v->kind() == Value::Kind::Float && v->as_float() == double(int(v->as_float()))) return int(v->as_float());+ if (v->kind() == Value::Kind::Float && v->as_float() >= -1e9 && v->as_float() <= 1e9 && v->as_float() == double(int(v->as_float()))) {+ return int(v->as_float());+ }Refuse(std::string("\"") + key + "\" must be an integer");}const Value& Object(const Value& n, const char* key) {diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cppindex c542f751..b496077a 100644--- a/laya-cpp/tests/tokenizer_test.cpp+++ b/laya-cpp/tests/tokenizer_test.cpp@@ -308,7 +308,9 @@ TEST_P(Shipped, DecodeMatchesPythonWholeAndSliced) {std::vector<int32_t> part(ids.begin() + long(start), ids.begin() + long(end));EXPECT_EQ(tok->Decode(part, true), s.Find("decoded")->as_string()) << "slice " << start;}- if (c.Find("slices")->size() == 0) EXPECT_TRUE(text.empty() && ids.empty());+ if (c.Find("slices")->size() == 0) {+ EXPECT_TRUE(text.empty() && ids.empty());+ }const Value& w = *c.Find("with_specials");auto with = Ints(*w.Find("ids"));EXPECT_EQ(tok->Decode(with, true), w.Find("skipping_specials")->as_string());@@ -480,6 +482,7 @@ TEST(TokenizerLoader, Refuses) {{"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},{"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},{"not JSON", "{", false},+ {"a Strip decoder start beyond int range", ByteLevelSpec(",\"decoder\":{\"type\":\"Strip\",\"content\":\" \",\"start\":1e30,\"stop\":0}"), false},{"a lossy non-byte-level tokenizer", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]}}", true},};for (const auto& c : cases) {