Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(cpp): tokenizer gates, braces for gcc -Wdangling-else, range check before float-to-int cast

Adeshina Lasisi90f8cfb5e920
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index e9aac0cc..cb24e6b3 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -42,7 +42,7 @@ concurrency:
env:
# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus
# compile_flags). Raise it with every test added.
- MIN_TESTS: "213"
+ MIN_TESTS: "253"
GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
LAYA_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints # the tokenizer tests need both shipped checkpoints
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cpp
index 710b0ddf..94a00c3a 100644
--- a/laya-cpp/src/tokenizer.cpp
+++ b/laya-cpp/src/tokenizer.cpp
@@ -659,7 +659,9 @@ int Int(const Value& n, const char* key, int def) {
const Value* v = Member(n, key);
if (v == nullptr) return def;
if (v->kind() == Value::Kind::Int && v->as_int() >= -1000000000 && v->as_int() <= 1000000000) return int(v->as_int());
- if (v->kind() == Value::Kind::Float && v->as_float() == double(int(v->as_float()))) return int(v->as_float());
+ if (v->kind() == Value::Kind::Float && v->as_float() >= -1e9 && v->as_float() <= 1e9 && v->as_float() == double(int(v->as_float()))) {
+ return int(v->as_float());
+ }
Refuse(std::string("\"") + key + "\" must be an integer");
}
const Value& Object(const Value& n, const char* key) {
diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cpp
index c542f751..b496077a 100644
--- a/laya-cpp/tests/tokenizer_test.cpp
+++ b/laya-cpp/tests/tokenizer_test.cpp
@@ -308,7 +308,9 @@ TEST_P(Shipped, DecodeMatchesPythonWholeAndSliced) {
std::vector<int32_t> part(ids.begin() + long(start), ids.begin() + long(end));
EXPECT_EQ(tok->Decode(part, true), s.Find("decoded")->as_string()) << "slice " << start;
}
- if (c.Find("slices")->size() == 0) EXPECT_TRUE(text.empty() && ids.empty());
+ if (c.Find("slices")->size() == 0) {
+ EXPECT_TRUE(text.empty() && ids.empty());
+ }
const Value& w = *c.Find("with_specials");
auto with = Ints(*w.Find("ids"));
EXPECT_EQ(tok->Decode(with, true), w.Find("skipping_specials")->as_string());
@@ -480,6 +482,7 @@ TEST(TokenizerLoader, Refuses) {
{"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},
{"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},
{"not JSON", "{", false},
+ {"a Strip decoder start beyond int range", ByteLevelSpec(",\"decoder\":{\"type\":\"Strip\",\"content\":\" \",\"start\":1e30,\"stop\":0}"), false},
{"a lossy non-byte-level tokenizer", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]}}", true},
};
for (const auto& c : cases) {