Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cpp): regex subset uses line anchors and refuses repeats of nullable groups; alternation target shift
Adeshina Lasisicbd4c231232c
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cppindex 9171c055..710b0ddf 100644--- a/laya-cpp/src/tokenizer.cpp+++ b/laya-cpp/src/tokenizer.cpp@@ -349,7 +349,7 @@ class Regex {at_++;// split(start of the left branches, next), the left branch jumps over the right.prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});- Shift(start + 1, 1);+ Shift(start, 1);int j = Emit(kJmp);jumps.push_back(j);prog_[size_t(start)].x = start + 1;@@ -362,7 +362,7 @@ class Regex {// Inserting an instruction at `from` moves everything after it, so targets at or past `from`// that were emitted before the insertion move by `by`.void Shift(int from, int by) {- for (size_t i = size_t(from); i < prog_.size(); i++) {+ for (size_t i = size_t(from) + 1; i < prog_.size(); i++) { // [from] is the inserted instructionInst& in = prog_[i];if (in.op == kSplit || in.op == kJmp) {if (in.x >= from) in.x += by;@@ -391,6 +391,11 @@ class Regex {} else break;bool lazy = More() && Peek() == '?';if (lazy) at_++;+ // Oniguruma (the crate's engine) ends a repeat whose body matched nothing; a backtracker+ // that remembers failed states would end it differently, so such a repeat is refused.+ if (!(lo == 0 && hi == 1) && Nullable(start, end)) {+ Unsupported("a Replace regex repeats a group that can match the empty string, whose repeat semantics differ between engines");+ }std::vector<Inst> body(prog_.begin() + start, prog_.begin() + end);for (Inst& in : body) {if (in.op == kSplit || in.op == kJmp) {@@ -435,6 +440,24 @@ class Regex {}}+ // Can the instructions [start, end) reach `end` without consuming a character (anchors pass)?+ bool Nullable(int start, int end) const {+ std::vector<char> seen(size_t(end - start) + 1, 0);+ std::vector<int> todo{start};+ while (!todo.empty()) {+ int pc = todo.back();+ todo.pop_back();+ if (pc == end) return true;+ if (pc < start || pc > end || seen[size_t(pc - start)]) continue;+ seen[size_t(pc - start)] = 1;+ const Inst& in = prog_[size_t(pc)];+ if (in.op == kJmp) todo.push_back(in.x);+ else if (in.op == kSplit) { todo.push_back(in.x); todo.push_back(in.y); }+ else if (in.op == kBol || in.op == kEol) todo.push_back(pc + 1);+ }+ return false;+ }+bool Counts(int* lo, int* hi) { // at '{'; false when it is not a repeat count (then a literal)at_++;auto number = [&](int* v) {@@ -587,8 +610,8 @@ class Regex {pc = in.x;continue;}- if (in.op == kBol) { if (pos != 0) break; pc++; continue; }- if (in.op == kEol) { if (pos != text.size()) break; pc++; continue; }+ if (in.op == kBol) { if (pos != 0 && text[pos - 1] != '\n') break; pc++; continue; }+ if (in.op == kEol) { if (pos != text.size() && text[pos] != '\n') break; pc++; continue; }if (pos >= text.size()) break;auto d = ud::DecodeRune(std::string_view(text).substr(pos));bool ok;diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cppindex 4fee3949..c542f751 100644--- a/laya-cpp/tests/tokenizer_test.cpp+++ b/laya-cpp/tests/tokenizer_test.cpp@@ -608,13 +608,19 @@ TEST(TokenizerRegex, SubsetBehaviour) {EXPECT_EQ(run("[^a-c]", "abxcy"), "ab_c_");EXPECT_EQ(run("^a", "aa"), "_a");EXPECT_EQ(run("a$", "aa"), "a_");+ EXPECT_EQ(run("^a", "a\na"), "_\n_") << "^ and $ are line anchors, as in the crate's Oniguruma";+ EXPECT_EQ(run("a$", "a\na"), "_\n_");+ EXPECT_EQ(run("(a?)?b", "ab"), "_");+ auto nullable = tk::RunNormalizerSpec(R"({"type":"Replace","pattern":{"Regex":"(a*)+b"},"content":"x"})", "b");+ ASSERT_FALSE(nullable.ok());+ EXPECT_TRUE(tk::IsUnsupported(nullable.error())) << "repeating a group that can match nothing";EXPECT_EQ(run("a.c", "abc a\\nc"), "_ a\\nc");EXPECT_EQ(run("x{2,3}", "xxxxxxx"), "__x");EXPECT_EQ(run("(?i)HeLLo", "Hello HELLO hello"), "_ _ _");EXPECT_EQ(run("é+", "ééa"), "_a");EXPECT_EQ(run("(a|ab)(c|bcd)", "abcd"), "_") << "leftmost-first: a then bcd";- // The failed-state memo keeps nested repeats linear.- EXPECT_EQ(run("(a*)*b", std::string(5000, 'a')), std::string(5000, 'a'));+ // The failed-state memo keeps repeats linear.+ EXPECT_EQ(run("(a|aa)+b", std::string(5000, 'a')), std::string(5000, 'a'));}TEST(TokenizerPre, SequenceAndCornerCases) {diff --git a/laya-cpp/tests/tools/laya_c2_cli.cpp b/laya-cpp/tests/tools/laya_c2_cli.cppindex c6ffc5b8..07bae017 100644--- a/laya-cpp/tests/tools/laya_c2_cli.cpp+++ b/laya-cpp/tests/tools/laya_c2_cli.cpp@@ -14,6 +14,7 @@namespace {std::string FromHex(const std::string& h) {std::string out;+ if (h == "-") return out; // "-" stands for the empty string, so fields never vanish from the linefor (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));return out;}