Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cpp): regex subset uses line anchors and refuses repeats of nullable groups; alternation target shift

Adeshina Lasisicbd4c231232c
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cpp
index 9171c055..710b0ddf 100644
--- a/laya-cpp/src/tokenizer.cpp
+++ b/laya-cpp/src/tokenizer.cpp
@@ -349,7 +349,7 @@ class Regex {
at_++;
// split(start of the left branches, next), the left branch jumps over the right.
prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});
- Shift(start + 1, 1);
+ Shift(start, 1);
int j = Emit(kJmp);
jumps.push_back(j);
prog_[size_t(start)].x = start + 1;
@@ -362,7 +362,7 @@ class Regex {
// Inserting an instruction at `from` moves everything after it, so targets at or past `from`
// that were emitted before the insertion move by `by`.
void Shift(int from, int by) {
- for (size_t i = size_t(from); i < prog_.size(); i++) {
+ for (size_t i = size_t(from) + 1; i < prog_.size(); i++) { // [from] is the inserted instruction
Inst& in = prog_[i];
if (in.op == kSplit || in.op == kJmp) {
if (in.x >= from) in.x += by;
@@ -391,6 +391,11 @@ class Regex {
} else break;
bool lazy = More() && Peek() == '?';
if (lazy) at_++;
+ // Oniguruma (the crate's engine) ends a repeat whose body matched nothing; a backtracker
+ // that remembers failed states would end it differently, so such a repeat is refused.
+ if (!(lo == 0 && hi == 1) && Nullable(start, end)) {
+ Unsupported("a Replace regex repeats a group that can match the empty string, whose repeat semantics differ between engines");
+ }
std::vector<Inst> body(prog_.begin() + start, prog_.begin() + end);
for (Inst& in : body) {
if (in.op == kSplit || in.op == kJmp) {
@@ -435,6 +440,24 @@ class Regex {
}
}
+ // Can the instructions [start, end) reach `end` without consuming a character (anchors pass)?
+ bool Nullable(int start, int end) const {
+ std::vector<char> seen(size_t(end - start) + 1, 0);
+ std::vector<int> todo{start};
+ while (!todo.empty()) {
+ int pc = todo.back();
+ todo.pop_back();
+ if (pc == end) return true;
+ if (pc < start || pc > end || seen[size_t(pc - start)]) continue;
+ seen[size_t(pc - start)] = 1;
+ const Inst& in = prog_[size_t(pc)];
+ if (in.op == kJmp) todo.push_back(in.x);
+ else if (in.op == kSplit) { todo.push_back(in.x); todo.push_back(in.y); }
+ else if (in.op == kBol || in.op == kEol) todo.push_back(pc + 1);
+ }
+ return false;
+ }
+
bool Counts(int* lo, int* hi) { // at '{'; false when it is not a repeat count (then a literal)
at_++;
auto number = [&](int* v) {
@@ -587,8 +610,8 @@ class Regex {
pc = in.x;
continue;
}
- if (in.op == kBol) { if (pos != 0) break; pc++; continue; }
- if (in.op == kEol) { if (pos != text.size()) break; pc++; continue; }
+ if (in.op == kBol) { if (pos != 0 && text[pos - 1] != '\n') break; pc++; continue; }
+ if (in.op == kEol) { if (pos != text.size() && text[pos] != '\n') break; pc++; continue; }
if (pos >= text.size()) break;
auto d = ud::DecodeRune(std::string_view(text).substr(pos));
bool ok;
diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cpp
index 4fee3949..c542f751 100644
--- a/laya-cpp/tests/tokenizer_test.cpp
+++ b/laya-cpp/tests/tokenizer_test.cpp
@@ -608,13 +608,19 @@ TEST(TokenizerRegex, SubsetBehaviour) {
EXPECT_EQ(run("[^a-c]", "abxcy"), "ab_c_");
EXPECT_EQ(run("^a", "aa"), "_a");
EXPECT_EQ(run("a$", "aa"), "a_");
+ EXPECT_EQ(run("^a", "a\na"), "_\n_") << "^ and $ are line anchors, as in the crate's Oniguruma";
+ EXPECT_EQ(run("a$", "a\na"), "_\n_");
+ EXPECT_EQ(run("(a?)?b", "ab"), "_");
+ auto nullable = tk::RunNormalizerSpec(R"({"type":"Replace","pattern":{"Regex":"(a*)+b"},"content":"x"})", "b");
+ ASSERT_FALSE(nullable.ok());
+ EXPECT_TRUE(tk::IsUnsupported(nullable.error())) << "repeating a group that can match nothing";
EXPECT_EQ(run("a.c", "abc a\\nc"), "_ a\\nc");
EXPECT_EQ(run("x{2,3}", "xxxxxxx"), "__x");
EXPECT_EQ(run("(?i)HeLLo", "Hello HELLO hello"), "_ _ _");
EXPECT_EQ(run("é+", "ééa"), "_a");
EXPECT_EQ(run("(a|ab)(c|bcd)", "abcd"), "_") << "leftmost-first: a then bcd";
- // The failed-state memo keeps nested repeats linear.
- EXPECT_EQ(run("(a*)*b", std::string(5000, 'a')), std::string(5000, 'a'));
+ // The failed-state memo keeps repeats linear.
+ EXPECT_EQ(run("(a|aa)+b", std::string(5000, 'a')), std::string(5000, 'a'));
}
TEST(TokenizerPre, SequenceAndCornerCases) {
diff --git a/laya-cpp/tests/tools/laya_c2_cli.cpp b/laya-cpp/tests/tools/laya_c2_cli.cpp
index c6ffc5b8..07bae017 100644
--- a/laya-cpp/tests/tools/laya_c2_cli.cpp
+++ b/laya-cpp/tests/tools/laya_c2_cli.cpp
@@ -14,6 +14,7 @@
namespace {
std::string FromHex(const std::string& h) {
std::string out;
+ if (h == "-") return out; // "-" stands for the empty string, so fields never vanish from the line
for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));
return out;
}