Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): identifier stripping is quadratic; 31 s -> 1.8 ms at 50k chars

Adeshina Lasisi9d4a4e9223ae
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ce48ff4f..0ffdce6a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,6 +96,7 @@ jobs:
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
python tests/test_lang_guess.py
+ python tests/test_identifier_complexity.py
python tests/test_calibration_persistence.py
python tests/test_context_manager.py
python tests/test_criteria_normalization.py
@@ -159,6 +160,7 @@ jobs:
run: |
for t in test_router test_criteria test_download test_shortlist \
test_decision_model test_packaging test_lang_guess \
+ test_identifier_complexity \
test_calibration_persistence test_context_manager \
test_criteria_normalization test_docker_entrypoint \
test_empty_questions test_router_memory test_shortlist_cosine \
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 03d38ea9..fcb503e3 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -118,7 +118,11 @@ const SHARED_WORDS: Set<string> = (() => {
})();
const WORD_RE = /[^\W\d_]+/gu;
-const IDENTIFIER_RE = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
+// Lookbehind for the same reason as the Python side (see laya/lang.py): without it the
+// greedy prefix is retried at every offset inside a run of word characters, which is
+// quadratic in the run's length -- 50 000 characters of one token took 1540 ms here.
+// It removes no match, because a leftmost match can only begin at a run start.
+const IDENTIFIER_RE = /(?<![\p{L}\p{N}_-])[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
const IS_ALPHA_RE = /\p{L}/u;
// Python uses two different boundaries on purpose: detect_script/script_profile count the IPA
diff --git a/laya-ts/tests/identifier-complexity.test.ts b/laya-ts/tests/identifier-complexity.test.ts
new file mode 100644
index 00000000..5c54d62f
--- /dev/null
+++ b/laya-ts/tests/identifier-complexity.test.ts
@@ -0,0 +1,108 @@
+// laya-ts/tests/identifier-complexity.test.ts
+//
+// Regression: stripping identifiers must stay linear in the length of a word-character run.
+//
+// IDENTIFIER_RE removes dot- and @-joined tokens before words are counted. Without the
+// leading lookbehind the greedy prefix is retried at every offset inside a run of word
+// characters, and each attempt rescans the run before failing on the absent [.@] --
+// quadratic in the run's length. Measured on node 20: 50 000 characters of one token took
+// 1540 ms, against 0.05 ms for ordinary prose of the same size. `analyse` runs on the
+// default routing path once per request, so the cost is reachable by any caller.
+//
+// The lookbehind removes no match: the word class and [.@] are disjoint, so an attempt
+// from inside a run consumes to exactly the same separator as one from the run's start.
+// Parity below is asserted against the previous pattern rather than typed expectations.
+import { describe, expect, it } from "vitest";
+import { analyse, latinProfile } from "../src/lang.js";
+
+const PREVIOUS = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
+const CURRENT = /(?<![\p{L}\p{N}_-])[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
+
+const strip = (re: RegExp, s: string) => s.replace(new RegExp(re.source, re.flags), " ");
+const elapsed = (fn: () => void) => {
+ const started = performance.now();
+ fn();
+ return performance.now() - started;
+};
+
+describe("identifier stripping: parity with the previous pattern", () => {
+ const cases = [
+ "github.com", "[email protected]", "v1.2.3", "U.S.A.", "arrivato.", "a.b", "a@b",
+ "sub.domain.co.uk", "[email protected]", "192.168.0.1", "-a.b-", "_x.y_",
+ "foo..bar", ".com", "a.", "@a", "a@", "e.g.", "Ü.Ö", "naïve.café", "a-b.c-d",
+ "x-a.b", "9.9", "a..b", "a.-b", "a-.b-.c", "", ".", "@",
+ "Contact [email protected] or github.com/acme for v2.10.1 details.",
+ "No identifiers here at all just words",
+ // A local part at RFC 5321's 64-character limit, and a run past any plausible bound:
+ // a length-bounded fix strips only part of these and leaves the rest as a word.
+ "a".repeat(64) + "@example.com",
+ "a".repeat(200) + ".example.com",
+ "grazie mille per " + "wzqxk".repeat(15) + ".example.com",
+ ];
+ for (const token of cases) {
+ it(`strips ${JSON.stringify(token.slice(0, 34))} identically`, () => {
+ expect(strip(CURRENT, token)).toBe(strip(PREVIOUS, token));
+ });
+ }
+
+ it("agrees with the previous pattern on 20 000 random strings", () => {
+ let seed = 12345;
+ const rnd = () => ((seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff);
+ const alphabets = ["aA1._@- ", "._@-", "áéÜß._@-", "abcXYZ019._@- -_"];
+ let mismatch: string | null = null;
+ for (let i = 0; i < 20_000 && mismatch === null; i++) {
+ const alphabet = alphabets[i % alphabets.length]!;
+ const len = Math.floor(rnd() * 60);
+ let s = "";
+ for (let j = 0; j < len; j++) s += alphabet[Math.floor(rnd() * alphabet.length)];
+ if (strip(CURRENT, s) !== strip(PREVIOUS, s)) mismatch = s;
+ }
+ expect(mismatch).toBeNull();
+ });
+
+ it("leaves detection of ordinary prose unchanged", () => {
+ expect(analyse("The customer was billed twice and wants a refund").isEnglish).toBe(true);
+ });
+});
+
+describe("identifier stripping: complexity", () => {
+ // These go through latinProfile / analyse -- the exported functions that use the
+ // shipped IDENTIFIER_RE -- so reverting src/lang.ts fails them. Testing the pattern
+ // literal above would only ever test the literal.
+ //
+ // Ceilings are set to fail the previous pattern, not merely to be generous. Previous
+ // cost / ceiling / current cost on node 20, via latinProfile:
+ // 20 000 chars 250 ms / 60 ms / 0.1 ms
+ // 50 000 chars 1 540 ms / 200 ms / 0.3 ms
+ it("latinProfile on 20 000 characters of one token is bounded", () => {
+ expect(elapsed(() => latinProfile("a".repeat(20_000)))).toBeLessThan(60);
+ });
+
+ it("latinProfile on 50 000 characters of one token is bounded", () => {
+ expect(elapsed(() => latinProfile("a".repeat(50_000)))).toBeLessThan(200);
+ });
+
+ it("a single trailing separator does not reopen the quadratic path", () => {
+ expect(elapsed(() => latinProfile("a".repeat(50_000) + "."))).toBeLessThan(200);
+ });
+
+ it("hyphen and underscore runs are linear too", () => {
+ expect(elapsed(() => latinProfile("a-".repeat(25_000)))).toBeLessThan(200);
+ expect(elapsed(() => latinProfile("a_".repeat(25_000)))).toBeLessThan(200);
+ });
+
+ // No assertion at the 4000 characters `stateText` caps detection to: `analyse` does
+ // enough non-regex work at that size that quadratic and linear are only a few-fold
+ // apart, which CI jitter covers either way. The 20 000 and 50 000 character cases
+ // above carry the guarantee -- they are 250x and 1500x apart.
+
+ it("4x the input costs under 8x the time (linear ~4, quadratic ~16)", () => {
+ const best = (n: number) =>
+ Math.min(...[0, 1, 2].map(() => elapsed(() => latinProfile("a".repeat(n)))));
+ const small = best(12_500);
+ const large = best(50_000);
+ // performance.now() granularity makes a sub-0.05 ms baseline meaningless; the
+ // absolute ceilings above already carry the guarantee in that case.
+ if (small > 0.05) expect(large / small).toBeLessThan(8);
+ });
+});
diff --git a/laya/lang.py b/laya/lang.py
index 0122c787..dde82c45 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -180,7 +180,14 @@ _WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
# state that was mostly links scored a language it does not contain, and two domains were enough to
# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs
# word characters on both sides of the dot.
-_IDENTIFIER = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
+# The lookbehind is what keeps this linear. Without it the greedy `[\w-]*` is retried
+# at every offset inside a run of word characters, and each attempt rescans the run
+# before failing on the absent `[.@]` -- quadratic in the run's length, so 4000
+# characters of one token cost 205 ms against 0.45 ms for ordinary prose. It removes
+# no match: `[\w-]` and `[.@]` are disjoint, so an attempt from inside a run consumes
+# to exactly the same separator as an attempt from the run's start and the two always
+# succeed or fail together -- a leftmost match can only ever begin at a run start.
+_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:
diff --git a/tests/test_identifier_complexity.py b/tests/test_identifier_complexity.py
new file mode 100644
index 00000000..d333a558
--- /dev/null
+++ b/tests/test_identifier_complexity.py
@@ -0,0 +1,150 @@
+"""Regression: stripping identifiers must stay linear in the length of a word-character run.
+
+`_IDENTIFIER` removes dot- and @-joined tokens (`github.com`, `[email protected]`, `v1.2.3`)
+before `_WORD` counts function words. Without the leading lookbehind, the greedy `[\\w-]*`
+is retried at every offset inside a run of word characters and each attempt rescans the
+run before failing on the absent `[.@]` -- quadratic in the run's length:
+
+ 4 000 characters of one token 205 ms (0.45 ms for prose of the same size)
+ 50 000 characters of one token 30 678 ms
+
+`analyse()` runs on the default routing path -- no `model`, no `lang` -- once per request,
+and `state_text` hands it up to 4000 characters, so an unauthenticated caller of
+`POST /v1/systemone` could buy ~115x the CPU of a normal request. `latin_profile` and
+`guess_latin_language` take unbounded text, where the same input costs ~31 s.
+
+The lookbehind removes no match: `[\\w-]` and `[.@]` are disjoint, so an attempt from
+inside a run consumes to exactly the same separator as an attempt from the run's start,
+and the two always succeed or fail together. The parity block below is the evidence --
+it is checked against the previous pattern, not against hand-written expectations.
+
+Run: python tests/test_identifier_complexity.py
+"""
+import os
+import re
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.lang import _IDENTIFIER, analyse, latin_profile # noqa: E402
+
+# The pattern as it was before the lookbehind, kept here so parity is asserted against
+# the real previous behaviour rather than against expectations someone typed out.
+_PREVIOUS = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def ok(name, cond, extra=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s%s" % (name, (" -> " + str(extra)) if extra else ""))
+
+
+# ---------------------------------------------------------------- parity ----
+# Every identifier the pattern exists to strip, plus the shapes that tempt a
+# bounded-quantifier fix into leaving a residue token behind.
+PARITY = [
+ "github.com", "[email protected]", "v1.2.3", "U.S.A.", "arrivato.", "a.b", "a@b",
+ "sub.domain.co.uk", "[email protected]", "192.168.0.1", "-a.b-", "_x.y_",
+ "foo..bar", ".com", "a.", "@a", "a@", "e.g.", "Ü.Ö", "naïve.café", "a-b.c-d",
+ "x-a.b", "9.9", "a..b", "a.-b", "a-.b-.c", "", ".", "@",
+ "Contact [email protected] or github.com/acme for v2.10.1 details.",
+ "No identifiers here at all just words",
+ # A local part at RFC 5321's 64-character limit, and a run past any plausible bound:
+ # a length-bounded pattern strips only part of these and leaves the rest as a word.
+ "a" * 64 + "@example.com",
+ "a" * 200 + ".example.com",
+ "grazie mille per " + "wzqxk" * 15 + ".example.com",
+]
+for token in PARITY:
+ check("parity/%r" % (token[:34],), _IDENTIFIER.sub(" ", token), _PREVIOUS.sub(" ", token))
+
+# Randomised differential test: the two patterns must agree on everything.
+import random # noqa: E402
+
+random.seed(11)
+_ALPHABETS = ["aA1._@- ", "._@-", "áéÜß._@-", "abcXYZ019._@- -_"]
+mismatch = None
+for i in range(20_000):
+ alphabet = _ALPHABETS[i % len(_ALPHABETS)]
+ s = "".join(random.choice(alphabet) for _ in range(random.randint(0, 60)))
+ if _IDENTIFIER.sub(" ", s) != _PREVIOUS.sub(" ", s):
+ mismatch = s
+ break
+ok("parity/20 000 random strings agree with the previous pattern",
+ mismatch is None, repr(mismatch))
+
+# Detection behaviour, not just the regex.
+check("a state of two domains is still not prose",
+ analyse({"body": "github.com acme.com"})["language"], None)
+check("the routing answer for ordinary prose is unchanged",
+ analyse({"body": "The customer was billed twice and wants a refund"})["is_english"], True)
+
+
+# ------------------------------------------------------------ complexity ----
+def elapsed(fn, *a):
+ started = time.perf_counter()
+ fn(*a)
+ return time.perf_counter() - started
+
+
+# Ceilings are set to fail the previous pattern rather than merely to be generous.
+# Previous cost / ceiling / current cost, measured on a 2023 laptop:
+# 4 000 chars 205 ms / 50 ms / 0.14 ms (~350x headroom)
+# 20 000 chars 4 860 ms / 1.00 s / 0.73 ms (~1370x headroom)
+for n, ceiling in ((4_000, 0.05), (20_000, 1.00)):
+ took = elapsed(_IDENTIFIER.sub, " ", "a" * n)
+ ok("%d word characters strip in under %.0f ms (took %.1f ms)" % (n, ceiling * 1000, took * 1000),
+ took < ceiling, "%.1f ms" % (took * 1000))
+
+# Hyphens and underscores are word characters here too, so the run can be built from them,
+# and a single trailing separator must not reopen the quadratic path.
+for label, text in (("hyphens", "a-" * 10_000),
+ ("underscores", "a_" * 10_000),
+ ("one trailing dot", "a" * 20_000 + ".")):
+ took = elapsed(_IDENTIFIER.sub, " ", text)
+ ok("%s: 20k strips in under 1 s (took %.1f ms)" % (label, took * 1000), took < 1.0,
+ "%.1f ms" % (took * 1000))
+
+# Scaling, which is machine-independent in a way a wall-clock ceiling is not: linear work
+# quadruples with the input, quadratic work grows sixteenfold. 8 sits between the two.
+small = min(elapsed(_IDENTIFIER.sub, " ", "a" * 5_000) for _ in range(3))
+large = min(elapsed(_IDENTIFIER.sub, " ", "a" * 20_000) for _ in range(3))
+ratio = large / small if small > 0 else 0
+ok("4x the input costs under 8x the time, ratio %.1f (linear ~4, quadratic ~16)" % ratio,
+ ratio < 8, "ratio %.1f (%.2f ms -> %.2f ms)" % (ratio, small * 1000, large * 1000))
+
+# The public entry point a request actually reaches. Compared against prose of the same
+# size rather than a wall-clock ceiling: at 4000 characters the gap between quadratic and
+# linear is only a few-fold, which CI jitter can cover, but the ratio against prose is
+# ~1x when the work is linear and ~100x when it is not.
+_PROSE_4K = ("The customer was billed twice and wants a refund. " * 80)[:4_000]
+
+
+def _best(state, reps=3):
+ return min(elapsed(analyse, state) for _ in range(reps))
+
+
+token_cost = _best({"body": "a" * 4_000})
+prose_cost = _best({"body": _PROSE_4K})
+ratio_4k = token_cost / max(prose_cost, 1e-5)
+ok("one long token costs about what prose of the same size costs (ratio %.1f)" % ratio_4k,
+ ratio_4k < 10, "%.2f ms vs %.2f ms" % (token_cost * 1000, prose_cost * 1000))
+took = elapsed(latin_profile, "a" * 50_000)
+ok("latin_profile() on 50k of one token is under 1 s (took %.1f ms)" % (took * 1000),
+ took < 1.0, "%.1f ms" % (took * 1000))
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)