Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): identifier stripping is quadratic; 31 s -> 1.8 ms at 50k chars
Adeshina Lasisi9d4a4e9223ae
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ce48ff4f..0ffdce6a 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -96,6 +96,7 @@ jobs:python tests/test_lazy_import.pypython tests/test_runtime_fixes.pypython tests/test_lang_guess.py+ python tests/test_identifier_complexity.pypython tests/test_calibration_persistence.pypython tests/test_context_manager.pypython tests/test_criteria_normalization.py@@ -159,6 +160,7 @@ jobs:run: |for t in test_router test_criteria test_download test_shortlist \test_decision_model test_packaging test_lang_guess \+ test_identifier_complexity \test_calibration_persistence test_context_manager \test_criteria_normalization test_docker_entrypoint \test_empty_questions test_router_memory test_shortlist_cosine \diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 03d38ea9..fcb503e3 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -118,7 +118,11 @@ const SHARED_WORDS: Set<string> = (() => {})();const WORD_RE = /[^\W\d_]+/gu;-const IDENTIFIER_RE = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;+// Lookbehind for the same reason as the Python side (see laya/lang.py): without it the+// greedy prefix is retried at every offset inside a run of word characters, which is+// quadratic in the run's length -- 50 000 characters of one token took 1540 ms here.+// It removes no match, because a leftmost match can only begin at a run start.+const IDENTIFIER_RE = /(?<![\p{L}\p{N}_-])[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;const IS_ALPHA_RE = /\p{L}/u;// Python uses two different boundaries on purpose: detect_script/script_profile count the IPAdiff --git a/laya-ts/tests/identifier-complexity.test.ts b/laya-ts/tests/identifier-complexity.test.tsnew file mode 100644index 00000000..5c54d62f--- /dev/null+++ b/laya-ts/tests/identifier-complexity.test.ts@@ -0,0 +1,108 @@+// laya-ts/tests/identifier-complexity.test.ts+//+// Regression: stripping identifiers must stay linear in the length of a word-character run.+//+// IDENTIFIER_RE removes dot- and @-joined tokens before words are counted. Without the+// leading lookbehind the greedy prefix is retried at every offset inside a run of word+// characters, and each attempt rescans the run before failing on the absent [.@] --+// quadratic in the run's length. Measured on node 20: 50 000 characters of one token took+// 1540 ms, against 0.05 ms for ordinary prose of the same size. `analyse` runs on the+// default routing path once per request, so the cost is reachable by any caller.+//+// The lookbehind removes no match: the word class and [.@] are disjoint, so an attempt+// from inside a run consumes to exactly the same separator as one from the run's start.+// Parity below is asserted against the previous pattern rather than typed expectations.+import { describe, expect, it } from "vitest";+import { analyse, latinProfile } from "../src/lang.js";++const PREVIOUS = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;+const CURRENT = /(?<![\p{L}\p{N}_-])[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;++const strip = (re: RegExp, s: string) => s.replace(new RegExp(re.source, re.flags), " ");+const elapsed = (fn: () => void) => {+ const started = performance.now();+ fn();+ return performance.now() - started;+};++describe("identifier stripping: parity with the previous pattern", () => {+ const cases = [+ "github.com", "[email protected]", "v1.2.3", "U.S.A.", "arrivato.", "a.b", "a@b",+ "sub.domain.co.uk", "[email protected]", "192.168.0.1", "-a.b-", "_x.y_",+ "foo..bar", ".com", "a.", "@a", "a@", "e.g.", "Ü.Ö", "naïve.café", "a-b.c-d",+ "x-a.b", "9.9", "a..b", "a.-b", "a-.b-.c", "", ".", "@",+ "Contact [email protected] or github.com/acme for v2.10.1 details.",+ "No identifiers here at all just words",+ // A local part at RFC 5321's 64-character limit, and a run past any plausible bound:+ // a length-bounded fix strips only part of these and leaves the rest as a word.+ "a".repeat(64) + "@example.com",+ "a".repeat(200) + ".example.com",+ "grazie mille per " + "wzqxk".repeat(15) + ".example.com",+ ];+ for (const token of cases) {+ it(`strips ${JSON.stringify(token.slice(0, 34))} identically`, () => {+ expect(strip(CURRENT, token)).toBe(strip(PREVIOUS, token));+ });+ }++ it("agrees with the previous pattern on 20 000 random strings", () => {+ let seed = 12345;+ const rnd = () => ((seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff);+ const alphabets = ["aA1._@- ", "._@-", "áéÜß._@-", "abcXYZ019._@- -_"];+ let mismatch: string | null = null;+ for (let i = 0; i < 20_000 && mismatch === null; i++) {+ const alphabet = alphabets[i % alphabets.length]!;+ const len = Math.floor(rnd() * 60);+ let s = "";+ for (let j = 0; j < len; j++) s += alphabet[Math.floor(rnd() * alphabet.length)];+ if (strip(CURRENT, s) !== strip(PREVIOUS, s)) mismatch = s;+ }+ expect(mismatch).toBeNull();+ });++ it("leaves detection of ordinary prose unchanged", () => {+ expect(analyse("The customer was billed twice and wants a refund").isEnglish).toBe(true);+ });+});++describe("identifier stripping: complexity", () => {+ // These go through latinProfile / analyse -- the exported functions that use the+ // shipped IDENTIFIER_RE -- so reverting src/lang.ts fails them. Testing the pattern+ // literal above would only ever test the literal.+ //+ // Ceilings are set to fail the previous pattern, not merely to be generous. Previous+ // cost / ceiling / current cost on node 20, via latinProfile:+ // 20 000 chars 250 ms / 60 ms / 0.1 ms+ // 50 000 chars 1 540 ms / 200 ms / 0.3 ms+ it("latinProfile on 20 000 characters of one token is bounded", () => {+ expect(elapsed(() => latinProfile("a".repeat(20_000)))).toBeLessThan(60);+ });++ it("latinProfile on 50 000 characters of one token is bounded", () => {+ expect(elapsed(() => latinProfile("a".repeat(50_000)))).toBeLessThan(200);+ });++ it("a single trailing separator does not reopen the quadratic path", () => {+ expect(elapsed(() => latinProfile("a".repeat(50_000) + "."))).toBeLessThan(200);+ });++ it("hyphen and underscore runs are linear too", () => {+ expect(elapsed(() => latinProfile("a-".repeat(25_000)))).toBeLessThan(200);+ expect(elapsed(() => latinProfile("a_".repeat(25_000)))).toBeLessThan(200);+ });++ // No assertion at the 4000 characters `stateText` caps detection to: `analyse` does+ // enough non-regex work at that size that quadratic and linear are only a few-fold+ // apart, which CI jitter covers either way. The 20 000 and 50 000 character cases+ // above carry the guarantee -- they are 250x and 1500x apart.++ it("4x the input costs under 8x the time (linear ~4, quadratic ~16)", () => {+ const best = (n: number) =>+ Math.min(...[0, 1, 2].map(() => elapsed(() => latinProfile("a".repeat(n)))));+ const small = best(12_500);+ const large = best(50_000);+ // performance.now() granularity makes a sub-0.05 ms baseline meaningless; the+ // absolute ceilings above already carry the guarantee in that case.+ if (small > 0.05) expect(large / small).toBeLessThan(8);+ });+});diff --git a/laya/lang.py b/laya/lang.pyindex 0122c787..dde82c45 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -180,7 +180,14 @@ _WORD = re.compile(r"[^\W\d_]+", re.UNICODE)# state that was mostly links scored a language it does not contain, and two domains were enough to# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs# word characters on both sides of the dot.-_IDENTIFIER = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)+# The lookbehind is what keeps this linear. Without it the greedy `[\w-]*` is retried+# at every offset inside a run of word characters, and each attempt rescans the run+# before failing on the absent `[.@]` -- quadratic in the run's length, so 4000+# characters of one token cost 205 ms against 0.45 ms for ordinary prose. It removes+# no match: `[\w-]` and `[.@]` are disjoint, so an attempt from inside a run consumes+# to exactly the same separator as an attempt from the run's start and the two always+# succeed or fail together -- a leftmost match can only ever begin at a run start.+_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:diff --git a/tests/test_identifier_complexity.py b/tests/test_identifier_complexity.pynew file mode 100644index 00000000..d333a558--- /dev/null+++ b/tests/test_identifier_complexity.py@@ -0,0 +1,150 @@+"""Regression: stripping identifiers must stay linear in the length of a word-character run.++`_IDENTIFIER` removes dot- and @-joined tokens (`github.com`, `[email protected]`, `v1.2.3`)+before `_WORD` counts function words. Without the leading lookbehind, the greedy `[\\w-]*`+is retried at every offset inside a run of word characters and each attempt rescans the+run before failing on the absent `[.@]` -- quadratic in the run's length:++ 4 000 characters of one token 205 ms (0.45 ms for prose of the same size)+ 50 000 characters of one token 30 678 ms++`analyse()` runs on the default routing path -- no `model`, no `lang` -- once per request,+and `state_text` hands it up to 4000 characters, so an unauthenticated caller of+`POST /v1/systemone` could buy ~115x the CPU of a normal request. `latin_profile` and+`guess_latin_language` take unbounded text, where the same input costs ~31 s.++The lookbehind removes no match: `[\\w-]` and `[.@]` are disjoint, so an attempt from+inside a run consumes to exactly the same separator as an attempt from the run's start,+and the two always succeed or fail together. The parity block below is the evidence --+it is checked against the previous pattern, not against hand-written expectations.++Run: python tests/test_identifier_complexity.py+"""+import os+import re+import sys+import time++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.lang import _IDENTIFIER, analyse, latin_profile # noqa: E402++# The pattern as it was before the lookbehind, kept here so parity is asserted against+# the real previous behaviour rather than against expectations someone typed out.+_PREVIOUS = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++def ok(name, cond, extra=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s%s" % (name, (" -> " + str(extra)) if extra else ""))+++# ---------------------------------------------------------------- parity ----+# Every identifier the pattern exists to strip, plus the shapes that tempt a+# bounded-quantifier fix into leaving a residue token behind.+PARITY = [+ "github.com", "[email protected]", "v1.2.3", "U.S.A.", "arrivato.", "a.b", "a@b",+ "sub.domain.co.uk", "[email protected]", "192.168.0.1", "-a.b-", "_x.y_",+ "foo..bar", ".com", "a.", "@a", "a@", "e.g.", "Ü.Ö", "naïve.café", "a-b.c-d",+ "x-a.b", "9.9", "a..b", "a.-b", "a-.b-.c", "", ".", "@",+ "Contact [email protected] or github.com/acme for v2.10.1 details.",+ "No identifiers here at all just words",+ # A local part at RFC 5321's 64-character limit, and a run past any plausible bound:+ # a length-bounded pattern strips only part of these and leaves the rest as a word.+ "a" * 64 + "@example.com",+ "a" * 200 + ".example.com",+ "grazie mille per " + "wzqxk" * 15 + ".example.com",+]+for token in PARITY:+ check("parity/%r" % (token[:34],), _IDENTIFIER.sub(" ", token), _PREVIOUS.sub(" ", token))++# Randomised differential test: the two patterns must agree on everything.+import random # noqa: E402++random.seed(11)+_ALPHABETS = ["aA1._@- ", "._@-", "áéÜß._@-", "abcXYZ019._@- -_"]+mismatch = None+for i in range(20_000):+ alphabet = _ALPHABETS[i % len(_ALPHABETS)]+ s = "".join(random.choice(alphabet) for _ in range(random.randint(0, 60)))+ if _IDENTIFIER.sub(" ", s) != _PREVIOUS.sub(" ", s):+ mismatch = s+ break+ok("parity/20 000 random strings agree with the previous pattern",+ mismatch is None, repr(mismatch))++# Detection behaviour, not just the regex.+check("a state of two domains is still not prose",+ analyse({"body": "github.com acme.com"})["language"], None)+check("the routing answer for ordinary prose is unchanged",+ analyse({"body": "The customer was billed twice and wants a refund"})["is_english"], True)+++# ------------------------------------------------------------ complexity ----+def elapsed(fn, *a):+ started = time.perf_counter()+ fn(*a)+ return time.perf_counter() - started+++# Ceilings are set to fail the previous pattern rather than merely to be generous.+# Previous cost / ceiling / current cost, measured on a 2023 laptop:+# 4 000 chars 205 ms / 50 ms / 0.14 ms (~350x headroom)+# 20 000 chars 4 860 ms / 1.00 s / 0.73 ms (~1370x headroom)+for n, ceiling in ((4_000, 0.05), (20_000, 1.00)):+ took = elapsed(_IDENTIFIER.sub, " ", "a" * n)+ ok("%d word characters strip in under %.0f ms (took %.1f ms)" % (n, ceiling * 1000, took * 1000),+ took < ceiling, "%.1f ms" % (took * 1000))++# Hyphens and underscores are word characters here too, so the run can be built from them,+# and a single trailing separator must not reopen the quadratic path.+for label, text in (("hyphens", "a-" * 10_000),+ ("underscores", "a_" * 10_000),+ ("one trailing dot", "a" * 20_000 + ".")):+ took = elapsed(_IDENTIFIER.sub, " ", text)+ ok("%s: 20k strips in under 1 s (took %.1f ms)" % (label, took * 1000), took < 1.0,+ "%.1f ms" % (took * 1000))++# Scaling, which is machine-independent in a way a wall-clock ceiling is not: linear work+# quadruples with the input, quadratic work grows sixteenfold. 8 sits between the two.+small = min(elapsed(_IDENTIFIER.sub, " ", "a" * 5_000) for _ in range(3))+large = min(elapsed(_IDENTIFIER.sub, " ", "a" * 20_000) for _ in range(3))+ratio = large / small if small > 0 else 0+ok("4x the input costs under 8x the time, ratio %.1f (linear ~4, quadratic ~16)" % ratio,+ ratio < 8, "ratio %.1f (%.2f ms -> %.2f ms)" % (ratio, small * 1000, large * 1000))++# The public entry point a request actually reaches. Compared against prose of the same+# size rather than a wall-clock ceiling: at 4000 characters the gap between quadratic and+# linear is only a few-fold, which CI jitter can cover, but the ratio against prose is+# ~1x when the work is linear and ~100x when it is not.+_PROSE_4K = ("The customer was billed twice and wants a refund. " * 80)[:4_000]+++def _best(state, reps=3):+ return min(elapsed(analyse, state) for _ in range(reps))+++token_cost = _best({"body": "a" * 4_000})+prose_cost = _best({"body": _PROSE_4K})+ratio_4k = token_cost / max(prose_cost, 1e-5)+ok("one long token costs about what prose of the same size costs (ratio %.1f)" % ratio_4k,+ ratio_4k < 10, "%.2f ms vs %.2f ms" % (token_cost * 1000, prose_cost * 1000))+took = elapsed(latin_profile, "a" * 50_000)+ok("latin_profile() on 50k of one token is under 1 s (took %.1f ms)" % (took * 1000),+ took < 1.0, "%.1f ms" % (took * 1000))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)