Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop the log redactor backtracking quadratically on a cut ANSI sequence (#10721)

Daniel Hand7e5c6baa096
diff --git a/studio/backend/tests/test_debug_log_redaction.py b/studio/backend/tests/test_debug_log_redaction.py
index b3bcbd59821..c0b8ed1764a 100644
--- a/studio/backend/tests/test_debug_log_redaction.py
+++ b/studio/backend/tests/test_debug_log_redaction.py
@@ -7,6 +7,7 @@ hides the failure the user opened the log to read."""
from __future__ import annotations
+import re
import sys
from pathlib import Path
@@ -289,3 +290,201 @@ def test_studio_s3_secret_key_spellings_are_masked():
def test_talking_about_the_s3_key_without_a_value_survives():
line = "secret_access_key is required when use_iam_role is false"
assert redact_log_text(line) == line
+
+
+# The lazy alternation this file's subject replaced. Kept verbatim as the oracle:
+# the change is a performance fix, so the contract is that _strip_ansi returns
+# exactly what this returns, on every input.
+_LAZY_ANSI_RE = re.compile(
+ r"\x1b\][\s\S]*?(?:\x07|\x1b\\|\x9c)"
+ r"|\x1b[P^_X][\s\S]*?(?:\x1b\\|\x9c)"
+ r"|\x1b\[[0-?]*[ -/]*[@-~]"
+ r"|\x1b[@-Z\\-_]"
+ r"|\x9b[0-?]*[ -/]*[@-~]"
+ r"|[\x9d\x90\x98\x9e\x9f][\s\S]*?(?:\x07|\x9c)"
+)
+
+_ANSI_SHAPES = [
+ # Well formed, every introducer and every terminator.
+ "\x1b[36m",
+ "\x1b[0m",
+ "\x1b[38;5;196m",
+ "\x1b[?25l",
+ "\x1b[2K",
+ "\x9b0m",
+ "\x1b]0;title\x07",
+ "\x1b]0;title\x1b\\",
+ "\x1b]0;title\x9c",
+ "\x9d0;title\x9c",
+ "\x1bPx\x1b\\",
+ "\x1b^x\x9c",
+ "\x1b_x\x1b\\",
+ "\x1bXx\x9c",
+ "\x90x\x07",
+ "\x98x\x9c",
+ "\x9ex\x07",
+ "\x9fx\x9c",
+ "\x1bM",
+ "\x1b7",
+ "\x1b(B",
+ # Truncated: an introducer whose sequence never terminates.
+ "\x1b]",
+ "\x1b]cut",
+ "\x1bP",
+ "\x1bPcut",
+ "\x1b^cut",
+ "\x1b_cut",
+ "\x1bXcut",
+ "\x9d",
+ "\x9dcut",
+ "\x90cut",
+ "\x98cut",
+ "\x9ecut",
+ "\x9fcut",
+ "\x1b",
+ "\x1b[",
+ "\x1b[38;5",
+ "\x9b",
+ "\x9b38;5",
+ "\x1b(",
+ # Cut, then a well formed sequence later in the same record.
+ "\x1b]cut\x1b]t\x07",
+ "\x9dcut\x9dt\x07",
+ "\x1bPcut\x1bPt\x1b\\",
+ "\x1b]cut\x1b[36m",
+ "\x1b]cut\x9b36m",
+ "\x1b\x1b[0m",
+ # Stray terminators with no introducer, and interleaved introducers.
+ "\x07",
+ "\x9c",
+ "\x1b\\",
+ "\x1b]\x9d\x07",
+ "\x9d\x1b]\x07",
+ "\x1b]\x1b]\x1b]\x07",
+]
+
+
+def test_the_strip_is_unchanged_by_the_rewrite():
+ """The contract. The lazy alternation was replaced because it backtracked,
+ not because its answers were wrong, so the walk has to agree with it
+ everywhere: same alternatives, same order, same lazy shortest match, same
+ fallthrough when a control string never terminates.
+
+ A redactor is the wrong place to smuggle a behaviour change into a
+ performance fix, and every way of "improving" the truncated cases that was
+ tried here moved a leak rather than removing one: consuming an aborted body
+ ate the separator out of "api_key<cut>=value", and dropping a lone escape
+ welded "prefix" onto "api_key".
+ """
+ from utils.log_redaction import _strip_ansi
+
+ lines = [
+ "api_key=abcdef123456",
+ "Authorization: Bearer abcdef123456",
+ "Cookie: session=abcdef123456",
+ "?token=abcdef123456&next=1",
+ "--password hunter2secret",
+ "INFO loading unsloth/Llama-3.2-1B revision 8f3a2b1c in 13ms",
+ "n_tokens = 4096, token_id=128009",
+ ]
+ checked = 0
+ for shape in _ANSI_SHAPES:
+ for line in lines:
+ middle = len(line) // 2
+ for text in (
+ shape + line,
+ line + shape,
+ shape + line + shape,
+ line[:middle] + shape + line[middle:],
+ ):
+ checked += 1
+ assert _strip_ansi(text) == _LAZY_ANSI_RE.sub("", text), text
+ assert checked > 1000
+
+
+def test_the_strip_is_unchanged_on_random_records():
+ """The shapes above are the ones someone thought of. This covers the ones
+ nobody did, which is where every finding on this change actually came
+ from."""
+ import random
+
+ from utils.log_redaction import _strip_ansi
+
+ rng = random.Random(20260913)
+ alphabet = (
+ list("\x1b\x07\x9c\x9b\x9d\x9e\x9f\x90\x98") * 4
+ + list("[]P^_X0123456789;?m\\ ") * 2
+ + list("abcdefghijklmnopqrstuvwxyzABCDEF =:\"'-_/.&?\n\r")
+ )
+ for _ in range(20000):
+ text = "".join(rng.choice(alphabet) for _ in range(rng.randint(1, 80)))
+ assert _strip_ansi(text) == _LAZY_ANSI_RE.sub("", text), repr(text)
+
+
+def test_an_unterminated_ansi_introducer_does_not_cost_quadratic_time():
+ """A lazy scan for the terminator backtracks: the introducer with no
+ terminator scans to end of string, fails, and falls through to the single
+ character Fe branch, so the cost grows with the square of the record.
+
+ Before the negated body classes, 40k of these took ~15.8s against ~0.005s
+ for the same length of ordinary text, and the log viewer hands whole lines
+ to this function once a second. An unterminated introducer is not exotic; a
+ rotated log or a writer cut mid sequence leaves one behind.
+
+ Timing is asserted loosely, as a shape rather than a number: quadratic here
+ is seconds and linear is milliseconds, so any threshold in between separates
+ them on any host.
+ """
+ import time
+
+ shapes = [
+ "\x9d",
+ "\x90",
+ "\x98",
+ "\x9e",
+ "\x9f",
+ "\x9b",
+ "\x1b",
+ "\x1b]",
+ "\x1bP",
+ "\x1b[",
+ # Interleaved: no single body class can run to the end, but each start
+ # still offers the next one a fresh full scan under a lazy body.
+ "\x1b]\x9d",
+ "\x1bP\x9e\x1b[",
+ # A terminator that belongs to nobody, and a key in front of it, which
+ # is what a colorized line cut at a page boundary actually looks like.
+ "api_key\x9d",
+ "\x1b]title\x9d",
+ ]
+ for shape in shapes:
+ text = shape * (40000 // len(shape))
+ started = time.monotonic()
+ redact_log_text(text)
+ elapsed = time.monotonic() - started
+ assert elapsed < 2.0, f"{shape!r} took {elapsed:.1f}s"
+
+
+# The seven introducers _ANSI_INTRODUCER_RE recognises, which is the set that
+# decides whether the strip runs at all.
+_INTRODUCERS = ("\x1b", "\x90", "\x98", "\x9b", "\x9d", "\x9e", "\x9f")
+
+
+def test_terminated_ansi_sequences_are_still_stripped():
+ """The walk must not cost the stripping the rules depend on: an escape
+ between a key and its value stops every anchored rule matching, and a
+ terminated sequence has to disappear whichever of the six forms it is."""
+ for text in (
+ "\x1b[36mpassword\x1b[0m=hunter2secret",
+ "\x1b]0;title\x1b\\api_key=abcdef123456",
+ "\x1b]0;title\x07api_key=abcdef123456",
+ "\x1b]0;title\x9capi_key=abcdef123456",
+ "\x1bPsome dcs\x1b\\api_key=abcdef123456",
+ "\x9dbody\x9capi_key=abcdef123456",
+ "\x9bmapi_key=abcdef123456",
+ "api\x1b[36m_key=abcdef123456",
+ "api\x1b[36mkey=abcdef123456",
+ ):
+ masked = redact_log_text(text)
+ assert "abcdef123456" not in masked and "hunter2secret" not in masked, text
+ assert REDACTED in masked, text
diff --git a/studio/backend/utils/log_redaction.py b/studio/backend/utils/log_redaction.py
index 21af24eca3d..85e5740c159 100644
--- a/studio/backend/utils/log_redaction.py
+++ b/studio/backend/utils/log_redaction.py
@@ -14,16 +14,77 @@ import re
REDACTED = "<redacted>"
-# Terminal control sequences, stripped BEFORE anything is matched. Order matters: OSC comes before the single-character Fe class, which covers 0x5C-0x5F and would otherwise swallow the "]". ECMA-48 5.4 (CSI) and 5.6 (OSC / DCS / SOS / PM / APC). A colorized writer puts an escape between key and value, and the "m" ending a colour code is a word character, so every anchored rule below stops matching.
-_ANSI_RE = re.compile(
- r"\x1b\][\s\S]*?(?:\x07|\x1b\\|\x9c)"
- r"|\x1b[P^_X][\s\S]*?(?:\x1b\\|\x9c)"
- r"|\x1b\[[0-?]*[ -/]*[@-~]"
- r"|\x1b[@-Z\\-_]"
- r"|\x9b[0-?]*[ -/]*[@-~]"
- r"|[\x9d\x90\x98\x9e\x9f][\s\S]*?(?:\x07|\x9c)"
-)
+# Stripped BEFORE anything is matched: a colorized writer puts an escape between a key and its value, and the "m" ending a colour code is a word character, so every anchored rule below stops matching. ECMA-48 5.4 (CSI) and 5.6 (OSC / DCS / SOS / PM / APC).
+# _strip_ansi replaces one alternation of lazy `[\s\S]*?` bodies whose FAILURE was quadratic: an unterminated introducer scanned to end of record, failed, and the engine retried from the next position. Its output is identical to that pattern on every input, by differential fuzzing, and must stay so: a redactor is the wrong place to smuggle a behaviour change into a performance fix, and every attempt to also improve the truncated cases moved a leak rather than removing one (unslothai/unsloth#10721).
+_CSI_7BIT_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+_CSI_8BIT_RE = re.compile(r"\x9b[0-?]*[ -/]*[@-~]")
+# Fe covers 0x40-0x5F, so it also claims a "]" or "P" whose control string never terminated. Hence tried last.
+_FE_RE = re.compile(r"\x1b[@-Z\\-_]")
_ANSI_INTRODUCER_RE = re.compile(r"[\x1b\x90\x98\x9b\x9d-\x9f]")
+_C1_STRING_INTRODUCERS = "\x9d\x90\x98\x9e\x9f"
+
+
+def _strip_ansi(text: str) -> str:
+ """Remove terminal control sequences. Same output as the lazy alternation this replaces, in linear time."""
+ first = _ANSI_INTRODUCER_RE.search(text)
+ if first is None:
+ return text
+ out: list[str] = []
+ written = 0
+ index = first.start()
+ length = len(text)
+ # index only moves forward, so a cached hit at or after it is still the next one and a cached miss stays a miss. This is what makes the scan linear.
+ found: dict[str, int] = {}
+
+ def next_index(needle: str, start: int) -> int:
+ cached = found.get(needle, -2)
+ if cached == -1:
+ return -1
+ if cached == -2 or cached < start:
+ cached = text.find(needle, start)
+ found[needle] = cached
+ return cached
+
+ def string_end(terminators: tuple[str, ...], start: int) -> int:
+ """End of the shortest body, which is what a lazy quantifier picks."""
+ best, best_length = -1, 0
+ for terminator in terminators:
+ at = next_index(terminator, start)
+ if at >= 0 and (best < 0 or at < best):
+ best, best_length = at, len(terminator)
+ return best + best_length if best >= 0 else -1
+
+ while index < length:
+ char = text[index]
+ end = -1
+ if char == "\x1b" and index + 1 < length and text[index + 1] == "]":
+ end = string_end(("\x07", "\x1b\\", "\x9c"), index + 2)
+ elif char == "\x1b" and index + 1 < length and text[index + 1] in "P^_X":
+ end = string_end(("\x1b\\", "\x9c"), index + 2)
+ elif char in _C1_STRING_INTRODUCERS:
+ end = string_end(("\x07", "\x9c"), index + 1)
+ if end < 0:
+ if char == "\x1b":
+ match = _CSI_7BIT_RE.match(text, index) or _FE_RE.match(text, index)
+ elif char == "\x9b":
+ match = _CSI_8BIT_RE.match(text, index)
+ else:
+ match = None
+ end = match.end() if match else -1
+ if end < 0:
+ # Skip to the next introducer, not the next character, so ordinary text is never walked one character at a time.
+ following = _ANSI_INTRODUCER_RE.search(text, index + 1)
+ if following is None:
+ break
+ index = following.start()
+ continue
+ out.append(text[written:index])
+ written = end
+ following = _ANSI_INTRODUCER_RE.search(text, end)
+ index = following.start() if following else length
+ out.append(text[written:])
+ return "".join(out)
+
# Key names whose VALUE is a secret. "token" alone is absent on purpose, so n_tokens = 4096 and token_id=128009 survive.
_SECRET_KEYS = (
@@ -158,7 +219,7 @@ def redact_log_text(text: str) -> str:
return text
# Nothing anchored below survives an escape between a key and its value, so strip first, guarded by one introducer scan: ordinary content is untouched.
if _ANSI_INTRODUCER_RE.search(text):
- text = _ANSI_RE.sub("", text)
+ text = _strip_ansi(text)
for pattern, replacement in _PATTERNS:
text = pattern.sub(replacement, text)
# Before the key/value rules: _KV_RE captures "Basic" from "Authorization: Basic dXNlcjpwdw==", masking the scheme and leaving the credential clear.