Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.
fix(gen): the error body is masked string by string before it is put on one line
Soohong Kim4b0165d24d17
diff --git a/docs/gen.md b/docs/gen.mdindex aa3309f..8232c94 100644--- a/docs/gen.md+++ b/docs/gen.md@@ -141,11 +141,15 @@ with `respect_moderation=false` in place of a code (docs/video.md).`<body>` is the provider's error body as received, last on the line: JSON on oneline, anything else as its text (line breaks written as `\n`), nothing when the body-was empty. Two things in it are masked with `[redacted]` (`refusal.redact`): strings+was empty. Two things in it are masked with `[redacted]` (`refusal.masked`): stringsshaped like an API key (`sk-…`, `xai-…`, a `Bearer` value, and the credential therequest carried; a key the provider already masked stays as it came) and the query-string of any URL, where a signature lives. The body can repeat the prompt, so a-product that shows sprite-gen's stderr to its own users decides what to cut there.+string of any URL wherever the URL starts, where a signature lives. Each string is+masked as the provider sent it, before the body is put on one line, so only a letter+or digit of the provider's own text in front makes `sk-`, `xai-` or `bearer` part of+a word (`task-…`); a line break, a tab or any other character does not. The body can+repeat the prompt, so a product that shows sprite-gen's stderr to its own users+decides what to cut there.The video verbs print what xAI said the same way, through the same mask. Because thebody comes after everything else, nothing it says moves the status, reason or codea caller reads off the front.diff --git a/sprite_gen/gen/refusal.py b/sprite_gen/gen/refusal.pyindex 9d03540..aa1f2a6 100644--- a/sprite_gen/gen/refusal.py+++ b/sprite_gen/gen/refusal.py@@ -16,9 +16,12 @@ quota or parameter errors without knowing any provider's vocabulary.`<body>` is the provider's error body as received — JSON on one line, anythingelse as its text — so an open-source tool shows the whole error and the productthat wraps it decides what its own users see. Only two things are masked-(`redact`): strings shaped like an API key (the credential the request carried-among them) and the query string of a URL, where a signature lives. The body comes last, so nothing in it can move what a caller-reads off the front of the line.+(`masked`): strings shaped like an API key (the credential the request carried+among them) and the query string of a URL, where a signature lives. The mask+runs on each string of the body as the provider sent it, before the body is put+on one line, so no quote or escape this module writes can hide a key from it.+The body comes last, so nothing in it can move what a caller reads off the front+of the line."""from __future__ import annotations@@ -33,12 +36,13 @@ CONTENT_POLICY = "refused by the provider's content policy"# A key is a known prefix and a long run of key characters. One the provider# already masked (`sk-proj-****…abcd`) carries `*` and is left as it came. Only a-# letter or digit before the prefix makes it part of a word (`task-…`); after-# `_` or `-` (`api_key_sk-…`) the key starts there.+# letter or digit of the provider's text before the prefix makes it part of a+# word (`task-…`); after `_` or `-` (`api_key_sk-…`), a line break or any other+# character the key starts there._KEY = re.compile(r"(?<![A-Za-z0-9])(sk|xai)-([A-Za-z0-9_*-]+)")_KEY_MIN_LENGTH = 16_BEARER = re.compile(r"(?i)(?<![A-Za-z0-9])(bearer\s+)[A-Za-z0-9._~+/=-]{16,}")-_URL_QUERY = re.compile(r"(?i)\b(https?://[^\s?#\"'<>\\]+)\?[^\s#\"'<>\\]*")+_URL_QUERY = re.compile(r"(?i)(https?://[^\s?#\"'<>\\]+)\?[^\s#\"'<>\\]*")REDACTED = "[redacted]"# Which provider codes are a content-policy block, per provider; the one table.@@ -73,6 +77,10 @@ def _key(match: re.Match[str]) -> str:def redact(text: str, secret: str | None = None) -> str:"""`text` with key-shaped strings and URL query strings masked; nothing else.+ `text` is one string as the provider sent it (`masked` hands each one over),+ never a line this module has quoted, escaped or cut: what stands in front of+ a key is the provider's own character.+`secret` is the credential the request was sent with: an echo of it is a keywhatever its shape (a login token need not start with `sk-` or `xai-`)."""@@ -83,16 +91,41 @@ def redact(text: str, secret: str | None = None) -> str:return _URL_QUERY.sub(lambda m: f"{m.group(1)}?{REDACTED}", text)+def masked(body: Any, secret: str | None = None) -> Any:+ """A copy of a parsed error body with every string in it redacted — dict keys+ and values, list items, a `RawBody`'s text — and nothing else changed. Mask a+ body with this before anything serializes, escapes or cuts it.++ A number is left alone unless the credential shows in it; then it becomes+ the masked text. Two field names of one object that mask to the same text+ keep the later value: a provider's field names are not key-shaped, so only+ an entry whose name was masked can drop out.+ """+ if isinstance(body, RawBody):+ return RawBody(redact(body.text, secret))+ if isinstance(body, str):+ return redact(body, secret)+ if isinstance(body, dict):+ return {masked(key, secret): masked(value, secret) for key, value in body.items()}+ if isinstance(body, list):+ return [masked(item, secret) for item in body]+ if isinstance(body, (int, float)) and not isinstance(body, bool) and secret and len(secret) >= 8:+ text = json.dumps(body)+ return redact(text, secret) if secret in text else body+ return body++def said(body: Any, secret: str | None = None) -> str:- """`: <body>` for the end of a refusal line — the body as received, on one- line and redacted — or "" when the provider sent nothing."""+ """`: <body>` for the end of a refusal line — the body as received, masked+ and then put on one line — or "" when the provider sent nothing."""+ body = masked(body, secret)if isinstance(body, RawBody):text = body.text.strip().replace("\r\n", "\n").replace("\r", "\n").replace("\n", "\\n")elif body in (None, {}, ""):text = ""else:text = json.dumps(body, ensure_ascii=False)- return f": {redact(text, secret)}" if text else ""+ return f": {text}" if text else ""def identifier(value: Any) -> str | None:diff --git a/sprite_gen/gen/video.py b/sprite_gen/gen/video.pyindex 1ebca98..f64f033 100644--- a/sprite_gen/gen/video.py+++ b/sprite_gen/gen/video.py@@ -125,12 +125,14 @@ def _data_url(path: Path) -> str:def _error_detail(body: Any, secret: str | None = None) -> str:# What the provider said, with keys and URL query strings masked — the same- # mask the image lines use (refusal.redact). Masked before it is cut, so a- # key the cut would split is still whole when the mask looks for it.+ # mask the image lines use (refusal.masked). Masked before it is quoted,+ # escaped or cut, so the mask sees each string as xAI sent it and a key the+ # cut would split is still whole.+ body = refusal.masked(body, secret)if not isinstance(body, dict):- return refusal.redact(str(body), secret)[:300]+ return str(body)[:300]parts = [f"{key}={body[key]!r}" for key in ("code", "error", "message", "status", "raw") if body.get(key)]- return refusal.redact(", ".join(parts), secret) if parts else refusal.redact(json.dumps(body), secret)[:300]+ return ", ".join(parts) if parts else json.dumps(body)[:300]def _redact_host(url: str) -> str | None:diff --git a/tests/gen/test_refusal.py b/tests/gen/test_refusal.pynew file mode 100644index 0000000..388e056--- /dev/null+++ b/tests/gen/test_refusal.py@@ -0,0 +1,160 @@+# SPDX-License-Identifier: Apache-2.0+"""The mask on a provider's error body sees each string as the provider sent it.++Every value here is synthetic. A key, a `Bearer` value or a URL query in a body+is masked whatever stands in front of it, except a key or `Bearer` value right+after an ASCII letter or digit of the provider's own text (`task-…`, `xbearer …`).+The credential the request carried is masked everywhere.+"""+from __future__ import annotations++import pytest++from sprite_gen.gen import refusal, video++SECRET = "synthetic-login-token-0123456789" # the credential the request carried++# kind -> (the token as a provider body repeats it, the part that must not be readable)+TOKENS = {+ "sk": ("sk-" + "Synthetic0Key1For2Tests" * 2, "Synthetic0Key1For2Tests"),+ "xai": ("xai-" + "Synthetic0Key1For2Tests" * 2, "Synthetic0Key1For2Tests"),+ "bearer": ("Bearer synthetic.token.value-0123456789", "synthetic.token.value-0123456789"),+ "url": ("https://files.example.invalid/o.png?X-Signature=0a1b2c3d&Expires=1", "X-Signature=0a1b2c3d"),+ "secret": (SECRET, SECRET),+}++# What stands right in front of the token in the provider's text. JSON, `repr`+# and the raw-body line join write most of these as an escape (`\n`, `\t`,+# `\x0b`, `→`), which a mask on the finished line read as a letter or digit.+SEPARATORS = {+ "start": "", "space": " ", "colon": ":", "equals": "=", "paren": "(", "dquote": '"', "squote": "'",+ "backslash": "\\", "underscore": "_", "hyphen": "-",+ "LF": "\n", "CRLF": "\r\n", "CR": "\r", "TAB": "\t", "VT": "\x0b", "FF": "\x0c", "BS": "\x08",+ "SOH": "\x01", "DEL": "\x7f", "NEL": "\x85", "NBSP": "\xa0", "ZWSP": "", "LSEP": " ",+ "arrow": "→", "e-acute": "é", "hangul": "한", "emoji": "\U0001f600",+ # A letter or digit in front: a key or Bearer value is part of a word.+ "letter": "x", "digit": "2",+ # The provider's own encoding: its text has a letter or digit in front, so the same.+ "provider-backslash-n": "\\n", "provider-percent-0A": "%0A",+}++# path -> how the provider's string reaches the printed line+PATHS = {+ "said:json-value": lambda s: refusal.said({"error": {"message": s}}, SECRET),+ "said:json-key": lambda s: refusal.said({"error": {s: 1}}, SECRET),+ "said:raw": lambda s: refusal.said(refusal.RawBody(s), SECRET),+ "video:repr": lambda s: video._error_detail({"message": s}, SECRET),+ "video:json": lambda s: video._error_detail({"detail": s}, SECRET),+ "video:list": lambda s: video._error_detail([s], SECRET),+}+++def _masked_by_rule(sep: str, kind: str) -> bool:+ before = sep[-1:]+ return kind in ("url", "secret") or not (before.isascii() and before.isalnum())++[email protected]("path", PATHS)[email protected]("kind", TOKENS)[email protected]("sep", SEPARATORS)+def test_masked_whatever_stands_in_front_unless_a_letter_or_digit_makes_it_a_word(sep, kind, path) -> None:+ """31 separators x 5 kinds x 6 paths: 858 cells masked, 72 left as a word."""+ token, hidden = TOKENS[kind]+ printed = PATHS[path](("x" + SEPARATORS[sep] if SEPARATORS[sep] else "") + token)+ if _masked_by_rule(SEPARATORS[sep], kind):+ assert hidden not in printed+ assert "[redacted]" in printed+ else:+ assert hidden in printed+ assert "[redacted]" not in printed+++def test_a_key_after_a_line_break_in_json() -> None:+ assert refusal.said({"error": {"message": "x\nsk-" + "A" * 40}}) == ': {"error": {"message": "x\\nsk-[redacted]"}}'+++def test_a_bearer_value_after_a_line_break_in_a_raw_body() -> None:+ assert refusal.said(refusal.RawBody("x\nBearer " + "t" * 32)) == ": x\\nBearer [redacted]"+++def test_a_bearer_value_after_a_line_break_on_the_video_lines() -> None:+ assert video._error_detail({"message": "x\nBearer " + "t" * 32}) == "message='x\\nBearer [redacted]'"+++def test_a_bearer_value_after_a_tab_in_json() -> None:+ assert refusal.said({"error": {"message": "x\tBearer " + "t" * 32}}) == ': {"error": {"message": "x\\tBearer [redacted]"}}'+++def test_a_key_after_a_non_ascii_character_on_the_video_json_fallback() -> None:+ assert video._error_detail({"x": ["→sk-" + "A" * 40]}) == '{"x": ["\\u2192sk-[redacted]"]}'++[email protected](("said", "printed"), [+ ("see\nhttps://files.example.invalid/o.png?X-Signature=0a1b",+ ': {"error": {"message": "see\\nhttps://files.example.invalid/o.png?[redacted]"}}'),+ ("see_https://files.example.invalid/o.png?X-Signature=0a1b",+ ': {"error": {"message": "see_https://files.example.invalid/o.png?[redacted]"}}'),+ ("seehttps://files.example.invalid/o.png?X-Signature=0a1b",+ ': {"error": {"message": "seehttps://files.example.invalid/o.png?[redacted]"}}'),+ ("한https://files.example.invalid/o.png?X-Signature=0a1b",+ ': {"error": {"message": "한https://files.example.invalid/o.png?[redacted]"}}'),+], ids=["line-break", "underscore", "letter", "hangul"])+def test_a_url_query_is_masked_wherever_the_url_starts(said, printed) -> None:+ assert refusal.said({"error": {"message": said}}) == printed++[email protected]("path", PATHS)[email protected]("gap", ["\t", "\n", "\r\n", "\xa0"], ids=["TAB", "LF", "CRLF", "NBSP"])+def test_a_bearer_value_after_any_whitespace(gap, path) -> None:+ assert "synthetic.token" not in PATHS[path]("x Bearer" + gap + "synthetic.token.value-0123456789")++[email protected]("path", PATHS)[email protected](("secret", "hidden"), [+ ('synthetic"quoted-token-0123', "quoted-token-0123"),+ ("synthetic-töken-0123456789", "ken-0123456789"),+], ids=["quote", "non-ascii"])+def test_the_credential_is_masked_even_with_a_character_a_serializer_escapes(secret, hidden, path) -> None:+ body = "echo " + secret+ shapes = {+ "said:json-value": lambda: refusal.said({"error": {"message": body}}, secret),+ "said:json-key": lambda: refusal.said({"error": {body: 1}}, secret),+ "said:raw": lambda: refusal.said(refusal.RawBody(body), secret),+ "video:repr": lambda: video._error_detail({"message": body}, secret),+ "video:json": lambda: video._error_detail({"detail": body}, secret),+ "video:list": lambda: video._error_detail([body], secret),+ }+ assert hidden not in shapes[path]()++[email protected]("path", ["said", "video:repr", "video:json"])+def test_a_number_equal_to_the_credential_is_masked(path) -> None:+ secret = "123456789012"+ printed = {+ "said": lambda: refusal.said({"error": {"n": 123456789012}}, secret),+ "video:repr": lambda: video._error_detail({"code": 123456789012}, secret),+ "video:json": lambda: video._error_detail({"n": 123456789012}, secret),+ }[path]()+ assert "123456789012" not in printed and "[redacted]" in printed+++def test_masked_changes_strings_only_and_keeps_a_raw_body_raw() -> None:+ key = TOKENS["sk"][0]+ body = {"error": {"code": None, "n": 3, "ok": False, "items": [1.5, key]}}+ assert refusal.masked(body) == {"error": {"code": None, "n": 3, "ok": False, "items": [1.5, "sk-[redacted]"]}}+ raw = refusal.masked(refusal.RawBody("x\n" + key))+ assert isinstance(raw, refusal.RawBody)+ assert raw.text == "x\nsk-[redacted]" and raw == {"raw": "x\nsk-[redacted]"}+++def test_masked_leaves_the_body_it_was_given_alone() -> None:+ key = TOKENS["sk"][0]+ body = {"error": {"message": key, "items": [key]}}+ refusal.masked(body, SECRET)+ assert body == {"error": {"message": key, "items": [key]}}+++def test_the_credential_is_masked_whole_and_a_key_keeps_its_prefix() -> None:+ key = TOKENS["sk"][0]+ assert refusal.said({"error": {"message": f"sent {key}"}}, key) == ': {"error": {"message": "sent [redacted]"}}'+ assert refusal.said({"error": {"message": f"sent {key}"}}) == ': {"error": {"message": "sent sk-[redacted]"}}'diff --git a/tests/gen/test_video.py b/tests/gen/test_video.pyindex a392a32..1834597 100644--- a/tests/gen/test_video.py+++ b/tests/gen/test_video.py@@ -474,7 +474,7 @@ def test_a_failed_poll_names_the_code_inside_its_error_object(tmp_path: Path) ->@pytest.mark.parametrize("scripted", ["post", "poll"])def test_what_xai_said_is_shown_with_keys_and_signatures_masked(tmp_path: Path, scripted) -> None:- """The image lines' mask (refusal.redact) on the clip lines, which already print the body."""+ """The image lines' mask (refusal.masked) on the clip lines, which already print the body."""token = "synthetic-login-token"said = (f"echo {token}, key xai-{'Synthetic0Key1For2Tests3Only4' * 3}, ""url https://vidgen.x.ai/v/1.mp4?sig=abc, (HTTP 999) code=decoy")