Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(gen): every provider's one vision call asks the question it is given

Soohong Kim06922b2a8b90
diff --git a/sprite_gen/gen/base.py b/sprite_gen/gen/base.py
index d354a3e..361b724 100644
--- a/sprite_gen/gen/base.py
+++ b/sprite_gen/gen/base.py
@@ -282,6 +282,11 @@ class Provider(Protocol):
def inspect_facing(self, path: Path, workdir: Path) -> tuple[str, dict[str, Any]]: ...
+ # One vision call asking `prompt` about the image at `path`, on this provider's own login
+ # (the facing check asks its own question through `inspect_facing`; `logo` asks what the
+ # lettering spells). `label` names the call in a billing line and a failure.
+ def inspect_image(self, path: Path, workdir: Path, prompt: str, *, label: str = ...) -> tuple[str, dict[str, Any]]: ...
+
@dataclass
class GenResult:
diff --git a/sprite_gen/gen/codex_provider.py b/sprite_gen/gen/codex_provider.py
index 5c1d6be..0eb477a 100644
--- a/sprite_gen/gen/codex_provider.py
+++ b/sprite_gen/gen/codex_provider.py
@@ -343,6 +343,10 @@ class CodexProvider:
from .facing_vision import codex_inspect
return codex_inspect(path, workdir)
+ def inspect_image(self, path: Path, workdir: Path, prompt: str, *, label: str = "vision check") -> tuple[str, dict]:
+ from .facing_vision import codex_inspect
+ return codex_inspect(path, workdir, prompt=prompt, label=label)
+
def generate(self, request: GenRequest, workdir: Path) -> ProviderRun:
# `image_gen` exposes no effort or size dial, so neither knob is carried
# into the prompt. Refusing beats returning a default-effort image under a
diff --git a/sprite_gen/gen/facing_vision.py b/sprite_gen/gen/facing_vision.py
index 439792d..f440b56 100644
--- a/sprite_gen/gen/facing_vision.py
+++ b/sprite_gen/gen/facing_vision.py
@@ -3,6 +3,8 @@
REST input/output contract: OpenAI and xAI Responses image understanding docs.
No generated image is sent to a different provider or authentication route.
+One question per call: the facing check asks `facing.INSPECTION_PROMPT` (the default,
+its body unchanged); `sprite-gen logo` asks what the lettering spells (`inspect_image`).
"""
from __future__ import annotations
@@ -17,20 +19,26 @@ from .facing import INSPECTION_PROMPT
OPENAI_MODEL = "gpt-4.1-mini"
GROK_MODEL = "grok-4.6"
+FACING_LABEL = "facing vision check"
-def request_body(path: Path, model: str) -> dict:
+def _tag(label: str) -> str:
+ """The prefix of a failure line: "facing vision check" -> "facing-vision"."""
+ return label.removesuffix(" check").replace(" ", "-")
+
+
+def request_body(path: Path, model: str, *, prompt: str = INSPECTION_PROMPT, detail: str = "low") -> dict:
return {"model": model, "store": False, "max_output_tokens": 256,
"input": [{"role": "user", "content": [
- {"type": "input_text", "text": INSPECTION_PROMPT},
+ {"type": "input_text", "text": prompt},
{"type": "input_image", "image_url": "data:image/png;base64," +
- base64.b64encode(path.read_bytes()).decode("ascii"), "detail": "low"},
+ base64.b64encode(path.read_bytes()).decode("ascii"), "detail": detail},
]}]}
-def response_text(status: int, reply) -> tuple[str, dict]:
+def response_text(status: int, reply, *, label: str = FACING_LABEL) -> tuple[str, dict]:
if status != 200:
- raise SystemExit(f"facing-vision: HTTP {status}; not asked again, no fallback")
+ raise SystemExit(f"{_tag(label)}: HTTP {status}; not asked again, no fallback")
if not isinstance(reply, dict):
raise ValueError("invalid vision envelope")
text = "".join(part["text"] for item in reply.get("output", [])
@@ -40,34 +48,37 @@ def response_text(status: int, reply) -> tuple[str, dict]:
"response_status": reply.get("status")}
-def codex_inspect(path: Path, workdir: Path) -> tuple[str, dict]:
+def codex_inspect(path: Path, workdir: Path, *, prompt: str = INSPECTION_PROMPT,
+ label: str = FACING_LABEL) -> tuple[str, dict]:
# Isolated cwd and ephemeral session: no project instructions, no stale
# output reuse and no persistent vision rollouts. Uses the existing login.
from .codex_provider import announce_codex_billing, codex_login
login = codex_login()
if login.source is None:
- raise SystemExit(f"facing-vision: {login.reason}")
- announce_codex_billing(login, " (facing vision check)")
+ raise SystemExit(f"{_tag(label)}: {login.reason}")
+ announce_codex_billing(login, f" ({label})")
with tempfile.TemporaryDirectory(prefix="facing-", dir=workdir) as directory:
output = Path(directory) / "answer.txt"
command = [provider_binary("codex"), "exec", "--sandbox", "read-only",
"--skip-git-repo-check", "--ephemeral", "--color", "never",
"-C", directory, "-i", str(path.resolve()), "-o", str(output), "-"]
- completed = subprocess.run(command, input=INSPECTION_PROMPT, capture_output=True,
+ completed = subprocess.run(command, input=prompt, capture_output=True,
text=True, encoding="utf-8", env=provider_subprocess_env(),
timeout=GEN_TIMEOUT_SECONDS)
if completed.returncode:
- raise SystemExit(f"facing-vision: codex exited {completed.returncode}")
+ raise SystemExit(f"{_tag(label)}: codex exited {completed.returncode}")
return output.read_text(encoding="utf-8"), {"transport": "codex-exec", "auth_source": login.source}
-def grok_inspect(path: Path, *, credential=None, call=None) -> tuple[str, dict]:
+def grok_inspect(path: Path, *, credential=None, call=None, prompt: str = INSPECTION_PROMPT,
+ detail: str = "low", label: str = FACING_LABEL) -> tuple[str, dict]:
from . import xai
from .base import announce_api_billing
credential = credential or xai.resolve_credential()
if credential.source == xai.AUTH_SOURCE_API_KEY:
- announce_api_billing("grok", xai.AUTH_ENV, " (facing vision check).")
+ announce_api_billing("grok", xai.AUTH_ENV, f" ({label}).")
status, reply, _sends = xai.post(call or xai.http_json, xai.API_BASE + "/responses", credential.token,
- request_body(path, GROK_MODEL), who="grok", label="facing-vision")
- text, metadata = response_text(status, reply)
+ request_body(path, GROK_MODEL, prompt=prompt, detail=detail),
+ who="grok", label=_tag(label))
+ text, metadata = response_text(status, reply, label=label)
return text, {**metadata, "auth_source": credential.source, "transport": "xai-api"}
diff --git a/sprite_gen/gen/grok_provider.py b/sprite_gen/gen/grok_provider.py
index e9348a1..67a462f 100644
--- a/sprite_gen/gen/grok_provider.py
+++ b/sprite_gen/gen/grok_provider.py
@@ -125,6 +125,10 @@ class GrokProvider:
from .facing_vision import grok_inspect
return grok_inspect(path)
+ def inspect_image(self, path: Path, workdir: Path, prompt: str, *, label: str = "vision check") -> tuple[str, dict]:
+ from .facing_vision import grok_inspect
+ return grok_inspect(path, prompt=prompt, detail="high", label=label)
+
def generate(self, request: GenRequest, workdir: Path) -> ProviderRun:
if request.native_alpha:
raise SystemExit("grok-gen: grok Imagine cannot return an alpha channel; generate on a chroma key instead")
diff --git a/sprite_gen/gen/openai_provider.py b/sprite_gen/gen/openai_provider.py
index d5c16c7..ff36b18 100644
--- a/sprite_gen/gen/openai_provider.py
+++ b/sprite_gen/gen/openai_provider.py
@@ -279,13 +279,21 @@ class OpenAIProvider:
transparency = TRANSPARENCY_NATIVE
def inspect_facing(self, path: Path, workdir: Path) -> tuple[str, dict]:
+ from . import facing_vision as vision
+ return self._inspect(vision.request_body(path, vision.OPENAI_MODEL), vision.FACING_LABEL)
+
+ def inspect_image(self, path: Path, workdir: Path, prompt: str, *, label: str = "vision check") -> tuple[str, dict]:
+ from . import facing_vision as vision
+ return self._inspect(vision.request_body(path, vision.OPENAI_MODEL, prompt=prompt, detail="high"), label)
+
+ def _inspect(self, body: dict, label: str) -> tuple[str, dict]:
from . import facing_vision as vision
token = resolve_credential()
- announce_api_billing(self.name, AUTH_ENV, " (facing vision check).")
- data = json.dumps(vision.request_body(path, vision.OPENAI_MODEL)).encode("utf-8")
+ announce_api_billing(self.name, AUTH_ENV, f" ({label}).")
+ data = json.dumps(body).encode("utf-8")
status, reply, _headers, _asks = _ask(lambda: http_image(API_BASE + "/responses", token, data, "application/json",
timeout=GEN_TIMEOUT_SECONDS))
- text, metadata = vision.response_text(status, reply)
+ text, metadata = vision.response_text(status, reply, label=label)
return text, {**metadata, "auth_source": AUTH_SOURCE_API_KEY, "transport": "openai-api"}
def generate(self, request: GenRequest, workdir: Path) -> ProviderRun:
diff --git a/tests/gen/test_vision_read.py b/tests/gen/test_vision_read.py
new file mode 100644
index 0000000..365c0ba
--- /dev/null
+++ b/tests/gen/test_vision_read.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Every provider's one vision call takes the question it is asked (`inspect_image`).
+
+The facing check asks where a face points; `logo` asks what the lettering spells. Both go
+through the same transport on the provider that drew the image, on its own login, once.
+The facing question and its request body stay what they were.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from sprite_gen.gen import codex_provider, facing, facing_vision, grok_provider, openai_provider, xai
+
+FIXTURES = Path(__file__).parents[1] / "fixtures" / "facing"
+QUESTION = "Read the lettering."
+
+
+def _answer() -> dict:
+ return {"model": "test-vision", "status": "completed",
+ "output": [{"type": "message", "content": [{"type": "output_text", "text": '{"lines": ["ION"]}'}]}]}
+
+
+def test_the_facing_body_is_unchanged() -> None:
+ body = facing_vision.request_body(FIXTURES / "left.png", "m")
+ content = body["input"][0]["content"]
+ assert body["max_output_tokens"] == 256 and content[0]["text"] == facing.INSPECTION_PROMPT
+ assert content[1]["detail"] == "low"
+
+
+def test_a_question_and_its_detail_are_carried() -> None:
+ body = facing_vision.request_body(FIXTURES / "left.png", "m", prompt=QUESTION, detail="high")
+ content = body["input"][0]["content"]
+ assert content[0]["text"] == QUESTION and content[1]["detail"] == "high"
+
+
[email protected]("provider", ["openai", "grok"])
+def test_rest_providers_ask_the_question_once(monkeypatch, tmp_path, provider) -> None:
+ seen = []
+ if provider == "openai":
+ monkeypatch.setattr(openai_provider, "resolve_credential", lambda: "synthetic")
+
+ def http(url, token, data, content_type, **kw):
+ seen.append(json.loads(data))
+ return 200, _answer(), {}
+
+ monkeypatch.setattr(openai_provider, "http_image", http)
+ backend = openai_provider.OpenAIProvider()
+ else:
+ monkeypatch.setattr(xai, "resolve_credential", lambda: xai.Credential("synthetic", "grok-login"))
+
+ def http(method, url, token, body, **kw):
+ seen.append(body)
+ return 200, _answer()
+
+ monkeypatch.setattr(xai, "http_json", http)
+ backend = grok_provider.GrokProvider()
+ text, metadata = backend.inspect_image(FIXTURES / "left.png", tmp_path, QUESTION)
+ assert json.loads(text) == {"lines": ["ION"]}
+ assert len(seen) == 1 and seen[0]["input"][0]["content"][0]["text"] == QUESTION
+ assert metadata["model"] == "test-vision"
+
+
+def test_codex_asks_the_question_on_its_own_login(monkeypatch, tmp_path) -> None:
+ seen = []
+
+ def run(command, **kwargs):
+ seen.append((command, kwargs))
+ Path(command[command.index("-o") + 1]).write_text('{"lines": ["ION"]}')
+ return SimpleNamespace(returncode=0)
+
+ monkeypatch.setattr(facing_vision.subprocess, "run", run)
+ monkeypatch.setattr(codex_provider, "codex_login", lambda: codex_provider.CodexLogin(codex_provider.AUTH_SOURCE_LOGIN))
+ text, metadata = codex_provider.CodexProvider().inspect_image(FIXTURES / "left.png", tmp_path, QUESTION)
+ assert json.loads(text) == {"lines": ["ION"]} and metadata["auth_source"] == "codex-login"
+ assert len(seen) == 1 and seen[0][1]["input"] == QUESTION
+ assert "--ephemeral" in seen[0][0] and "read-only" in seen[0][0]
+
+
+def test_the_facing_check_still_asks_the_facing_question(monkeypatch, tmp_path) -> None:
+ seen = []
+
+ def run(command, **kwargs):
+ seen.append(kwargs["input"])
+ Path(command[command.index("-o") + 1]).write_text("left")
+ return SimpleNamespace(returncode=0)
+
+ monkeypatch.setattr(facing_vision.subprocess, "run", run)
+ monkeypatch.setattr(codex_provider, "codex_login", lambda: codex_provider.CodexLogin(codex_provider.AUTH_SOURCE_LOGIN))
+ assert codex_provider.CodexProvider().inspect_facing(FIXTURES / "left.png", tmp_path)[0] == "left"
+ assert seen == [facing.INSPECTION_PROMPT]