Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

feat(agent): image generation + anchor picker (3-phase director loop)

mrbizarro7d347b842210
diff --git a/agent/image_engine.py b/agent/image_engine.py
new file mode 100644
index 0000000..912479c
--- /dev/null
+++ b/agent/image_engine.py
@@ -0,0 +1,276 @@
+"""Image generation engines for the Agentic Flows planner.
+
+The director-collaboration loop:
+
+ 1. Agent plans a multi-shot piece in plain text.
+ 2. Agent calls `generate_shot_images(shot_label, prompt, n=4)` per shot.
+ Each call returns a list of candidate PNGs saved on disk.
+ 3. UI renders the candidates as a thumbnail grid in the tool-result
+ card. User clicks the best one — that POSTs to
+ `/agent/sessions/<id>/anchors/select` which records the choice in
+ `session.tool_state["selected_anchors"]`.
+ 4. User types "render". Agent reads the selected anchors and submits
+ i2v shots with each anchor as `ref_image_path`. The video model
+ fills the motion between known frames; the look is locked.
+
+This module is the dispatch layer. Two backends ship in v1:
+
+ - **mock** — flat-colored PNGs drawn with PIL. Zero deps beyond what
+ LTX already needs. Used for testing the UX without spending API
+ credits and as a fallback when no API key is set.
+ - **bfl** — Black Forest Labs (api.bfl.ml). Async submit + poll.
+ Requires `bfl_api_key`. Models: flux-dev (cheap, 25 steps),
+ flux-pro (better, slower), flux-schnell (4 steps, fastest).
+
+Future backends (v2): mflux for fully-local Mac generation; Replicate;
+fal.ai. Slot in by adding a `_generate_<kind>()` function and a clause
+in `generate()`.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.request
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+@dataclass
+class ImageEngineConfig:
+ """How the agent generates anchor stills.
+
+ Persisted in `state/agent_image_config.json`. The HTTP `/agent/image/config`
+ GET masks `bfl_api_key` (returns `has_bfl_api_key` bool only).
+ """
+
+ kind: str = "mock" # "mock" | "bfl"
+ bfl_api_key: str = ""
+ bfl_model: str = "flux-dev" # flux-dev | flux-pro | flux-schnell
+ bfl_base_url: str = "https://api.bfl.ml/v1"
+
+ def to_public_dict(self) -> dict:
+ d = asdict(self)
+ if d.get("bfl_api_key"):
+ d["bfl_api_key"] = ""
+ d["has_bfl_api_key"] = True
+ else:
+ d["has_bfl_api_key"] = False
+ return d
+
+
+# Aspect ratios → (width, height). Flux is most stable on these.
+ASPECT_DIMS = {
+ "16:9": (1024, 576),
+ "4:3": (1024, 768),
+ "1:1": (768, 768),
+ "9:16": (576, 1024),
+ "3:4": (768, 1024),
+ "21:9": (1280, 544),
+}
+
+
+def generate(*, prompt: str, n: int, output_dir: Path,
+ aspect: str = "16:9",
+ base_seed: int | None = None,
+ config: ImageEngineConfig) -> list[dict]:
+ """Generate `n` candidate images for one shot. Saves under `output_dir`.
+
+ Returns a list of `{png_path, seed, engine, width, height}` dicts in
+ submission order.
+ """
+ output_dir = Path(output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ width, height = ASPECT_DIMS.get(aspect, ASPECT_DIMS["16:9"])
+
+ if config.kind == "mock":
+ return _generate_mock(prompt, n, width, height, output_dir, base_seed)
+ if config.kind == "bfl":
+ return _generate_bfl(prompt, n, width, height, output_dir, base_seed, config)
+ raise ValueError(f"unknown image engine kind: {config.kind!r}")
+
+
+def health_check(config: ImageEngineConfig) -> tuple[bool, str]:
+ """Light readiness probe for the configured backend."""
+ if config.kind == "mock":
+ try:
+ from PIL import Image # noqa: F401
+ return True, "mock engine ready (PIL available)"
+ except ImportError:
+ return False, "PIL not installed — pip install Pillow"
+ if config.kind == "bfl":
+ if not config.bfl_api_key:
+ return False, "BFL API key not configured"
+ return True, f"BFL configured for {config.bfl_model}"
+ return False, f"unknown engine kind: {config.kind!r}"
+
+
+# ---- Backends ---------------------------------------------------------------
+def _generate_mock(prompt: str, n: int, width: int, height: int,
+ output_dir: Path, base_seed: int | None) -> list[dict]:
+ """Draw `n` distinguishable colored PNGs locally. Each carries a label
+ with the candidate index + first ~60 chars of the prompt so the user
+ can verify the UX flow without confusing identical images.
+ """
+ try:
+ from PIL import Image, ImageDraw, ImageFont
+ except ImportError as e:
+ raise RuntimeError(
+ "PIL not available — install Pillow to use the mock engine, "
+ "or configure a real backend in Settings → Image generation."
+ ) from e
+
+ palette = [
+ (220, 100, 80), (80, 150, 220), (220, 200, 80), (130, 220, 130),
+ (180, 100, 200), (240, 160, 100), (100, 200, 200), (180, 180, 180),
+ ]
+ font = None
+ for path in [
+ "/System/Library/Fonts/Supplemental/Arial.ttf",
+ "/System/Library/Fonts/Helvetica.ttc",
+ ]:
+ try:
+ font = ImageFont.truetype(path, max(18, height // 28))
+ break
+ except OSError:
+ continue
+ if font is None:
+ font = ImageFont.load_default()
+
+ results = []
+ for i in range(n):
+ color = palette[i % len(palette)]
+ img = Image.new("RGB", (width, height), color)
+ draw = ImageDraw.Draw(img)
+
+ # Subtle gradient stripe on top so candidates aren't perfectly flat
+ for y in range(0, height // 4):
+ tint = 1 - (y / (height // 4)) * 0.4
+ stripe = tuple(min(255, int(c * tint + 20)) for c in color)
+ draw.line([(0, y), (width, y)], fill=stripe)
+
+ title = f"Mock candidate {i + 1}"
+ body = (prompt or "")[:120]
+ draw.text((24, 24), title, fill=(255, 255, 255), font=font)
+ # Word-wrap the body
+ words = body.split()
+ line = ""
+ y = 24 + (font.size if hasattr(font, "size") else 18) + 12
+ for w in words:
+ test = (line + " " + w).strip()
+ if len(test) > 36:
+ draw.text((24, y), line, fill=(255, 255, 255), font=font)
+ y += (font.size if hasattr(font, "size") else 18) + 6
+ line = w
+ else:
+ line = test
+ if line:
+ draw.text((24, y), line, fill=(255, 255, 255), font=font)
+
+ seed = (base_seed if base_seed is not None else 1000) + i
+ out_path = output_dir / f"cand_{i:02d}_mock.png"
+ img.save(out_path, "PNG")
+ results.append({
+ "png_path": str(out_path), "seed": seed, "engine": "mock",
+ "width": width, "height": height,
+ })
+ return results
+
+
+def _generate_bfl(prompt: str, n: int, width: int, height: int,
+ output_dir: Path, base_seed: int | None,
+ config: ImageEngineConfig) -> list[dict]:
+ """Submit + poll Black Forest Labs API for each candidate.
+
+ BFL's API is async-poll. POST `/{model}` returns `{id}`; GET
+ `/get_result?id=...` returns `{status, result}` where status flips
+ to "Ready" with `result.sample` (a download URL).
+ """
+ if not config.bfl_api_key:
+ raise RuntimeError(
+ "BFL API key not configured. Add one under Settings → Image "
+ "generation, or pick the mock engine to test the workflow."
+ )
+ headers = {"Content-Type": "application/json", "X-Key": config.bfl_api_key}
+ poll_headers = {"X-Key": config.bfl_api_key}
+ submit_url = f"{config.bfl_base_url.rstrip('/')}/{config.bfl_model}"
+ poll_url_base = f"{config.bfl_base_url.rstrip('/')}/get_result"
+
+ results = []
+ for i in range(n):
+ body = {
+ "prompt": prompt,
+ "width": width, "height": height,
+ "prompt_upsampling": False,
+ "safety_tolerance": 2,
+ }
+ if base_seed is not None:
+ body["seed"] = base_seed + i
+
+ # Submit
+ req = urllib.request.Request(
+ submit_url,
+ data=json.dumps(body).encode("utf-8"),
+ headers=headers, method="POST",
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=30) as r:
+ resp = json.loads(r.read())
+ except urllib.error.HTTPError as e:
+ detail = ""
+ try:
+ detail = e.read().decode("utf-8", errors="replace")
+ except Exception: # noqa: BLE001
+ pass
+ raise RuntimeError(
+ f"BFL submit failed (HTTP {e.code} {e.reason}): {detail[:400]}"
+ ) from e
+ except urllib.error.URLError as e:
+ raise RuntimeError(f"BFL unreachable: {e.reason}") from e
+
+ task_id = resp.get("id") or resp.get("task_id")
+ if not task_id:
+ raise RuntimeError(f"BFL submit returned no task id: {resp!r}")
+
+ # Poll
+ deadline = time.time() + 180
+ sample_url = None
+ seed_used = base_seed + i if base_seed is not None else 0
+ while time.time() < deadline:
+ time.sleep(1.5)
+ poll_url = f"{poll_url_base}?id={task_id}"
+ try:
+ with urllib.request.urlopen(
+ urllib.request.Request(poll_url, headers=poll_headers),
+ timeout=20,
+ ) as r:
+ rs = json.loads(r.read())
+ except urllib.error.URLError:
+ continue
+ status = rs.get("status", "")
+ if status == "Ready":
+ inner = rs.get("result") or {}
+ sample_url = inner.get("sample")
+ if "seed" in inner:
+ seed_used = inner["seed"]
+ break
+ if status in ("Error", "Failed", "Content Moderated"):
+ raise RuntimeError(f"BFL task {task_id} {status}: {rs!r}")
+ if sample_url is None:
+ raise RuntimeError(f"BFL task {task_id} timed out after 180s")
+
+ # Download
+ try:
+ with urllib.request.urlopen(sample_url, timeout=60) as r:
+ data = r.read()
+ except urllib.error.URLError as e:
+ raise RuntimeError(f"BFL sample download failed: {e.reason}") from e
+
+ out_path = output_dir / f"cand_{i:02d}_bfl.png"
+ out_path.write_bytes(data)
+ results.append({
+ "png_path": str(out_path), "seed": seed_used, "engine": "bfl",
+ "width": width, "height": height,
+ })
+ return results
diff --git a/agent/prompts.py b/agent/prompts.py
index 25753fc..e56305d 100644
--- a/agent/prompts.py
+++ b/agent/prompts.py
@@ -252,9 +252,9 @@ If you want a documentary feel, **never use these words.** Even
break identity. If you need a wide shot, accept the face will be
unrecognizable in those frames.
- **Multi-shot character drift.** Same prompt + new seed = new
- person. (See "T2V is the primary mode" below — don't auto-suggest
- i2v anchoring; the user is currently evaluating image-to-video
- models. Treat character drift as a creative constraint to embrace.)
+ person. The 3-phase anchor workflow above largely solves this —
+ one Flux still per shot, the user picks, every video starts from
+ that exact frame.
- **On-screen text** (signs, screens, badges, name tags). Almost
always gibberish. Don't put text in the prompt.
- **Camera moves.** "Pulls back to reveal", "cuts to", "transitions"
@@ -280,22 +280,47 @@ that runs.
prompt point at exactly one human face the camera will read?* If no,
rewrite. If yes, ship.
-# T2V is the primary mode — i2v is on hold
-
-Always default to `mode: "t2v"`. The user is currently evaluating
-image-to-video models for the panel and has paused i2v workflows. Do
-NOT suggest i2v as a fix for character continuity, do NOT chain
-shots via `extract_frame` → i2v anchoring, do NOT propose multi-
-keyframe interpolation as a workflow unless the user explicitly asks.
-
-If a script benefits from cross-shot character continuity, treat it
-as a known limitation and note it in your plan ("character drift
-between shots is unavoidable in t2v; same actor description on every
-shot helps but won't be perfect"). Don't reach for i2v as a band-aid.
-
-`extend` mode is available but expensive (~16 min/+3s on Comfortable)
-and audio chains poorly across extensions. Use only when explicitly
-asked.
+# Modes — when to use which
+
+**Default workflow uses `i2v` with curated anchors** (Phase A/B/C
+above). The user picks one Flux-generated still per shot; you pass
+that PNG as `ref_image_path` and submit the video render with
+`mode: "i2v"`. This is dramatically more reliable than blind t2v
+because the look is locked at frame 0 by a still the user actually
+chose.
+
+`mode: "t2v"` only when the user explicitly opts out of the anchor
+workflow ("just t2v, skip the picker"), or the shot is so abstract
+that no still serves as a useful anchor (a generic ambient cutaway).
+
+`mode: "keyframe"` (FFLF) — multi-frame anchored interpolation. Useful
+when the user wants both start AND end frames pinned. Tier-clamped to
+{max_dim_kf}px on Comfortable. Requires Q8 weights.
+
+`mode: "extend"` — append seconds to an existing clip. Slow
+(~16 min/+3s on Comfortable) and audio chains poorly across
+extensions. Use only when explicitly asked.
+
+# Writing prompts FOR ANCHOR STILLS (Phase B)
+
+A still prompt is NOT a video prompt. Differences:
+
+- **No dialogue.** Audio comes with the video render in Phase C, not
+ the still. Don't put `'spoken text'` in image prompts; describe
+ the *expression* the speaker has during that line instead.
+- **Composition + light + character.** "Medium close-up of a tired
+ developer in a white restraint vest, fluorescent overhead light,
+ pale tired face, slight handheld feel."
+- **Specific look words.** "Documentary realism, full-frame 16:9,
+ shallow depth of field, soft natural lighting, realistic face,
+ no letterbox, no text, no logos."
+- **No camera moves** in image prompts (it's a still). Save "slow
+ push in" / "tracking" for the video prompt in Phase C.
+- **60-120 words.** Tighter than video prompts; the still has no
+ temporal beats to fill.
+
+The video prompt in Phase C carries the dialogue, the action beats,
+and any camera motion. The anchor still controls the look.
# Director's craft — translating script to LTX prompts
diff --git a/agent/tools.py b/agent/tools.py
index 00649ef..d8c6744 100644
--- a/agent/tools.py
+++ b/agent/tools.py
@@ -648,6 +648,106 @@ def _write_session_manifest(args: dict, ops: PanelOps, session: dict) -> dict:
}
+@tool("generate_shot_images")
+def _generate_shot_images(args: dict, ops: PanelOps, session: dict) -> dict:
+ """Generate candidate anchor stills for ONE shot.
+
+ Used in the director-collaboration workflow:
+ Phase A: agent calls this for each shot — 4 candidates appear in chat.
+ Phase B: user picks the best one (UI thumbnail click).
+ Phase C: agent submits the i2v video render with the chosen anchor.
+
+ Args:
+ shot_label: str (required) — the same label you'll pass to submit_shot
+ in phase C, so anchors and renders match.
+ prompt: str (required) — visual description for the still. Focus on
+ framing, lighting, character, expression,
+ setting. Do NOT include dialogue (audio comes
+ with the video render, not the still). Keep
+ it 60-120 words.
+ n: int = 4 — number of candidates (1-8). 4 is the sweet spot.
+ aspect: "16:9" | "9:16" | "1:1" | "4:3" | "21:9" | "3:4" = "16:9"
+ seed_base: int = -1 — when >= 0, candidates use seed_base + i so the
+ run is reproducible. Use -1 for random seeds.
+
+ Returns: {shot_label, prompt, candidates: [{png_path, seed, ...}, ...],
+ elapsed_seconds, engine}.
+
+ The UI renders the candidates as a clickable thumbnail grid in the
+ tool-result card. When the user clicks one, that selection is recorded
+ in `session.tool_state["selected_anchors"][shot_label]`. The agent
+ can read the selections later via `get_selected_anchors`.
+ """
+ from agent import image_engine as _image_engine
+
+ label = (args.get("shot_label") or "").strip() or "untitled"
+ prompt = (args.get("prompt") or "").strip()
+ if not prompt:
+ raise _ToolValidationError("prompt is required")
+ n = max(1, min(8, int(args.get("n", 4))))
+ aspect = args.get("aspect", "16:9")
+ seed_base = int(args.get("seed_base", -1))
+
+ # Image engine config travels in PanelOps.capabilities so tools.py
+ # stays free of panel imports.
+ cfg_dict = (ops.capabilities or {}).get("image_engine_config") or {}
+ cfg = _image_engine.ImageEngineConfig(**cfg_dict)
+
+ safe_label = re.sub(r"[^a-z0-9_-]", "_", label.lower())[:40] or "untitled"
+ sid = session.get("session_id") or "session"
+ out_dir = ops.uploads_dir / "agentflow" / sid / safe_label
+
+ t0 = time.time()
+ candidates = _image_engine.generate(
+ prompt=prompt, n=n, aspect=aspect,
+ output_dir=out_dir,
+ base_seed=(seed_base if seed_base >= 0 else None),
+ config=cfg,
+ )
+ elapsed = round(time.time() - t0, 2)
+
+ # Track in session state.
+ session.setdefault("anchor_candidates", {})
+ session["anchor_candidates"][label] = {
+ "prompt": prompt,
+ "candidates": candidates,
+ "generated_at": time.time(),
+ }
+
+ return {
+ "shot_label": label,
+ "prompt": prompt,
+ "candidates": candidates,
+ "engine": cfg.kind,
+ "elapsed_seconds": elapsed,
+ }
+
+
+@tool("get_selected_anchors")
+def _get_selected_anchors(args: dict, ops: PanelOps, session: dict) -> dict:
+ """Return the user's anchor selections so far.
+
+ Between phase B (generate) and phase C (render), the user clicks
+ thumbnails in the UI; those clicks are recorded in
+ `session.tool_state["selected_anchors"]`. Call this at the START of
+ phase C (when the user types "render") to discover which still goes
+ with which shot, then submit each video with `mode: "i2v"` and the
+ matching `ref_image_path`.
+
+ Returns: {selected_anchors, missing, total_candidates, total_selected}.
+ `missing` is a list of shot_labels with candidates but no user pick.
+ """
+ selected = session.get("selected_anchors", {})
+ candidates = session.get("anchor_candidates", {})
+ missing = [lbl for lbl in candidates.keys() if lbl not in selected]
+ return {
+ "selected_anchors": selected,
+ "missing": missing,
+ "total_candidates": len(candidates),
+ "total_selected": len(selected),
+ }
+
+
@tool("upload_image")
def _upload_image(args: dict, ops: PanelOps, session: dict) -> dict:
"""Resolve an image path the user attached to the chat.
diff --git a/mlx_ltx_panel.py b/mlx_ltx_panel.py
index adc27ad..e68dff7 100644
--- a/mlx_ltx_panel.py
+++ b/mlx_ltx_panel.py
@@ -34,6 +34,7 @@ from urllib.parse import parse_qs, quote, urlparse
# this panel); we wire it up via a PanelOps callback object built at the
# end of this module just before the HTTP handler.
from agent import engine as agent_engine
+from agent import image_engine as agent_image_engine
from agent import local_server as agent_local_server
from agent import prompts as agent_prompts
from agent import runtime as agent_runtime
@@ -3376,8 +3377,10 @@ def worker_loop() -> None:
# state/ (see install.js / CLAUDE.md §7 "fs.link"). No extra plumbing needed.
AGENT_CONFIG_PATH = STATE_DIR / "agent_config.json"
+AGENT_IMAGE_CONFIG_PATH = STATE_DIR / "agent_image_config.json"
AGENT_LOCK = threading.RLock()
_AGENT_CONFIG_CACHE: agent_engine.EngineConfig | None = None
+_AGENT_IMAGE_CONFIG_CACHE: agent_image_engine.ImageEngineConfig | None = None
def _default_agent_engine_config() -> agent_engine.EngineConfig:
@@ -3447,12 +3450,52 @@ def _save_agent_config(updates: dict) -> agent_engine.EngineConfig:
return cfg
+def _load_agent_image_config() -> agent_image_engine.ImageEngineConfig:
+ """Read state/agent_image_config.json (creating defaults on first call)."""
+ global _AGENT_IMAGE_CONFIG_CACHE
+ with AGENT_LOCK:
+ if _AGENT_IMAGE_CONFIG_CACHE is not None:
+ return _AGENT_IMAGE_CONFIG_CACHE
+ if AGENT_IMAGE_CONFIG_PATH.is_file():
+ try:
+ data = json.loads(AGENT_IMAGE_CONFIG_PATH.read_text(encoding="utf-8"))
+ cfg = agent_image_engine.ImageEngineConfig(**data)
+ except (OSError, json.JSONDecodeError, TypeError) as e:
+ push(f"agent: invalid agent_image_config.json ({e}); using mock defaults")
+ cfg = agent_image_engine.ImageEngineConfig()
+ else:
+ cfg = agent_image_engine.ImageEngineConfig()
+ _AGENT_IMAGE_CONFIG_CACHE = cfg
+ return cfg
+
+
+def _save_agent_image_config(updates: dict) -> agent_image_engine.ImageEngineConfig:
+ global _AGENT_IMAGE_CONFIG_CACHE
+ with AGENT_LOCK:
+ cur = _load_agent_image_config()
+ merged = {**cur.__dict__}
+ for k, v in (updates or {}).items():
+ if k not in merged:
+ continue
+ if k == "bfl_api_key" and v == "":
+ continue # blank means "leave as-is"
+ merged[k] = v
+ cfg = agent_image_engine.ImageEngineConfig(**merged)
+ STATE_DIR.mkdir(parents=True, exist_ok=True)
+ atomic_write_text(AGENT_IMAGE_CONFIG_PATH, json.dumps(cfg.__dict__, indent=2))
+ _AGENT_IMAGE_CONFIG_CACHE = cfg
+ return cfg
+
+
def _agent_capabilities() -> dict:
"""Hardware-tier snapshot the agent uses to clamp its plan.
Mirrors the SYSTEM_CAPS table in tier-friendly form so the system
- prompt can show the user what this Mac can actually render.
+ prompt can show the user what this Mac can actually render. Also
+ carries the current image-engine config so the `generate_shot_images`
+ tool can dispatch without circular imports.
"""
+ img_cfg = _load_agent_image_config()
return {
"tier": SYSTEM_TIER,
"tier_label": SYSTEM_CAPS.get("label", SYSTEM_TIER),
@@ -3463,6 +3506,7 @@ def _agent_capabilities() -> dict:
"allows_q8": SYSTEM_CAPS.get("allows_q8", False),
"allows_keyframe": SYSTEM_CAPS.get("allows_keyframe", False),
"allows_extend": SYSTEM_CAPS.get("allows_extend", False),
+ "image_engine_config": dict(img_cfg.__dict__),
}
@@ -4055,6 +4099,16 @@ class Handler(BaseHTTPRequestHandler):
})
return
+ if parsed.path == "/agent/image/config":
+ cfg = _load_agent_image_config()
+ ok, msg = agent_image_engine.health_check(cfg)
+ self._json({
+ "image_engine": cfg.to_public_dict(),
+ "ok": ok,
+ "message": msg,
+ })
+ return
+
self.send_error(404)
def do_POST(self) -> None:
@@ -4600,6 +4654,66 @@ class Handler(BaseHTTPRequestHandler):
self._json({"ok": ok})
return
+ # Image-engine config (pluggable: mock | bfl).
+ if path == "/agent/image/config":
+ try:
+ payload = (json.loads(body or "{}")
+ if ctype.startswith("application/json")
+ else {k: v[0] if v else "" for k, v in form.items()})
+ except json.JSONDecodeError as e:
+ self._json({"error": f"bad JSON: {e}"}, 400); return
+ cfg = _save_agent_image_config(payload)
+ push(f"agent: image engine updated to {cfg.kind}"
+ + (f" ({cfg.bfl_model})" if cfg.kind == "bfl" else ""))
+ ok, msg = agent_image_engine.health_check(cfg)
+ self._json({"ok": True, "image_engine": cfg.to_public_dict(),
+ "health_ok": ok, "health_message": msg})
+ return
+
+ # Anchor selection — user clicked a thumbnail in the chat.
+ # Records {shot_label: candidate_dict} in the session's tool_state.
+ if path.startswith("/agent/sessions/") and path.endswith("/anchors/select"):
+ sid = path[len("/agent/sessions/"):-len("/anchors/select")]
+ sess = agent_runtime.load_session(sid, STATE_DIR)
+ if sess is None:
+ self._json({"error": "session not found"}, 404); return
+ try:
+ payload = (json.loads(body or "{}")
+ if ctype.startswith("application/json")
+ else {k: v[0] if v else "" for k, v in form.items()})
+ except json.JSONDecodeError as e:
+ self._json({"error": f"bad JSON: {e}"}, 400); return
+ label = (payload.get("shot_label") or "").strip()
+ png_path = (payload.get("png_path") or "").strip()
+ if not label or not png_path:
+ self._json({"error": "shot_label and png_path are required"}, 400); return
+ # Validate path is real and inside our uploads dir (no traversal).
+ try:
+ p = Path(png_path).resolve()
+ _ = p.relative_to(UPLOADS.resolve())
+ if not p.is_file():
+ raise ValueError("not a file")
+ except (ValueError, OSError) as e:
+ self._json({"error": f"invalid png_path: {e}"}, 400); return
+ # Find the matching candidate by path (so we keep seed/engine info)
+ candidates = ((sess.tool_state.get("anchor_candidates") or {})
+ .get(label, {}).get("candidates")) or []
+ picked = next((c for c in candidates if c.get("png_path") == str(p)), None)
+ if picked is None:
+ # Path didn't match a known candidate — accept anyway with a
+ # synthetic record. Useful when the user uploads their own.
+ picked = {"png_path": str(p), "engine": "user", "seed": -1}
+ sess.tool_state.setdefault("selected_anchors", {})[label] = picked
+ agent_runtime.save_session(sess, STATE_DIR)
+ push(f"agent: anchor picked for {label}: {p.name}")
+ self._json({
+ "ok": True,
+ "shot_label": label,
+ "selected": picked,
+ "all_selected": sess.tool_state.get("selected_anchors", {}),
+ })
+ return
+
self.send_error(404)
@@ -6686,6 +6800,100 @@ HTML = r"""<!doctype html>
color: var(--text);
}
+ /* ---- Anchor candidate grid (Phase B of the director workflow) ---- */
+ .anchor-grid-wrap {
+ padding: 12px 14px 14px;
+ border-top: 1px solid var(--border);
+ }
+ .anchor-grid-meta {
+ font-size: 11px; color: var(--muted);
+ margin-bottom: 8px;
+ display: flex; align-items: center; gap: 8px;
+ }
+ .anchor-grid-meta .label-pill {
+ padding: 2px 8px; border-radius: 999px;
+ background: var(--accent-dim); color: var(--accent-bright);
+ font-weight: 600; font-size: 10px;
+ letter-spacing: 0.4px; text-transform: uppercase;
+ }
+ .anchor-grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
+ gap: 10px;
+ }
+ .anchor-cell {
+ position: relative;
+ aspect-ratio: 16/9;
+ border-radius: 8px;
+ overflow: hidden;
+ cursor: pointer;
+ border: 2px solid transparent;
+ background: var(--bg);
+ padding: 0;
+ transition: border-color 0.15s, transform 0.15s;
+ }
+ .anchor-cell:hover {
+ border-color: var(--accent);
+ transform: scale(1.02);
+ }
+ .anchor-cell.selected {
+ border-color: #2ea043;
+ box-shadow: 0 0 0 3px rgba(46,160,67,0.25);
+ }
+ .anchor-cell img {
+ width: 100%; height: 100%;
+ object-fit: cover;
+ display: block;
+ }
+ .anchor-cell .check {
+ position: absolute; top: 6px; right: 6px;
+ width: 22px; height: 22px;
+ border-radius: 50%;
+ background: var(--bg-2);
+ color: var(--muted);
+ display: flex; align-items: center; justify-content: center;
+ font-size: 12px;
+ font-weight: 800;
+ border: 1px solid var(--border);
+ transition: background 0.15s, color 0.15s;
+ }
+ .anchor-cell.selected .check {
+ background: #2ea043;
+ color: white;
+ border-color: #2ea043;
+ }
+ .anchor-cell .seed {
+ position: absolute; bottom: 4px; left: 4px;
+ font-size: 9px;
+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+ padding: 2px 6px;
+ background: rgba(0,0,0,0.55);
+ color: white;
+ border-radius: 4px;
+ letter-spacing: 0.3px;
+ }
+ .anchor-cell .engine-tag {
+ position: absolute; bottom: 4px; right: 4px;
+ font-size: 9px;
+ padding: 2px 5px;
+ background: rgba(0,0,0,0.55);
+ color: white;
+ border-radius: 4px;
+ text-transform: uppercase;
+ letter-spacing: 0.4px;
+ }
+ .anchor-prompt {
+ font-size: 11px; color: var(--muted);
+ line-height: 1.5;
+ margin-bottom: 10px;
+ padding: 8px 10px;
+ background: var(--bg);
+ border-radius: 6px;
+ border: 1px solid var(--border);
+ font-style: italic;
+ max-height: 80px; overflow-y: auto;
+ }
+
/* ---- Typing indicator ---- */
.agent-typing-row {
display: flex; gap: 12px;
@@ -7846,9 +8054,45 @@ HTML = r"""<!doctype html>
</div>
</div>
+ <hr style="border:none;border-top:1px solid var(--border);margin:18px 0 12px">
+
+ <h2 style="font-size:14px">Image generation
+ <span class="pill" id="agentImagePill" style="font-size:10px;padding:3px 8px;border-radius:999px;background:var(--bg-2);color:var(--muted);border:1px solid var(--border)">…</span>
+ </h2>
+ <div class="subtitle" style="margin-bottom:12px">
+ For the director-collaboration workflow: agent generates 4 candidate stills per
+ shot, you pick the best one, then i2v anchors lock the look before each render.
+ </div>
+
+ <div class="field">
+ <label>Backend</label>
+ <select id="agentImageKind" onchange="agentImageKindChanged()">
+ <option value="mock">Mock (PIL placeholders, free, instant — for testing the flow)</option>
+ <option value="bfl">Black Forest Labs API (real Flux, ~$0.025/img on flux-dev)</option>
+ </select>
+ </div>
+
+ <div class="field" id="agentBflModelField" style="display:none">
+ <label>BFL model</label>
+ <select id="agentBflModel">
+ <option value="flux-dev">flux-dev — 25 steps, ~$0.025 per image (recommended)</option>
+ <option value="flux-pro">flux-pro — premium quality, ~$0.05 per image</option>
+ <option value="flux-pro-1.1">flux-pro-1.1 — newer pro, ~$0.04 per image</option>
+ <option value="flux-schnell">flux-schnell — 4 steps, ~$0.003 per image (fastest)</option>
+ </select>
+ </div>
+
+ <div class="field" id="agentBflKeyField" style="display:none">
+ <label>BFL API key</label>
+ <input type="password" id="agentBflKey" placeholder="(leave blank to keep saved)">
+ <div style="margin-top:6px;font-size:11px;color:var(--muted)">
+ Get one at <code>https://api.bfl.ml</code> · stored in <code>state/agent_image_config.json</code>, only sent as the <code>X-Key</code> header.
+ </div>
+ </div>
+
<div class="actions">
<button type="button" class="ghost" onclick="closeAgentSettings()">Cancel</button>
- <button type="button" class="save" onclick="agentSaveSettings()">Save engine config</button>
+ <button type="button" class="save" onclick="agentSaveSettings()">Save settings</button>
</div>
</div>
</div>
@@ -10862,7 +11106,9 @@ window.AGENT = {
config: null,
busy: false,
models: [],
- sessions: [], // cached list for the dropdown
+ sessions: [], // cached list for the dropdown
+ selectedAnchors: {}, // {shot_label: candidate_obj} — synced from session.tool_state
+ imageConfig: null, // {kind, has_bfl_api_key, ...}
};
function workflowSwitch(name) {
@@ -10973,6 +11219,7 @@ async function agentLoadSession(sid) {
if (!r.ok) {
try { localStorage.removeItem('phos_agent_session'); } catch(e) {}
window.AGENT.sessionId = null;
+ window.AGENT.selectedAnchors = {};
agentSetSessionTitle('New chat', null);
agentRender([]);
return;
@@ -10980,6 +11227,7 @@ async function agentLoadSession(sid) {
const j = await r.json();
window.AGENT.sessionId = sid;
const sess = j.session || {};
+ window.AGENT.selectedAnchors = (sess.tool_state || {}).selected_anchors || {};
agentSetSessionTitle(sess.title || 'Untitled', sid);
agentRender(j.rendered_messages || []);
} catch (e) { console.error('agentLoadSession', e); }
@@ -11263,18 +11511,128 @@ function renderToolResultCard(result) {
<span class="summary">${escapeHtml(summary)}</span>
<span class="chevron">›</span>
`;
+
const body = document.createElement('div');
body.className = 'body';
const pre = document.createElement('pre');
try { pre.textContent = JSON.stringify(ok ? inner : result, null, 2); }
catch(e) { pre.textContent = String(result); }
body.appendChild(pre);
+
card.appendChild(head);
card.appendChild(body);
head.addEventListener('click', () => card.classList.toggle('open'));
+
+ // Phase B of the director workflow: when the result carries
+ // `candidates`, render an interactive thumbnail grid below the head.
+ // The card stays expanded by default so the user can immediately pick.
+ if (ok && inner && Array.isArray(inner.candidates) && inner.candidates.length > 0) {
+ card.classList.add('open'); // open by default
+ const grid = renderAnchorGrid(inner);
+ card.appendChild(grid);
+ }
return card;
}
+function renderAnchorGrid(payload) {
+ const wrap = document.createElement('div');
+ wrap.className = 'anchor-grid-wrap';
+ const label = payload.shot_label || 'shot';
+ const prompt = payload.prompt || '';
+ const engine = payload.engine || '';
+
+ const meta = document.createElement('div');
+ meta.className = 'anchor-grid-meta';
+ meta.innerHTML = `
+ <span class="label-pill">${escapeHtml(label)}</span>
+ <span>${payload.candidates.length} candidates · ${escapeHtml(engine)}</span>
+ <span style="flex:1"></span>
+ <span style="font-size:10px">click to pick</span>
+ `;
+ wrap.appendChild(meta);
+
+ if (prompt) {
+ const p = document.createElement('div');
+ p.className = 'anchor-prompt';
+ p.textContent = prompt;
+ wrap.appendChild(p);
+ }
+
+ const grid = document.createElement('div');
+ grid.className = 'anchor-grid';
+
+ const selected = window.AGENT.selectedAnchors || {};
+ const currentPick = (selected[label] || {}).png_path;
+
+ for (const cand of payload.candidates) {
+ const cell = document.createElement('button');
+ cell.type = 'button';
+ cell.className = 'anchor-cell' + (cand.png_path === currentPick ? ' selected' : '');
+ cell.dataset.shotLabel = label;
+ cell.dataset.pngPath = cand.png_path;
+
+ const img = document.createElement('img');
+ img.src = '/image?path=' + encodeURIComponent(cand.png_path);
+ img.alt = label + ' candidate';
+ img.loading = 'lazy';
+ cell.appendChild(img);
+
+ const check = document.createElement('span');
+ check.className = 'check';
+ check.textContent = '✓';
+ cell.appendChild(check);
+
+ if (typeof cand.seed === 'number' && cand.seed >= 0) {
+ const s = document.createElement('span');
+ s.className = 'seed';
+ s.textContent = 'seed ' + cand.seed;
+ cell.appendChild(s);
+ }
+ if (cand.engine) {
+ const e = document.createElement('span');
+ e.className = 'engine-tag';
+ e.textContent = cand.engine;
+ cell.appendChild(e);
+ }
+
+ cell.addEventListener('click', () => agentPickAnchor(label, cand, grid));
+ grid.appendChild(cell);
+ }
+
+ wrap.appendChild(grid);
+ return wrap;
+}
+
+async function agentPickAnchor(label, cand, gridEl) {
+ if (!window.AGENT.sessionId) return;
+ // Optimistic UI: mark this cell selected, deselect siblings
+ if (gridEl) {
+ gridEl.querySelectorAll('.anchor-cell').forEach(c => c.classList.remove('selected'));
+ const me = gridEl.querySelector(`[data-png-path="${CSS.escape(cand.png_path)}"]`);
+ if (me) me.classList.add('selected');
+ }
+ window.AGENT.selectedAnchors[label] = cand;
+
+ try {
+ const r = await fetch(
+ '/agent/sessions/' + encodeURIComponent(window.AGENT.sessionId) + '/anchors/select',
+ {
+ method: 'POST',
+ headers: {'Content-Type': 'application/json'},
+ body: JSON.stringify({shot_label: label, png_path: cand.png_path}),
+ }
+ );
+ const j = await r.json();
+ if (!r.ok || j.error) {
+ alert('Could not save selection: ' + (j.error || ('HTTP ' + r.status)));
+ return;
+ }
+ window.AGENT.selectedAnchors = j.all_selected || window.AGENT.selectedAnchors;
+ } catch (e) {
+ console.error('agentPickAnchor', e);
+ }
+}
+
function summarizeToolCall(call) {
const t = call.tool || '';
const a = call.args || {};
@@ -11489,12 +11847,24 @@ document.addEventListener('DOMContentLoaded', () => {
});
// ---- Engine settings drawer -----------------------------------------------
+async function agentRefreshImageConfig() {
+ try {
+ const r = await fetch('/agent/image/config');
+ const j = await r.json();
+ window.AGENT.imageConfig = j;
+ return j;
+ } catch (e) {
+ return null;
+ }
+}
+
function openAgentSettings() {
- agentRefreshConfig().then(() => {
+ Promise.all([agentRefreshConfig(), agentRefreshImageConfig()]).then(() => {
const modal = document.getElementById('agentSettingsModal');
if (!modal) return;
const cfg = (window.AGENT.config && window.AGENT.config.engine) || {};
const local = (window.AGENT.config && window.AGENT.config.local_server) || {};
+ const imgCfg = (window.AGENT.imageConfig && window.AGENT.imageConfig.image_engine) || {};
document.getElementById('agentKind').value = cfg.kind || 'phosphene_local';
document.getElementById('agentBaseUrl').value = cfg.base_url || '';
document.getElementById('agentRemoteModel').value = cfg.kind === 'custom' ? (cfg.model || '') : '';
@@ -11504,6 +11874,22 @@ function openAgentSettings() {
document.getElementById('agentTemp').value = cfg.temperature ?? 0.4;
document.getElementById('agentMaxTokens').value = cfg.max_tokens ?? 3072;
+ // Image-engine fields
+ document.getElementById('agentImageKind').value = imgCfg.kind || 'mock';
+ document.getElementById('agentBflModel').value = imgCfg.bfl_model || 'flux-dev';
+ document.getElementById('agentBflKey').value = '';
+ document.getElementById('agentBflKey').placeholder =
+ imgCfg.has_bfl_api_key ? '(saved key — leave blank to keep)' : 'Paste BFL API key';
+ const imgPill = document.getElementById('agentImagePill');
+ if (imgPill) {
+ const okMsg = window.AGENT.imageConfig || {};
+ imgPill.textContent = okMsg.ok === false ? 'needs config' : (imgCfg.kind || 'mock');
+ imgPill.style.color = okMsg.ok === false ? '#f49a9e' : '#9be7a4';
+ imgPill.style.borderColor = okMsg.ok === false ? 'rgba(207,34,46,0.5)' : 'rgba(46,160,67,0.5)';
+ imgPill.title = okMsg.message || '';
+ }
+ agentImageKindChanged();
+
const sel = document.getElementById('agentLocalModel');
sel.innerHTML = '';
const models = (window.AGENT.config && window.AGENT.config.available_models) || [];
@@ -11540,6 +11926,12 @@ function agentKindChanged() {
document.getElementById('agentRemoteModelField').style.display = kind === 'custom' ? '' : 'none';
}
+function agentImageKindChanged() {
+ const kind = document.getElementById('agentImageKind').value;
+ document.getElementById('agentBflModelField').style.display = kind === 'bfl' ? '' : 'none';
+ document.getElementById('agentBflKeyField').style.display = kind === 'bfl' ? '' : 'none';
+}
+
function agentLocalRefreshRow(local) {
const pill = document.getElementById('agentLocalPill');
const detail = document.getElementById('agentLocalDetail');
@@ -11608,8 +12000,31 @@ async function agentSaveSettings() {
alert('Could not save: ' + (j.error || ('HTTP ' + r.status)));
return;
}
+
+ // Save image-engine config too (separate file).
+ const imgPayload = {
+ kind: document.getElementById('agentImageKind').value,
+ bfl_model: document.getElementById('agentBflModel').value,
+ };
+ const bk = (document.getElementById('agentBflKey').value || '').trim();
+ if (bk) imgPayload.bfl_api_key = bk;
+ try {
+ const ir = await fetch('/agent/image/config', {
+ method: 'POST',
+ headers: {'Content-Type': 'application/json'},
+ body: JSON.stringify(imgPayload),
+ });
+ const ij = await ir.json();
+ if (!ir.ok || ij.error) {
+ alert('Image config could not be saved: ' + (ij.error || ('HTTP ' + ir.status)));
+ }
+ } catch(e) {
+ alert('Image config save failed: ' + e);
+ }
+
closeAgentSettings();
await agentRefreshConfig();
+ await agentRefreshImageConfig();
}
// Initial workflow tab restore from localStorage.