Skip to content

The agent that grows with you

fix(image_gen,video_gen): poll the FAL queue every 2 s instead of every 0.1 s (#141368)

Tekniumbc3cbd50dba3
diff --git a/plugins/video_gen/fal/__init__.py b/plugins/video_gen/fal/__init__.py
index 8c31db7029c..66e602db4fb 100644
--- a/plugins/video_gen/fal/__init__.py
+++ b/plugins/video_gen/fal/__init__.py
@@ -268,7 +268,7 @@ def _get_managed_fal_video_client(managed_gateway):
def _submit_fal_video_request(endpoint: str, arguments: dict[str, Any]):
- """Submit via direct credentials or the managed queue gateway; ``.get()`` blocks."""
+ """Submit via direct credentials or the managed queue gateway; ``wait_fal_queue_result`` blocks."""
client = _load_fal_client()
headers = {"x-idempotency-key": str(uuid.uuid4())}
managed_gateway = _resolve_managed_fal_video_gateway()
@@ -309,7 +309,8 @@ def _upscale_video(video_url: str, source_request_id: Optional[str] = None) -> O
if not source_request_id:
raise RuntimeError("Managed SeedVR upscale requires the source FAL request id")
arguments["source_request_id"] = source_request_id
- result = _submit_fal_video_request(UPSCALER_ENDPOINT, arguments).get()
+ from tools.fal_common import wait_fal_queue_result
+ result = wait_fal_queue_result(_submit_fal_video_request(UPSCALER_ENDPOINT, arguments))
except Exception as exc:
logger.warning("Video upscale failed: %s", exc)
return None
@@ -402,9 +403,10 @@ class FALVideoGenProvider(VideoGenProvider):
payload = _build_payload(family, prompt=prompt, image_url=image_url_norm, duration=duration, aspect_ratio=aspect_ratio,
resolution=resolution, negative_prompt=negative_prompt, audio=audio, seed=seed)
try:
+ from tools.fal_common import wait_fal_queue_result
handle = _submit_fal_video_request(endpoint, payload)
source_request_id = getattr(handle, "request_id", None)
- video, url = _video_url_from_result(handle.get())
+ video, url = _video_url_from_result(wait_fal_queue_result(handle))
except Exception as exc:
logger.warning("FAL video gen failed (family=%s, endpoint=%s): %s", family_id, endpoint, exc, exc_info=True)
return _fal_error(f"FAL video generation failed: {exc}", "api_error", prompt, model=family_id, aspect_ratio=aspect_ratio)
diff --git a/tests/plugins/video_gen/test_fal_plugin.py b/tests/plugins/video_gen/test_fal_plugin.py
index 7e57e62d11f..e3bc8f41c5e 100644
--- a/tests/plugins/video_gen/test_fal_plugin.py
+++ b/tests/plugins/video_gen/test_fal_plugin.py
@@ -197,6 +197,9 @@ class TestFamilyRouting:
captured = {"endpoint": None, "arguments": None}
class FakeHandle:
+ def iter_events(self, **_):
+ return iter(())
+
def get(self):
return {"video": {"url": "https://fake/out.mp4"}}
@@ -539,6 +542,9 @@ class TestUpscalePass:
captured = {"calls": []}
class FakeHandle:
+ def iter_events(self, **_):
+ return iter(())
+
def __init__(self, endpoint):
self._endpoint = endpoint
@@ -611,6 +617,9 @@ class TestUpscalePass:
captured = {}
class FakeHandle:
+ def iter_events(self, **_):
+ return iter(())
+
def get(self):
return {"video": {"url": "https://fake/upscaled.mp4"}}
diff --git a/tests/tools/test_fal_sdk.py b/tests/tools/test_fal_sdk.py
index 66c3a214763..8ed361d4297 100644
--- a/tests/tools/test_fal_sdk.py
+++ b/tests/tools/test_fal_sdk.py
@@ -184,3 +184,20 @@ def test_older_sdk_timeout_default_and_optional_hint(transport, monkeypatch, tim
assert client.submit("app", {}, hint="ignored-on-old-sdk").request_id == "r1"
assert transport[0][0].extensions["timeout"]["read"] == expected
assert "x-fal-runner-hint" not in transport[0][0].headers
+
+
+def test_image_wait_polls_queue_status_at_gateway_cadence(transport, monkeypatch):
+ """The SDK's ``handle.get()`` polls every 0.1 s: one job alone burns the gateway's 200 req/min."""
+ import tools.image_generation_tool as image_tool
+ from tools.fal_common import FAL_QUEUE_POLL_SECONDS
+
+ requests, replies = transport
+ sleeps = []
+ monkeypatch.setattr(fal_client.client.time, "sleep", sleeps.append)
+ replies.extend([accepted(), (200, {"status": "IN_QUEUE", "queue_position": 0}),
+ (200, {"status": "IN_PROGRESS", "logs": None}),
+ *[(200, {"status": "COMPLETED", "logs": None})] * 2, (200, {"images": ["done"]})])
+ handle = _ManagedFalSyncClient(fal_client, key="token", queue_run_origin="https://queue.test").submit("app", {})
+ assert image_tool._wait_fal_result(handle) == {"images": ["done"]}
+ assert sleeps == [FAL_QUEUE_POLL_SECONDS] * 2
+ assert [r.url.path for r in requests] == ["/app", "/status", "/status", "/status", "/status", "/result"]
diff --git a/tests/tools/test_image_generation.py b/tests/tools/test_image_generation.py
index 1ad8b6a3a9b..19b882ec06b 100644
--- a/tests/tools/test_image_generation.py
+++ b/tests/tools/test_image_generation.py
@@ -518,6 +518,9 @@ class TestManagedPortalRouting:
# ---------------------------------------------------------------------------
class _FakeHandle:
+ def iter_events(self, **_):
+ return iter(())
+
def __init__(self, result):
self._result = result
diff --git a/tests/tools/test_image_generation_image_to_image.py b/tests/tools/test_image_generation_image_to_image.py
index d3d6a3d9849..375d27b7b0e 100644
--- a/tests/tools/test_image_generation_image_to_image.py
+++ b/tests/tools/test_image_generation_image_to_image.py
@@ -37,7 +37,7 @@ def cfg_home(tmp_path, monkeypatch):
def _write_cfg(home, cfg: dict):
- (home / "config.yaml").write_text(yaml.safe_dump(cfg))
+ (home / "config.yaml").write_text(yaml.safe_dump(cfg), encoding="utf-8")
# ---------------------------------------------------------------------------
@@ -113,6 +113,9 @@ class TestMandatoryKeysSurviveWhitelist:
class TestFalRouting:
def _patch_submit(self, monkeypatch, image_tool, capture: dict):
class _Handler:
+ def iter_events(self_inner, **_):
+ return iter(())
+
def get(self_inner):
return {"images": [{"url": "https://out/img.png", "width": 1, "height": 1}]}
diff --git a/tests/tools/test_image_generation_interrupt.py b/tests/tools/test_image_generation_interrupt.py
index 1e9a49e4ef9..a074a546ed2 100644
--- a/tests/tools/test_image_generation_interrupt.py
+++ b/tests/tools/test_image_generation_interrupt.py
@@ -16,6 +16,9 @@ class _SlowHandler:
self.delay = delay
self._result = result if result is not None else {"images": []}
+ def iter_events(self, **_):
+ return iter(())
+
def get(self):
time.sleep(self.delay)
return self._result
@@ -43,7 +46,7 @@ def test_wait_fal_result_raises_on_interrupt():
def test_wait_fal_result_propagates_handler_error():
- class _ErrHandler:
+ class _ErrHandler(_SlowHandler):
def get(self):
raise ValueError("boom")
diff --git a/tests/tools/test_video_generation_tool_surface_matrix.py b/tests/tools/test_video_generation_tool_surface_matrix.py
index d3c853cadac..09ebc902256 100644
--- a/tests/tools/test_video_generation_tool_surface_matrix.py
+++ b/tests/tools/test_video_generation_tool_surface_matrix.py
@@ -52,6 +52,8 @@ def matrix_env(tmp_path, monkeypatch):
fake_fal.subscribe = _subscribe # type: ignore
class _FalHandle:
+ def iter_events(self, **_):
+ return iter(())
def __init__(self, result):
self._result = result
def get(self):
@@ -117,7 +119,7 @@ def matrix_env(tmp_path, monkeypatch):
def _invoke_tool(home, cfg: dict, args: dict, tool_name: str = "video_generate") -> dict:
"""Write config, invoke the registered tool handler, return parsed JSON."""
- (home / "config.yaml").write_text(yaml.safe_dump(cfg))
+ (home / "config.yaml").write_text(yaml.safe_dump(cfg), encoding="utf-8")
import hermes_cli.config as cfg_mod
if hasattr(cfg_mod, "_invalidate_load_config_cache"):
cfg_mod._invalidate_load_config_cache()
diff --git a/tools/fal_common.py b/tools/fal_common.py
index 413340981ea..32f736bcb4c 100644
--- a/tools/fal_common.py
+++ b/tools/fal_common.py
@@ -33,6 +33,18 @@ from urllib.parse import urlencode
# A 429 whose Retry-After is longer than this is reported, not waited out inside a tool call.
MANAGED_FAL_RATE_LIMIT_RETRY_CAP_SECONDS = 30.0
+# Queue status poll cadence. ``handle.get()`` polls every 0.1 s (+RTT): one 12 s image cost
+# ~208 requests/min against the Nous gateway's 200/min window, so a second job 429s. 2 s is
+# ~25 requests/min per job (8 concurrent jobs fit) for at most +2 s latency per job.
+FAL_QUEUE_POLL_SECONDS = 2.0
+
+
+def wait_fal_queue_result(handle: Any, *, interval: float = FAL_QUEUE_POLL_SECONDS) -> Any:
+ """``handle.get()`` without the SDK's 0.1 s status poll: drain ``iter_events`` at ``interval``,
+ then ``get()`` sees Completed on its first status check and fetches the result."""
+ for _ in handle.iter_events(with_logs=False, interval=interval):
+ pass
+ return handle.get()
def import_fal_client() -> Any:
diff --git a/tools/image_generation_tool.py b/tools/image_generation_tool.py
index 93d4241792a..96280f5c64a 100644
--- a/tools/image_generation_tool.py
+++ b/tools/image_generation_tool.py
@@ -31,7 +31,7 @@ def _load_fal_client() -> Any:
from tools.debug_helpers import DebugSession
from tools.fal_common import (
_ManagedFalSyncClient, _extract_http_status, _managed_fal_billing_error,
- _normalize_fal_queue_url_format, submit_managed_fal_with_rate_limit_retry,
+ _normalize_fal_queue_url_format, submit_managed_fal_with_rate_limit_retry, wait_fal_queue_result,
)
from tools.image_generation_catalog import (
DEFAULT_ASPECT_RATIO, DEFAULT_MODEL, FAL_MODELS, UPSCALER_CREATIVITY, UPSCALER_DEFAULT_PROMPT,
@@ -94,7 +94,7 @@ class ImageGenerationInterrupted(Exception):
def _wait_fal_result(handler, *, poll_seconds: float = 0.5):
- """Interrupt-aware ``handler.get()``: the SDK blocks 30-60s, hiding user interrupts.
+ """Interrupt-aware ``wait_fal_queue_result``: the wait blocks 30-60s, hiding user interrupts.
The get runs on a daemon worker; the interrupt bit is polled between join slices and on
interrupt the worker is abandoned (remote job keeps running).
@@ -104,7 +104,7 @@ def _wait_fal_result(handler, *, poll_seconds: float = 0.5):
error_box: list = []
def _get():
try:
- result_box.append(handler.get())
+ result_box.append(wait_fal_queue_result(handler))
except BaseException as exc:
error_box.append(exc)
worker = threading.Thread(target=_get, daemon=True, name="fal-result-wait")