The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(client): consume estimated-duration headers for progress display (#16152)
Alexander Piskun9113c08c2e14
diff --git a/comfy_api_nodes/util/_helpers.py b/comfy_api_nodes/util/_helpers.pyindex ddfb3b65c..607ba5b28 100644--- a/comfy_api_nodes/util/_helpers.py+++ b/comfy_api_nodes/util/_helpers.py@@ -79,9 +79,8 @@ async def sleep_with_interrupt(node_cls: type[IO.ComfyNode] | None,label: str | None = None,start_ts: float | None = None,- estimated_total: int | None = None,*,- display_callback: Callable[[type[IO.ComfyNode], str, int, int | None], None] | None = None,+ display_callback: Callable[[type[IO.ComfyNode], str, int], None] | None = None,):"""Sleep in 1s slices while:@@ -95,7 +94,7 @@ async def sleep_with_interrupt(now = time.monotonic()if start_ts is not None and label and display_callback:with contextlib.suppress(Exception):- display_callback(node_cls, label, int(now - start_ts), estimated_total)+ display_callback(node_cls, label, int(now - start_ts))if now >= end:breakawait asyncio.sleep(min(1.0, end - now))diff --git a/comfy_api_nodes/util/client.py b/comfy_api_nodes/util/client.pyindex 4f62c297b..f89221694 100644--- a/comfy_api_nodes/util/client.py+++ b/comfy_api_nodes/util/client.py@@ -6,7 +6,7 @@ import mathimport timeimport uuidimport weakref-from collections.abc import Callable, Iterable+from collections.abc import Callable, Iterable, Mappingfrom dataclasses import dataclassfrom enum import Enumfrom io import BytesIO@@ -18,6 +18,11 @@ from aiohttp.client_exceptions import ClientError, ContentTypeErrorfrom pydantic import BaseModelfrom comfy import utils+from comfy.model_management import (+ InterruptProcessingException,+ interrupt_current_processing,+ throw_exception_if_processing_interrupted,+)from comfy_api.latest import IOfrom server import PromptServer@@ -65,7 +70,6 @@ class _RequestConfig:retry_backoff: floatwait_label: str = "Waiting"monitor_progress: bool = True- estimated_total: int | None = Nonefinal_label_on_success: str | None = "Completed"progress_origin_ts: float | None = Noneprice_extractor: Callable[[dict[str, Any]], float | None] | None = None@@ -80,6 +84,8 @@ class _PollUIState:is_queued: bool = Trueprice: float | None = Noneestimated_duration: int | None = None+ estimated_p90: int | None = None+ estimate_includes_queue: bool = Falsebase_processing_elapsed: float = 0.0 # sum of completed active intervalsactive_since: float | None = None # start time of current active interval (None if queued)@@ -91,8 +97,22 @@ PRICE_CREDITS_HEADER = "X-Comfy-Credits-Used""""Proxy response header with the actual cost in Comfy credits. When present on any successful proxied response,it takes precedence over ``price_extractor``."""+ESTIMATED_DURATION_HEADER = "X-Comfy-Estimated-Duration-Seconds"+ESTIMATED_DURATION_P90_HEADER = "X-Comfy-Estimated-Duration-P90-Seconds"+ESTIMATE_SOURCE_HEADER = "X-Comfy-Estimate-Source"+_credits_used_by_execution: "weakref.WeakKeyDictionary[type, float]" = weakref.WeakKeyDictionary()"""Last PRICE_CREDITS_HEADER value per node execution, keyed by the node's per-execution class clone."""+++@dataclass+class _ServerEstimate:+ p50_seconds: int+ p90_seconds: int | None+ source: str | None+++_server_estimates_by_execution: "weakref.WeakKeyDictionary[type, _ServerEstimate]" = weakref.WeakKeyDictionary()COMPLETED_STATUSES = ["succeeded", "succeed", "success", "completed", "finished", "done", "complete"]FAILED_STATUSES = ["cancelled", "canceled", "canceling", "fail", "failed", "error"]QUEUED_STATUSES = ["created", "queued", "queueing", "submitted", "initializing", "wait", "in_queue"]@@ -117,6 +137,32 @@ def _get_remembered_credits_used(node_cls: type[IO.ComfyNode]) -> float | None:return _credits_used_by_execution.get(node_cls)+def _parse_estimate_seconds(header_value: str | None) -> int | None:+ if not header_value:+ return None+ try:+ seconds = int(header_value.strip())+ except ValueError:+ logging.debug("Ignoring malformed estimate header value: %r", header_value)+ return None+ if not 1 <= seconds <= 86400:+ logging.debug("Ignoring out-of-range estimate header value: %r", header_value)+ return None+ return seconds+++def _maybe_remember_server_estimate(node_cls: type[IO.ComfyNode], headers: Mapping[str, str]) -> None:+ p50 = _parse_estimate_seconds(headers.get(ESTIMATED_DURATION_HEADER))+ if p50 is None:+ return+ p90 = _parse_estimate_seconds(headers.get(ESTIMATED_DURATION_P90_HEADER))+ if p90 is not None and p90 < p50:+ p90 = p50+ source = headers.get(ESTIMATE_SOURCE_HEADER) or None+ _server_estimates_by_execution[node_cls] = _ServerEstimate(p50, p90, source)+ logging.debug("Server duration estimate: p50=%ss, p90=%s, source=%s", p50, p90, source)++async def sync_op(cls: type[IO.ComfyNode],endpoint: ApiEndpoint,@@ -132,7 +178,6 @@ async def sync_op(retry_delay: float = 1.0,retry_backoff: float = 2.0,wait_label: str = "Waiting for server",- estimated_duration: int | None = None,final_label_on_success: str | None = "Completed",progress_origin_ts: float | None = None,monitor_progress: bool = True,@@ -152,7 +197,6 @@ async def sync_op(retry_delay=retry_delay,retry_backoff=retry_backoff,wait_label=wait_label,- estimated_duration=estimated_duration,as_binary=False,final_label_on_success=final_label_on_success,progress_origin_ts=progress_origin_ts,@@ -228,7 +272,6 @@ async def sync_op_raw(retry_delay: float = 1.0,retry_backoff: float = 2.0,wait_label: str = "Waiting for server",- estimated_duration: int | None = None,as_binary: bool = False,final_label_on_success: str | None = "Completed",progress_origin_ts: float | None = None,@@ -261,7 +304,6 @@ async def sync_op_raw(retry_backoff=retry_backoff,wait_label=wait_label,monitor_progress=monitor_progress,- estimated_total=estimated_duration,final_label_on_success=final_label_on_success,progress_origin_ts=progress_origin_ts,price_extractor=price_extractor,@@ -311,11 +353,22 @@ async def poll_op_raw(progress_bar = utils.ProgressBar(100) if progress_extractor else Nonelast_progress: int | None = None- state = _PollUIState(started=started, estimated_duration=estimated_duration)+ server_estimate = _server_estimates_by_execution.pop(cls, None)+ if server_estimate is not None:+ state = _PollUIState(+ started=started,+ estimated_duration=server_estimate.p50_seconds,+ estimated_p90=server_estimate.p90_seconds,+ estimate_includes_queue=True,+ )+ else:+ state = _PollUIState(started=started, estimated_duration=estimated_duration)+ estimate_bar = utils.ProgressBar(100) if progress_bar is None and server_estimate is not None else Nonestop_ticker = asyncio.Event()async def _ticker():"""Emit a UI update every second while polling is in progress."""+ last_estimate_pct = -1try:while not stop_ticker.is_set():if is_processing_interrupted():@@ -324,6 +377,15 @@ async def poll_op_raw(proc_elapsed = state.base_processing_elapsed + ((now - state.active_since) if state.active_since is not None else 0.0)+ if estimate_bar is not None:+ pct = _estimate_progress_pct(now - state.started, state.estimated_duration, state.estimated_p90)+ if pct != last_estimate_pct:+ try:+ estimate_bar.update_absolute(pct, total=100)+ except InterruptProcessingException:+ interrupt_current_processing()+ break+ last_estimate_pct = pct_display_time_progress(cls,status=state.status_label,@@ -333,6 +395,8 @@ async def poll_op_raw(is_queued=state.is_queued,processing_elapsed_seconds=int(proc_elapsed),extra_text=extra_text,+ estimated_p90=state.estimated_p90,+ estimate_includes_queue=state.estimate_includes_queue,)await asyncio.sleep(1.0)except Exception as exc:@@ -351,7 +415,6 @@ async def poll_op_raw(retry_delay=retry_delay_per_poll,retry_backoff=retry_backoff_per_poll,wait_label="Checking",- estimated_duration=None,as_binary=False,final_label_on_success=None,monitor_progress=False,@@ -367,7 +430,6 @@ async def poll_op_raw(timeout=cancel_timeout,max_retries=0,wait_label="Cancelling task",- estimated_duration=None,as_binary=False,final_label_on_success=None,monitor_progress=False,@@ -381,15 +443,26 @@ async def poll_op_raw(status = Noneif price_extractor:- new_price = price_extractor(resp_json)+ try:+ new_price = price_extractor(resp_json)+ except Exception as e:+ logging.error("Price extraction failed: %s", e)+ new_price = Noneif new_price is not None:state.price = new_priceif progress_extractor:- new_progress = progress_extractor(resp_json)+ try:+ new_progress = progress_extractor(resp_json)+ except Exception as e:+ logging.error("Progress extraction failed: %s", e)+ new_progress = Noneif new_progress is not None and last_progress != new_progress:- progress_bar.update_absolute(new_progress, total=100)- last_progress = new_progress+ try:+ progress_bar.update_absolute(new_progress, total=100)+ last_progress = new_progress+ except InterruptProcessingException:+ interrupt_current_processing()now_ts = time.monotonic()is_queued = status in queued_states@@ -414,12 +487,14 @@ async def poll_op_raw(if progress_bar and last_progress != 100:progress_bar.update_absolute(100, total=100)+ if estimate_bar is not None:+ estimate_bar.update_absolute(100, total=100)_display_time_progress(cls,status=status if status else "Completed",elapsed_seconds=int(now_ts - started),- estimated_total=estimated_duration,+ estimated_total=None,price=state.price,is_queued=False,processing_elapsed_seconds=int(state.base_processing_elapsed),@@ -428,12 +503,13 @@ async def poll_op_raw(return resp_jsonif status in failed_states:+ throw_exception_if_processing_interrupted()msg = f"Task failed: {json.dumps(resp_json)}"logging.error(msg)raise Exception(msg)try:- await sleep_with_interrupt(poll_interval, cls, None, None, None)+ await sleep_with_interrupt(poll_interval, cls, None, None)except ProcessingInterrupted:if cancel_endpoint:with contextlib.suppress(Exception):@@ -443,7 +519,6 @@ async def poll_op_raw(timeout=cancel_timeout,max_retries=0,wait_label="Cancelling task",- estimated_duration=None,as_binary=False,final_label_on_success=None,monitor_progress=False,@@ -503,17 +578,37 @@ def _display_time_progress(is_queued: bool | None = None,processing_elapsed_seconds: int | None = None,extra_text: str | None = None,+ estimated_p90: int | None = None,+ estimate_includes_queue: bool = False,) -> None:- if estimated_total is not None and estimated_total > 0 and is_queued is False:- pe = processing_elapsed_seconds if processing_elapsed_seconds is not None else elapsed_seconds- remaining = max(0, int(estimated_total) - int(pe))- time_line = f"Time elapsed: {int(elapsed_seconds)}s (~{remaining}s remaining)"- else:- time_line = f"Time elapsed: {int(elapsed_seconds)}s"+ time_line = f"Time elapsed: {int(elapsed_seconds)}s"+ if estimated_total is not None and estimated_total > 0:+ if estimate_includes_queue:+ if elapsed_seconds < estimated_total:+ time_line += f" (~{int(estimated_total) - int(elapsed_seconds)}s remaining)"+ elif estimated_p90 is not None and elapsed_seconds < estimated_p90:+ time_line += " (should finish soon)"+ else:+ time_line += " (taking longer than usual)"+ elif is_queued is False:+ pe = processing_elapsed_seconds if processing_elapsed_seconds is not None else elapsed_seconds+ remaining = max(0, int(estimated_total) - int(pe))+ time_line += f" (~{remaining}s remaining)"text = f"{time_line}\n\n{extra_text}" if extra_text else time_line_display_text(node_cls, text, status=status, price=price)+def _estimate_progress_pct(elapsed_seconds: float, p50_seconds: int | None, p90_seconds: int | None) -> int:+ if not p50_seconds or p50_seconds <= 0 or elapsed_seconds <= 0:+ return 0+ if elapsed_seconds < p50_seconds:+ return int(90.0 * elapsed_seconds / p50_seconds)+ horizon = p90_seconds if p90_seconds is not None and p90_seconds > p50_seconds else p50_seconds+ if elapsed_seconds >= horizon:+ return 95+ return 90 + int(5.0 * (elapsed_seconds - p50_seconds) / (horizon - p50_seconds))++async def _diagnose_connectivity() -> dict[str, bool]:"""Best-effort connectivity diagnostics to distinguish local vs. server issues."""results = {@@ -694,9 +789,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):if is_processing_interrupted():returnif cfg.monitor_progress:- _display_time_progress(- cfg.node_cls, cfg.wait_label, int(time.monotonic() - start_ts), cfg.estimated_total- )+ _display_time_progress(cfg.node_cls, cfg.wait_label, int(time.monotonic() - start_ts))await asyncio.sleep(1.0)except asyncio.CancelledError:return # normal shutdown@@ -851,7 +944,6 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):cfg.node_cls,cfg.wait_label if cfg.monitor_progress else None,start_time if cfg.monitor_progress else None,- cfg.estimated_total,display_callback=_display_time_progress if cfg.monitor_progress else None,)continue@@ -878,13 +970,12 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):if is_processing_interrupted():raise ProcessingInterrupted("Task cancelled")if cfg.monitor_progress:- _display_time_progress(- cfg.node_cls, cfg.wait_label, int(now - start_time), cfg.estimated_total- )+ _display_time_progress(cfg.node_cls, cfg.wait_label, int(now - start_time))bytes_payload = bytes(buff)resp_headers = {k.lower(): v for k, v in resp.headers.items()}if is_comfy_api_request:_maybe_remember_credits_used(cfg.node_cls, resp.headers.get(PRICE_CREDITS_HEADER))+ _maybe_remember_server_estimate(cfg.node_cls, resp.headers)if cfg.price_extractor:with contextlib.suppress(Exception):extracted_price = cfg.price_extractor(resp_headers)@@ -914,6 +1005,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):response_content_to_log = payload if isinstance(payload, dict) else textif is_comfy_api_request:_maybe_remember_credits_used(cfg.node_cls, resp.headers.get(PRICE_CREDITS_HEADER))+ _maybe_remember_server_estimate(cfg.node_cls, resp.headers)with contextlib.suppress(Exception):extracted_price = cfg.price_extractor(payload) if cfg.price_extractor else Noneoperation_succeeded = True@@ -956,7 +1048,6 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):cfg.node_cls,cfg.wait_label if cfg.monitor_progress else None,start_time if cfg.monitor_progress else None,- cfg.estimated_total,display_callback=_display_time_progress if cfg.monitor_progress else None,)delay *= cfg.retry_backoff@@ -1007,7 +1098,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool):if final_elapsed_seconds is not Noneelse int(time.monotonic() - start_time)),- estimated_total=cfg.estimated_total,+ estimated_total=None,price=extracted_price,is_queued=False,processing_elapsed_seconds=final_elapsed_seconds,diff --git a/comfy_api_nodes/util/download_helpers.py b/comfy_api_nodes/util/download_helpers.pyindex 4344bf7dc..f4b8724b3 100644--- a/comfy_api_nodes/util/download_helpers.py+++ b/comfy_api_nodes/util/download_helpers.py@@ -143,7 +143,7 @@ async def download_url_to_bytesio()if resp.status in _RETRY_STATUS and attempt <= max_retries:- await sleep_with_interrupt(delay, cls, None, None, None)+ await sleep_with_interrupt(delay, cls, None, None)delay *= retry_backoffcontinueraise Exception(f"Failed to download (HTTP {resp.status}).")@@ -200,7 +200,7 @@ async def download_url_to_bytesio(request_url=url,error_message=f"{type(e).__name__}: {str(e)} (will retry)",)- await sleep_with_interrupt(delay, cls, None, None, None)+ await sleep_with_interrupt(delay, cls, None, None)delay *= retry_backoffcontinuediff --git a/comfy_api_nodes/util/upload_helpers.py b/comfy_api_nodes/util/upload_helpers.pyindex f7029ee78..d7eb702ef 100644--- a/comfy_api_nodes/util/upload_helpers.py+++ b/comfy_api_nodes/util/upload_helpers.py@@ -330,7 +330,6 @@ async def upload_file(cls,wait_label,start_ts,- None,display_callback=_display_time_progress if wait_label else None,)delay *= retry_backoff@@ -362,7 +361,6 @@ async def upload_file(cls,wait_label,start_ts,- None,display_callback=_display_time_progress if wait_label else None,)delay *= retry_backoff