Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Whole-video progress and the time left are estimated from this computer's earlier generations

Bowen Xue978791238596
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py
index 143f3b1..0d7cfce 100644
--- a/freevideo_engine/agent/jobs.py
+++ b/freevideo_engine/agent/jobs.py
@@ -992,7 +992,7 @@ class Jobs:
'step_elapsed_seconds', 'estimated_step_seconds') if message.get(key) is not None}
overall = message.get('overall') if isinstance(message.get('overall'), dict) else {}
overall = {key: overall[key] for key in ('fraction', 'estimated', 'status', 'phase_start_fraction', 'phase_weight',
- 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted')
+ 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted', 'remaining_source')
if overall.get(key) is not None}
if overall:
value['overall'] = overall
diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py
index be11145..954a74c 100644
--- a/freevideo_engine/comfy_bridge.py
+++ b/freevideo_engine/comfy_bridge.py
@@ -11,6 +11,7 @@ import subprocess
import sys
import time
import uuid
+from collections import deque
from .long_paths import io_path, resolved, plain, plain_data, plain_errors
from . import processes
@@ -75,260 +76,729 @@ def progress_history_forecast(root, machine, canvas):
return {} # A progress estimate cannot prevent generation.
-# The decoder's phase events, as decode.py and macos_decode.py print them.
+class _SafeProgress:
+ """Any failure of the estimate falls back to plain progress messages."""
+ def __init__(self, inner):
+ self.inner = inner
+
+ def _call(self, name, *args, **kwargs):
+ try:
+ return getattr(self.inner, name)(*args, **kwargs)
+ except Exception:
+ self.inner = _PlainProgress() # A progress estimate cannot prevent generation.
+ return getattr(self.inner, name)(*args, **kwargs)
+
+ def annotate(self, message, *, event=None):
+ return self._call('annotate', message, event=event)
+
+ def observe(self, event):
+ return self._call('observe', event)
+
+ def heartbeat(self):
+ return self._call('heartbeat')
+
+ def configure(self, **facts):
+ return self._call('configure', **facts)
+
+
+class _PlainProgress:
+ """Pass engine messages through unchanged when no estimate can be built."""
+ def annotate(self, message, *, event=None):
+ return dict(message)
+
+ def observe(self, event):
+ pass
+
+ def heartbeat(self):
+ return None
+
+ def configure(self, **facts):
+ pass
+
+
+def _engine_version():
+ from . import __version__
+ return __version__
+
+
+# Names emitted by both decoders; forecast keys use the same suffixes.
_DECODE_PARTS = {'Loading video VAE': 'vae_load', 'Decoding the video': 'video_decode',
'Loading and decoding audio': 'audio_load_decode', 'Saving MP4 and audio': 'encode'}
class WholeVideoProgress:
- """Annotate phase events with one report-weighted whole-video progress.
+ """One elapsed/remaining clock, bounded by evidence of completed work.
- Local reports take precedence. Without history, live sampling supplies a
- rough decoder estimate which real decoded-frame timings then replace.
- These presentation estimates never change resource admission or placement.
+ This display estimator never changes the inference plan. Stage predictions
+ set the initial ruler; only real events release its count ceiling. Replay
+ options control smoothing, silent countdown and the ceiling's time shares.
"""
- ORDER = ('encoding', 'load', 'sampling', 'decode')
- # The decoder's own steps, in the order it runs them, by the name of each one's forecast.
- DECODE_PARTS = ('vae_load', 'video_decode', 'audio_load_decode', 'encode')
-
- def __init__(self, clock=time.perf_counter):
- self.clock = clock
- self.started = clock()
- self.phase_started = self.started
- self.phase = None
- self.completed = set()
- self.weights = {}
+ DECODE_PARTS = tuple(_DECODE_PARTS.values())
+
+ def __init__(self, clock=time.perf_counter, *, hardware=None, request=None,
+ cold_flags=None, records=(), rate=.005, smoothing='track',
+ silence='stage', ceiling='stretch', tau=10., max_rise=.019, final=.98, slow_start=True, started=None):
+ from . import stage_estimate
+ from .effort_forecast import stage_forecast
+ if not math.isfinite(rate) or rate < 0:
+ raise ValueError('Progress smoothing rate must be finite and nonnegative')
+ for name, value, choices in (('smoothing', smoothing, ('countdown', 'hold', 'track')),
+ ('silence', silence, ('freeze', 'stage')),
+ ('ceiling', ceiling, ('prior', 'current', 'stretch'))):
+ if value not in choices:
+ raise ValueError(f'Invalid progress {name}: {value!r}')
+ self.clock, self.rate = clock, rate
+ self.smoothing, self.silence, self.ceiling = smoothing, silence, ceiling
+ # tau: time constant of a downward correction ('track'); max_rise: the
+ # most the shown fraction may rise in any one second (None: no cap).
+ self.tau, self.max_rise = tau, max_rise
+ # final: once every frame is decoded, the bar approaches this fraction
+ # over the predicted audio and save time (None: no approach).
+ self.final = final
+ # slow_start: on a 16 GB machine, a load much slower than predicted
+ # scales the later stages until history or measured speed takes over.
+ self.slow_start = slow_start
+ self.started = self.phase_started = self.last_tick = self.last_event = clock() if started is None else started
+ self.last_publish = self.started
+ self.rise_window = deque()
+ self.hardware = dict(hardware or {})
+ self.request = dict(width=1344, height=768, frames=243, base_steps=8,
+ refine_steps=3, two_pass=True, flow='full', task='t2va')
+ self.request.update({k: v for k, v in (request or {}).items() if v is not None})
+ self.cold_flags = dict(cold_flags or {})
+ self.records = records
+ self.params = stage_estimate._params(None)
+ self.prior = stage_estimate.predict_prior(self.hardware, self.request, self.cold_flags, self.params)
+ self.weights = {k: self.prior['stages'][k]['seconds'] for k in stage_estimate.STAGES}
+ local = stage_forecast(records, self.request, self.cold_flags, self.weights, version=_engine_version())
+ self.remaining_source = 'local' if local.get('stages') else 'similar'
+ self.typical = local['typical']
+ self.local = local
+ self.weights.update(local.get('stages', {}))
+ self.completed = {k for k, v in self.weights.items() if v == 0}
self.measured = {}
- # Seconds each phase has run over every stretch of it, the forecast's
- # own weights, phases the forecast closed and phases a retry does not repeat.
self.spent = {}
- self.forecasts = {}
- self.closed = set()
+ self.forecasts = dict(self.weights)
+ self.phase_counts = {}
self.reused = set()
+ self.phase = 'encode'
+ self.completed.discard(self.phase)
+ self.sampling_plan = {}
+ self.steps = {'pass1': {}, 'pass2': {}}
+ self.live_calibration = True
+ self.count = None
+ self.fraction = 0.
+ self.ceiling_base = 0.
+ self.last_ceiling = 0.
+ self.ceiling_weights = dict(self.weights)
+ self.remaining_target = self.remaining_shown = sum(self.weights.values())
+ self.total = self.remaining_target
+ self.last_message = {}
+ self.finished = False
+ self.decode_remaining = None
self.decode_parts = {}
+ self.decode_part = None
self.decode_rest = None
- self.total = 0.
- self.forecast_ready = False
- self.failed = False
- self.sampling_plan = {}
- self.decode_from_sampling = False
- self.sampling_time_weighted = False
- self.sampling_rest = None
- self.decode_tail = 0.
- self.shown = 0.
-
- def _sampling_estimate(self, message, now):
- if self._step_estimate(message, now) or not self.sampling_rest or self.sampling_rest[0] != message.get('done'):
- return
- # No new step time from this message. Time that passes outside a step, as between the two passes, is not
- # taken out of the steps still to run.
- inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0.
- self.weights['sampling'] = max(self.weights.get('sampling', 0.),
- now - self.phase_started + max(0., self.sampling_rest[1] - inside))
-
- def _step_estimate(self, message, now):
- estimate = _timing_seconds(message.get('estimated_step_seconds'))
- done, total = message.get('done'), message.get('total')
- if estimate is None or type(done) is not int or type(total) is not int or not 0 <= done <= total:
+ self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds'])
+ self.retry_decode_only = False
+ self.retry_reuse_first = False
+ self.history = None # (output directory, machine, opened): read late by _late_history.
+ self.shown_source = self.remaining_source
+
+ @classmethod
+ def for_request(cls, machine, canvas, output_directory, *, base_steps=8,
+ refine_steps=3, two_pass=True, preview=False, upscale=False,
+ encoder_prewarm=None, options=None):
+ import platform
+ from .effort_forecast import start_context
+ started, opened = time.perf_counter(), time.monotonic()
+ observed, records = start_context(output_directory, machine)
+ hardware = cls._hardware(observed, machine)
+ # machine.vram_gib/ram_gib can be user policy caps, not physical totals.
+ # A worker process being alive is not proof that a model is resident.
+ warm = encoder_prewarm or {}
+ flags = {'encode_cached': True if upscale else None}
+ if warm.get('gpu_preload') is True and warm.get('state') in ('ready', 'complete') and not warm.get('worker_released'):
+ flags['encoder_cold'] = False
+ tracker = cls(hardware=hardware, records=records, cold_flags=flags, started=started,
+ request=dict(canvas, base_steps=base_steps, refine_steps=refine_steps,
+ two_pass=two_pass, flow='upscale' if upscale else 'preview' if preview else 'full'),
+ **(options or {}))
+ tracker.history = (output_directory, machine, opened)
+ return tracker
+
+ @staticmethod
+ def _hardware(observed, machine):
+ import platform
+ observed = dict(observed)
+ if isinstance(machine.get('hardware'), dict):
+ observed.update(machine['hardware'])
+ device = machine.get('device_identity')
+ device = device if isinstance(device, dict) else {}
+ hardware = dict(gpu_name=observed.get('gpu_name') or machine.get('gpu_name') or device.get('name'),
+ compute_capability=observed.get('capability'),
+ os=observed.get('system') or machine.get('system') or platform.system())
+ for target, source in (('ram_total_gib', 'ram_total'), ('vram_total_gib', 'vram_total')):
+ value = observed.get(source) or device.get('unified_ram_bytes')
+ if _timing_seconds(value):
+ hardware[target] = value / 2**30
+ return hardware
+
+ def _late_history(self):
+ """Use history the background read finished after this request began."""
+ from .effort_forecast import start_context, history_read_at
+ if self.history is None or self.remaining_source == 'measured':
return
- # Within a pass's first step, a layer event extrapolates the whole step
- # from the first layers, which can include compilation and read-ahead
- # (one slow layer once became a 142-minute estimate). With an estimate
- # already in hand, use layer-based step times only after the pass has
- # completed a step; a first run without one still gets its first number.
- base = self.sampling_plan.get('base_steps')
- if (self.forecast_ready and message.get('stage') == 'layers'
- and (done == 0 or type(base) is int and 0 < base == done < total)):
+ output_directory, machine, opened = self.history
+ if (history_read_at(output_directory, machine) or 0.) < opened:
return
- full_step = estimate
- remaining = (total - done) * estimate
- if message.get('uniform_remaining_steps') is False:
- plan = self.sampling_plan
- first, second = plan.get('first', {}), plan.get('second') or {}
- area = first.get('width', 0) * first.get('height', 0)
- if not area or not second.get('width') or not second.get('height'):
- return # Do not treat the two larger refinement NFEs as small NFEs.
- full_step *= second['width'] * second['height'] / area
- remaining = max(0, plan['base_steps'] - done) * estimate + plan['refine_steps'] * full_step
- self.sampling_rest = done, remaining
- remaining = max(0., remaining - (_timing_seconds(message.get('step_elapsed_seconds')) or 0.))
- self.weights['sampling'] = max(.001, now - self.phase_started + remaining)
- self.sampling_time_weighted = True
- if 'decode' not in self.weights or self.decode_from_sampling:
- # Windows 1344x768x243 measurements: decode/save was 1.71, 1.90
- # and 2.01 final-resolution warm NFEs on 4060 Ti / 5060 Ti / 5090.
- # This is an approximate first-run ruler, not a GPU speed claim.
- # It scales with this request's work, then yields to frame timing.
- self.weights['decode'] = 1.9 * full_step
- self.decode_from_sampling = True
- self.forecast_ready = True
- return True
+ self.history = None # One late update per request.
+ # Work on the live state, but restore all of it if anything fails.
+ saved = {key: dict(value) if isinstance(value, dict) else value for key, value in vars(self).items()}
+ try:
+ observed, rows = start_context(output_directory, machine, timeout=0.)
+ for key, value in self._hardware(observed, machine).items():
+ if value is not None and self.hardware.get(key) is None:
+ self.hardware[key] = value
+ self.records = rows
+ self._reforecast()
+ self.remaining_target = self._target(self.clock())
+ except Exception:
+ # Damaged history keeps the estimate the request already has.
+ vars(self).clear()
+ vars(self).update(saved)
- @staticmethod
- def _forecast_weights(forecast):
- stages = forecast.get('stages') if isinstance(forecast, dict) else None
- if not isinstance(stages, dict):
- return {}
- values = {
- 'encoding': _timing_seconds(stages.get('text_encode_seconds')),
- 'load': _timing_seconds(stages.get('load_seconds')),
- 'sampling': _timing_seconds(stages.get('sample_seconds')),
- 'decode': _timing_seconds(stages.get('decode_save_seconds')),
- }
- values = {key: value for key, value in values.items() if value is not None}
- # Sparse phase forecasts expose work_seconds plus its independently
- # predicted components. Keep the measured residual as the final
- # save/cleanup tail instead of silently dropping it or double-counting
- # the worker total.
- work = _timing_seconds(stages.get('work_seconds'))
- residual = work - sum(values.get(key, 0.) for key in ('load', 'sampling', 'decode')) if work is not None else 0.
- if residual > 0:
- values['decode'] = values.get('decode', 0.) + residual
- return values
+ def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None):
+ """Use facts already obtained by the bridge; never inspect media/models."""
+ self._late_history()
+ from .media_request import task_for
+ if sampled:
+ # The engine samples this canvas, which can differ from the requested size.
+ self.request.update({k: sampled[k] for k in ('width', 'height', 'frames') if sampled.get(k) is not None})
+ if plan:
+ self.sampling_plan = plan
+ self.request.update(two_pass=bool(plan.get('enabled')), base_steps=plan.get('base_steps', 8),
+ refine_steps=plan.get('refine_steps', 0),
+ sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)})
+ for key in ('pass1_resident_blocks', 'pass2_resident_blocks'):
+ value = plan.get(key)
+ if type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 50:
+ self.request[key] = value
+ if placement:
+ # generate.py's video_start/decode_resume carry the selected full
+ # canvas engine/decoder plan. Pass 1 can admit extra temporary
+ # cache, so its residency stays unknown for a two-pass request.
+ engine, decoder = placement.get('engine'), placement.get('decoder')
+ engine = engine if isinstance(engine, dict) else {}
+ decoder = decoder if isinstance(decoder, dict) else {}
+ blocks = engine.get('resident_blocks')
+ if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 50:
+ key = 'pass2_resident_blocks' if self.request['two_pass'] else 'pass1_resident_blocks'
+ self.request[key] = blocks
+ blocks = decoder.get('resident_blocks')
+ if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 36:
+ self.request['decoder_resident_blocks'] = blocks
+ if type(decoder.get('offload')) is bool:
+ self.request['decoder_offload'] = decoder['offload']
+ if media is not None:
+ self.request.update(task=task_for(media), lora_count=len(media.get('loras') or []),
+ reference_count=len(media.get('references') or []) +
+ sum(bool(media.get(k)) for k in ('first', 'last')))
+ if conditioning:
+ self.cold_flags['encode_cached'] = True
+ self._reforecast()
- @classmethod
- def _forecast_parts(cls, forecast):
- stages = forecast.get('stages') if isinstance(forecast, dict) else None
- values = {name: _timing_seconds(stages.get(name + '_seconds')) for name in cls.DECODE_PARTS} \
- if isinstance(stages, dict) else {}
- return {name: value for name, value in values.items() if value is not None}
+ def _reforecast(self, *, reset_source=False):
+ from .stage_estimate import predict_prior, STAGES
+ from .effort_forecast import stage_forecast
+ self.prior = predict_prior(self.hardware, self.request, self.cold_flags, self.params)
+ weights = {k: self.prior['stages'][k]['seconds'] for k in STAGES}
+ local = stage_forecast(self.records, self.request, self.cold_flags, weights, version=_engine_version())
+ weights.update(local.get('stages', {}))
+ self.typical = local['typical']
+ self.local = local
+ if reset_source or self.remaining_source != 'measured':
+ self.remaining_source = 'local' if local.get('stages') else 'similar'
+ for key, value in weights.items():
+ if key not in self.completed:
+ self.weights[key] = value
+ self.forecasts[key] = value
+ self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds'])
def forecast(self, value):
- value = value.get('forecast') if isinstance(value, dict) and 'forecast' in value else value
- weights = self._forecast_weights(value)
- if weights:
- # The prediction is emitted after prompt encoding has completed.
- # Close that phase before the first video-model event so its
- # measured weight is credited instead of starting a second timer.
- if self.phase == 'encoding':
- self._close(self.clock())
- self.closed.add('encoding')
- self.phase = None
- self.forecasts.update(weights)
- self.decode_parts = self._forecast_parts(value)
- self.weights.update(weights)
- if 'decode' in weights:
- self.decode_from_sampling = False
- self.forecast_ready = all(name in self.weights for name in ('sampling', 'decode'))
- self._settle()
- self.total = sum(self.weights.values())
-
- def _base(self):
- return sum(self.weights.get(name, 0.) for name in self.completed)
+ """Consume older worker forecasts without closing or crediting a phase."""
+ value = value.get('forecast', value) if isinstance(value, dict) else {}
+ if not isinstance(value, dict):
+ return
+ stages = value.get('stages') or {}
+ if not isinstance(stages, dict):
+ return
+ # Decoder substeps remain useful after sampling speed is measured.
+ for part in self.DECODE_PARTS:
+ seconds = _timing_seconds(stages.get(part + '_seconds'))
+ if seconds is not None:
+ self.decode_parts[part] = seconds
+ # Worker predictions lack the bridge's two-record similarity evidence;
+ # they must never create a local typical or replace live measurements.
+ if self.remaining_source == 'measured':
+ return
+ mapping = {'encode': 'text_encode_seconds', 'load': 'load_seconds', 'decode_save': 'decode_save_seconds'}
+ for key, source in mapping.items():
+ seconds = _timing_seconds(stages.get(source))
+ if seconds is not None and key not in self.completed:
+ self.weights[key] = seconds
+ sample = _timing_seconds(stages.get('sample_seconds'))
+ if sample:
+ keys = [k for k in ('pass1', 'upscale', 'pass2') if k not in self.completed]
+ total = sum(self.weights[k] for k in keys)
+ if total:
+ for key in keys:
+ self.weights[key] *= sample / total
+ work = _timing_seconds(stages.get('work_seconds'))
+ parts = [_timing_seconds(stages.get(k)) for k in ('load_seconds', 'sample_seconds', 'decode_save_seconds')]
+ if work and all(v is not None for v in parts) and 'decode_save' not in self.completed:
+ self.weights['decode_save'] += max(0., work - sum(parts))
+ self.forecasts.update({k: v for k, v in self.weights.items() if k not in self.completed})
- def _close(self, now):
+ def _enter(self, phase, now):
+ from .stage_estimate import STAGES
+ if phase == self.phase:
+ return
+ self._late_history()
+ if phase == 'load' and self.phase == 'upscale':
+ # The model handed over between the passes belongs to the gap
+ # between them, which is a stage of its own; a retry reloads
+ # through _retry instead.
+ return
if self.phase is not None:
self.spent[self.phase] = max(0., now - self.phase_started)
-
- def _enter(self, phase, now):
- """Phases do not always come one after the other: a preset check
- runs for a moment while the prompt is encoded, and an encoder event
- can follow the forecast. A phase that comes back goes on from the
- time it already has, and only the phases before the one now running
- are finished; a later one that ran for a moment keeps its forecast."""
- self._close(now)
+ self.phase_counts[self.phase] = self.count
+ self.forecasts[self.phase] = self.weights[self.phase]
+ # A brief model check can interrupt encoding, and loading can return
+ # between passes. Only predecessors of the active phase are finished.
+ self.completed = (set(STAGES[:STAGES.index(phase)]) | self.reused) - {phase}
+ for key in STAGES:
+ if key in self.completed and key in self.spent:
+ self.measured[key] = self.weights[key] = max(.001, self.spent[key])
+ elif key in self.measured:
+ self.measured.pop(key)
+ self.weights[key] = max(self.forecasts.get(key, 0.), self.spent.get(key, 0.))
self.phase, self.phase_started = phase, now - self.spent.get(phase, 0.)
- self.closed.discard(phase)
- self._settle()
-
- def _settle(self):
- before = self.ORDER[:self.ORDER.index(self.phase)] if self.phase else ()
- self.completed = (set(before) | self.closed | self.reused) - {self.phase}
- for name in self.ORDER:
- seconds = self.spent.get(name)
- if name in self.completed:
- if seconds is not None:
- self.measured[name] = self.weights[name] = max(.001, seconds)
- elif name not in self.reused:
- self.weights.pop(name, None) # Passed without running: nothing of it is left to wait for.
- elif name in self.measured:
- del self.measured[name]
- self.weights[name] = max(self.forecasts.get(name, 0.), seconds or 0., .001)
-
- def _message_phase(self, message):
- if message.get('timing_phase') in ('encoding', 'load', 'sampling', 'decode'):
- return message['timing_phase']
- phase = message.get('phase')
- return {'sampling': 'sampling', 'sample_finalize': 'decode',
- 'complete': 'decode'}.get(phase) if isinstance(phase, str) else None
-
- def annotate(self, message):
- message = dict(message)
- if isinstance(message.get('sampling_plan'), dict):
- self.sampling_plan = message['sampling_plan']
+ self.count = self.phase_counts.get(phase, self._initial_count())
+ self.decode_remaining = None
+
+ def _initial_count(self):
+ from .stage_estimate import canvases, step_counts
+ if self.phase == 'load':
+ return (0, 50, 1)
+ if self.phase in ('pass1', 'pass2'):
+ count = step_counts(self.request)[0 if self.phase == 'pass1' else 1]
+ return (0, count * 50, 1)
+ if self.phase == 'decode_save':
+ return (0, canvases(self.request)[2]['frames'], 1)
+ return None
+
+ def _set_count(self, done, total, margin=1):
+ # Duplicate/delayed logs cannot undo already witnessed counts. Using
+ # the smallest known unit avoids lowering the ceiling on the first
+ # counted event after an uncounted stage-start notification.
+ candidate = (max(0, done), total, margin)
+ old = self.count
+ if old is None or (candidate[0] + margin) / max(total, 1) >= (old[0] + old[2]) / max(old[1], 1):
+ self.count = candidate
+
+ def _phase_for(self, message, event):
+ name = event.get('event')
+ if self.retry_decode_only:
+ return 'decode_save'
+ if name == 'decode_resume' or message.get('phase') == 'sample_finalize':
+ return 'decode_save'
+ if message.get('stage') in ('latent_upscale', 'first_pass_reused'):
+ return 'upscale'
+ phase = message.get('timing_phase') or message.get('phase')
+ if phase == 'encoding' and any(self.steps.values()):
+ # After synchronized sampling steps, encoder-file lines are late
+ # notifications from the finished prompt, not a new encoding run.
+ return None
+ if phase == 'sampling':
+ # `loaded` falsely looks like sampling 0/N in the legacy converter.
+ if name == 'loaded':
+ return 'load'
+ done = message.get('done', 0)
+ base = self.request['base_steps']
+ if self.request['flow'] == 'upscale' or self.retry_reuse_first:
+ return 'pass2'
+ if self.request['flow'] != 'preview' and self.request['two_pass']:
+ if done > base or (done == base and (name == 'sampling_start' or message.get('stage') in ('layers', 'inputs'))):
+ return 'pass2'
+ return 'pass1'
+ return {'encoding': 'encode', 'load': 'load', 'decode': 'decode_save'}.get(phase) if isinstance(phase, str) else None
+
+ def _retry(self, retry, now):
+ # A retry changes the plan; the earlier load no longer says how fast
+ # this machine is, and measured steps arrive again soon.
+ self.slow_start = False
+ self.retry_decode_only = retry.get('reuse_sampling') is True
+ self.retry_reuse_first = retry.get('reuse_first_pass') is True
+ start = 'decode_save' if self.retry_decode_only else 'load'
+ if not self.retry_decode_only:
+ self.steps['pass2'].clear()
+ if not retry.get('reuse_first_pass'):
+ self.steps['pass1'].clear()
+ from .stage_estimate import STAGES
+ self.spent = {k: v for k, v in self.spent.items() if STAGES.index(k) < STAGES.index(start)}
+ self.phase_counts = {}
+ self.decode_part, self.decode_rest = None, None
+ self.reused = set(STAGES[:STAGES.index(start)])
+ self.completed = set(self.reused)
+ if self.retry_reuse_first and not self.retry_decode_only:
+ self.completed.add('pass1')
+ self.reused.add('pass1')
+ self.phase, self.phase_started = start, now
+ self.count, self.decode_remaining = self._initial_count(), None
+ self.live_calibration = False
+ # Recovery can change placement. Wait for the new attempt's plan;
+ # retained step durations are not new measurements of pending work.
+ for key in ('pass1_resident_blocks', 'pass2_resident_blocks',
+ 'decoder_resident_blocks', 'decoder_offload'):
+ self.request.pop(key, None)
+ self._reforecast(reset_source=True)
+ self.ceiling_base = self.fraction
+ self.ceiling_weights = {k: v for k, v in self.weights.items() if k not in self.completed}
+
+ def _counts_and_live(self, message, event, now):
+ from .stage_estimate import predict_live, step_counts
+ phase = self.phase
+ done, total = message.get('done'), message.get('total')
+ if phase in ('pass1', 'pass2'):
+ index = 0 if phase == 'pass1' else 1
+ count = step_counts(self.request)[index]
+ offset = self.request['base_steps'] if index else 0
+ if type(done) is int:
+ local_done = max(0, min(count, done - offset))
+ blocks, block = message.get('blocks'), message.get('block')
+ if type(blocks) is int and blocks > 0 and type(block) is int:
+ # One step of margin: layer events are dispatch estimates,
+ # steps are the synchronized counts (a first step may compile).
+ self._set_count(local_done * blocks + max(0, block - 1), count * blocks,
+ blocks if self.ceiling == 'stretch' else 1)
+ else:
+ # Keep the same one-layer margin across step boundaries;
+ # changing from one step to one layer could lower a ceiling.
+ self._set_count(local_done * 50, count * 50, 50 if self.ceiling == 'stretch' else 1)
+ seconds = _timing_seconds(message.get('last_step_seconds'))
+ if seconds and local_done > 0:
+ self.steps[phase][local_done] = seconds
+ if 1 in self.steps[phase] and 2 in self.steps[phase]:
+ self.live_calibration = True
+ elif phase == 'load':
+ if event.get('event') == 'loaded':
+ done, total = 50, 50
+ elif event.get('event') == 'prepared_blocks':
+ done, total = event.get('blocks'), 50
+ elif event.get('event') == 'model_load_phase':
+ done, total = event.get('done'), event.get('total')
+ # LoRA groups describe a substage, not completion of the whole load.
+ if type(done) is int and type(total) is int and total > 0 and message.get('unit') != 'bytes':
+ self._set_count(done, total)
+ elif phase == 'decode_save' and message.get('stage') == 'video':
+ elapsed = _timing_seconds(message.get('elapsed_seconds'))
+ if type(done) is int and type(total) is int and 0 <= done <= total and total > 0:
+ self._set_count(done, total)
+ if elapsed and done > 0:
+ self.decode_remaining = elapsed * (total - done) / done + self._after_frames(total)
+ self.weights[phase] = now - self.phase_started + self.decode_remaining
+ self.remaining_source = 'measured'
+ elif phase == 'decode_save' and (event.get('event') == 'decode_phase' or message.get('decode_part')):
+ # These real transitions finish video frames even for a decoder
+ # that does not stream progress callbacks.
+ part = message.get('decode_part') or _DECODE_PARTS.get(event.get('phase') if isinstance(event.get('phase'), str) else '')
+ if part in self.DECODE_PARTS and part != self.decode_part:
+ rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):]
+ self.decode_part = part
+ if all(k in self.decode_parts for k in rest):
+ self.decode_rest = sum(self.decode_parts[k] for k in rest[1:])
+ self.decode_remaining = self.decode_parts[part] + self.decode_rest
+ self.weights[phase] = now - self.phase_started + self.decode_remaining
+ if part in ('audio_load_decode', 'encode'):
+ total = self.count[1] if self.count else self.request['frames']
+ self._set_count(total, total)
+ if self.decode_remaining is None:
+ # Without frame callbacks this is the first sign that the
+ # video is decoded; only the measured tail remains.
+ self.decode_remaining = self._after_frames(total)
+ self.weights[phase] = now - self.phase_started + self.decode_remaining
+ self.remaining_source = 'measured'
+ # Only contiguous, actually completed steps calibrate the model.
+ steps = {}
+ for key, values in self.steps.items():
+ steps[key] = []
+ for i in range(1, len(values) + 1):
+ if i not in values:
+ break
+ steps[key].append(values[i])
+ if self.live_calibration and any(len(v) >= 2 for v in steps.values()):
+ done_stages = [('decode' if k == 'decode_save' else k) for k in self.completed]
+ live = predict_live(dict(hardware=self.hardware, request=self.request, cold_flags=self.cold_flags,
+ pass1_steps=steps['pass1'], pass2_steps=steps['pass2'], completed_stages=done_stages,
+ current_stage=phase, step_elapsed_seconds=message.get('step_elapsed_seconds', 0.),
+ pass_wall_overhead_done=[k for k, v in steps.items() if v]), self.params)
+ # With matching history on this machine, scale its own stage times
+ # by the measured warm step instead of the general model.
+ from statistics import median
+ history, warm, ratio = self.local.get('stages') or {}, self.local.get('warm') or {}, None
+ for key in ('pass2', 'pass1'):
+ if len(steps[key]) >= 2 and warm.get(key):
+ ratio = min(2., max(.5, median(steps[key][1:]) / warm[key]))
+ break
+ for key in ('pass1', 'upscale', 'pass2', 'decode_save'):
+ if key in self.completed or (key == 'decode_save' and self.decode_remaining is not None):
+ continue
+ if ratio is not None and key in history:
+ self.weights[key] = history[key] * (1. if key == 'decode_save' else ratio)
+ continue
+ remaining = live['stages'][key]['seconds']
+ if key == phase:
+ if key == 'decode_save':
+ # decode/save prediction is a full duration until frames arrive.
+ self.weights[key] = remaining
+ else:
+ self.weights[key] = now - self.phase_started + remaining
+ else:
+ self.weights[key] = remaining
+ self.decode_tail = max(0., live['stages']['decode_save']['seconds'] - live['stages']['video_decode']['seconds'])
+ self.remaining_source = 'measured'
+
+ def _after_frames(self, frames):
+ # Audio, MP4 and saving after the last decoded frame: about 2 s plus
+ # 0.03 s per frame (measured, slightly below the median, so the bar
+ # rarely jumps at saving), never more than the model's tail.
+ if self.decode_rest is not None and self.decode_part == 'video_decode':
+ return self.decode_rest
+ return min(self.decode_tail, 2. + .03 * frames)
+
+ def _slow_factor(self, now):
+ # A disk-bound 16 GB machine that loads much slower than predicted is
+ # slower in every later stage too (reports: about the square root of
+ # the load ratio). Larger memory or a slow encode does not predict it.
+ if not self.slow_start or self.remaining_source != 'similar':
+ return 1.
+ ram = self.hardware.get('ram_total_gib')
+ if ram is None or ram > 16.5:
+ return 1.
+ predicted = self.prior['stages']['load']['seconds']
+ actual = self.measured.get('load') or (now - self.phase_started if self.phase == 'load' else None)
+ if not actual or predicted <= 1 or actual / predicted <= 2:
+ return 1.
+ return min(4., math.sqrt(actual / predicted))
+
+ def _target(self, now):
+ factor = self._slow_factor(now)
+ later = ('pass1', 'upscale', 'pass2', 'decode_save')
+ remaining = sum(v * (factor if k in later else 1.)
+ for k, v in self.weights.items() if k not in self.completed and k != self.phase)
+ elapsed = max(0., now - self.phase_started)
+ # The stage under way is scaled too until its speed is measured.
+ current = self.weights.get(self.phase, 0.) * (factor if self.phase in later else 1.) - elapsed
+ if current < 0:
+ current = max(3., .1 * elapsed)
+ return max(0., remaining + current)
+
+ def _ceiling(self, now):
+ weights = (self.ceiling_weights if self.ceiling == 'prior' else
+ {k: self.weights[k] for k in self.ceiling_weights})
+ stretch = self.ceiling == 'stretch' and (self.count is None or self.count[0] <= 0)
+ if stretch and self.phase in weights:
+ # An overrunning stage without counted work keeps its end ahead of
+ # it, by the same remainder the remaining-time target assumes.
+ elapsed = max(0., now - self.phase_started)
+ weights[self.phase] = max(weights[self.phase], elapsed + max(3., .1 * elapsed))
+ total = sum(weights.values())
+ if total <= 0:
+ if self.ceiling in ('current', 'stretch'):
+ self.last_ceiling = .99
+ return .99
+ credit = sum(v for k, v in weights.items() if k in self.completed)
+ # An uncounted stage earns its allowance over time, not at its entry.
+ # This also avoids releasing all of upscale's share at a boundary.
+ duration = weights.get(self.phase, 0.)
+ fraction = min(1., max(0., now - self.phase_started) / max(duration, .001))
+ if self.count is not None and not stretch:
+ done, count, margin = self.count
+ fraction = min(1., max(0., (done + margin) / max(count, 1)))
+ credit += duration * fraction
+ value = min(.99, self.ceiling_base + (1. - self.ceiling_base) * credit / total)
+ if self.ceiling in ('current', 'stretch'):
+ self.last_ceiling = max(self.last_ceiling, value)
+ value = self.last_ceiling
+ return value
+
+ def _smooth(self, dt):
+ if self.smoothing == 'track':
+ shown, target = self.remaining_shown, self.remaining_target
+ if target > shown:
+ # Slower than predicted: stop counting down, then rise gently.
+ # A time that ran out starts again from the remainder an
+ # overrunning stage keeps; from zero it could never rise.
+ shown = max(shown, min(target, 3.))
+ self.remaining_shown = min(target, shown + self.rate * shown * dt)
+ else:
+ shown = max(target, shown - dt)
+ self.remaining_shown = max(0., shown - (shown - target) * (1. - math.exp(-dt / max(self.tau, .001))))
+ return
+ # Hold suppresses countdown only for an upward target correction.
+ # Both policies approach the target without crossing it by correction;
+ # countdown itself can pass below it. Keep a positive running remainder.
+ if self.smoothing == 'hold' and self.remaining_target > self.remaining_shown:
+ shown = self.remaining_shown
+ else:
+ floor = 3. if self.smoothing == 'countdown' else min(3., self.remaining_shown)
+ shown = max(floor, self.remaining_shown - dt)
+ limit = self.rate * shown * dt
+ self.remaining_shown = max(0., shown + max(-limit, min(limit, self.remaining_target - shown)))
+
+ def _render(self, message, now, *, heartbeat=False):
+ if self.remaining_source == 'local' and self.shown_source == 'similar':
+ # History read after the start replaces the estimate from similar
+ # settings: this computer's record corrects it once, so do not smooth.
+ # (A first measurement keeps smoothing; two steps are still noisy.)
+ self.remaining_shown = self.remaining_target
+ self.shown_source = self.remaining_source
+ elapsed = max(0., now - self.started)
+ candidate = elapsed / max(elapsed + self.remaining_shown, .001)
+ if (self.final is not None and self.phase == 'decode_save' and self.count
+ and self.count[0] >= self.count[1] > 0):
+ # Every frame is decoded; only audio, MP4 and saving remain, often
+ # for a few seconds only. Move there as fast as the rise limit allows.
+ candidate = max(candidate, self.final)
+ ceiling = self._ceiling(now)
+ limit = .99
+ # Retain the value in force one second ago and all newer changes.
+ while len(self.rise_window) > 1 and self.rise_window[1][0] <= now - 1.:
+ self.rise_window.popleft()
+ if self.max_rise is not None:
+ base = self.rise_window[0][1] if self.rise_window else self.fraction
+ limit = min(limit, base + self.max_rise)
+ self.fraction = max(self.fraction, min(limit, candidate, ceiling))
+ if not self.rise_window or self.rise_window[-1][1] != self.fraction:
+ if len(self.rise_window) > 1 and self.rise_window[-1][0] == now:
+ self.rise_window.pop()
+ self.rise_window.append((now, self.fraction))
+ overall = dict(status='complete' if self.finished else 'running', estimated=True,
+ fraction=1. if self.finished else self.fraction,
+ remaining_seconds=0. if self.finished else self.remaining_shown,
+ elapsed_seconds=elapsed, remaining_source=self.remaining_source,
+ phase_start_fraction=self.fraction, phase_weight=0., remaining_floor_seconds=0.,
+ at_ceiling=not self.finished and candidate >= ceiling,
+ sampling_time_weighted=True)
+ key = self.phase
+ if key in self.steps:
+ key += '_first' if not self.steps[key] else '_warm'
+ if key in self.typical:
+ overall.update(typical_seconds=self.typical[key], typical_source='local')
+ if heartbeat:
+ overall['heartbeat'] = True
+ self.last_publish = now
+ return dict(message, overall=overall)
+
+ def annotate(self, message, *, event=None):
+ message, event = dict(message), event or {}
+ now = self.clock()
+ if self.finished:
+ return self._render(message, now)
+ dt = max(0., now - self.last_tick)
+ old_phase, old_part = self.phase, self.decode_part
+ waiting_for_pass = (self.phase in self.steps and self.count is not None
+ and self.count[0] >= self.count[1] > 0)
+ plan = event if event.get('event') == 'sampling_plan' else message.get('sampling_plan')
+ if isinstance(plan, dict):
+ self.configure(plan=plan)
if isinstance(message.get('prediction'), dict):
self.forecast(message['prediction'])
- retry = message.get('retry') if isinstance(message.get('retry'), dict) else {}
- if retry.get('reuse_sampling') is True:
- self.reused.add('sampling')
- self.completed.add('sampling')
- phase = self._message_phase(message)
- now = self.clock()
- if phase and phase != self.phase:
+ retried = isinstance(message.get('retry'), dict) or message.get('stage') == 'preview_mismatch'
+ if isinstance(message.get('retry'), dict):
+ self._retry(message['retry'], now)
+ if message.get('stage') == 'preview_mismatch':
+ self.request['flow'] = 'full'
+ self._retry({}, now)
+ if event.get('event') in ('video_start', 'decode_resume'):
+ self.configure(placement=event)
+ phase = self._phase_for(message, event)
+ if phase:
self._enter(phase, now)
- if phase == 'sampling':
- self._sampling_estimate(message, now)
- if phase == 'decode' and message.get('stage') == 'video':
- done, total = message.get('done'), message.get('total')
- elapsed = _timing_seconds(message.get('elapsed_seconds'))
- if elapsed and type(done) is int and type(total) is int and 0 < done <= total:
- video_seconds = elapsed * total / done
- # Reports put audio/MP4/post-decode cleanup at about 6-8% of
- # video decode time. Retain this tail until audio/save starts.
- self.decode_tail = max(video_seconds * .08, self.decode_rest or 0.)
- self.weights['decode'] = now - self.phase_started + video_seconds - elapsed + self.decode_tail
- self.forecast_ready = True
- elif phase == 'decode':
- self.decode_tail = 0.
- part = message.get('decode_part')
- rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] if part in self.DECODE_PARTS else ()
- if rest and all(name in self.decode_parts for name in rest):
- # The decoder says which of its steps begins: what is left of
- # the phase is this step and the ones after it, not the whole
- # forecast made before the run.
- self.decode_rest = sum(self.decode_parts[name] for name in rest[1:])
- self.weights['decode'] = now - self.phase_started + self.decode_parts[part] + self.decode_rest
+ if phase in self.steps and phase != old_phase and phase not in self.spent:
+ # A first layer reports time inside its step, not the model
+ # handover before it. Backdate only that witnessed work.
+ inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0.
+ self.phase_started = now - inside
+ if waiting_for_pass:
+ dt = min(dt, inside)
+ if event.get('event') == 'conditioning_cache_hit':
+ self.cold_flags['encode_cached'] = True
+ if event.get('event') == 'conditioning_reused':
+ self.cold_flags['encode_cached'] = True
+ if event.get('event') == 'encoder_device_reuse':
+ self.cold_flags['encoder_cold'] = False
+ if event.get('event') == 'compute_device':
+ self.hardware.update(gpu_name=event.get('name') or self.hardware.get('gpu_name'))
+ if _timing_seconds(event.get('vram_total_bytes')):
+ self.hardware['vram_total_gib'] = event['vram_total_bytes'] / 2**30
+ self._reforecast()
+ if not retried:
+ self._counts_and_live(message, event, now)
+ self.remaining_target = self._target(now)
+ self._smooth(dt)
+ if retried:
+ # A new attempt is estimated afresh: the time left of the attempt
+ # that failed says nothing about it, and may already have run out.
+ self.remaining_shown = self.remaining_target
+ if self.phase == 'decode_save' and self.decode_rest is not None and (
+ self.decode_part != old_part or message.get('stage') == 'video'):
+ # A decoder boundary gives the pending substeps directly; the
+ # fraction still obeys its monotonic, per-second rise limit.
+ self.remaining_shown = self.remaining_target
+ self.last_tick = self.last_event = now
self.total = sum(self.weights.values())
- if message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete':
- message['overall'] = {'status': 'complete', 'fraction': 1., 'estimated': False,
- 'elapsed_seconds': now - self.started, 'remaining_seconds': 0.}
- return message
- overall = {'status': 'running', 'estimated': self.forecast_ready,
- 'elapsed_seconds': now - self.started}
- if self.forecast_ready and self.total > 0:
- weight = self.weights.get(self.phase, 0.)
- elapsed = max(0., now - self.phase_started)
- seconds = self.weights.get(self.phase)
- remaining = max(0., self.total - self._base() - min(elapsed, seconds or 0.))
- future = sum(value for name, value in self.weights.items()
- if name not in self.completed and name != self.phase)
- total = self.total
- if weight and elapsed > .95 * weight and .95 * future > .05 * self._base():
- # A phase that outruns its forecast (a model read from a cold
- # disk) is given the time it has taken, so the fraction keeps
- # rising instead of standing at the forecast's end; it does so
- # only while enough of the run is still ahead for that to
- # raise it. The time left is unchanged: what follows is.
- seconds = elapsed / .95
- total += seconds - weight
- weight = seconds
- start = self._base() / total
- fraction = start
- if weight:
- fraction += weight / total * min(.95, elapsed / max(seconds, 1e-9))
- # What has been given out is not taken back. A phase that
- # returns (a model loaded again between the passes), a decoder
- # step that ran past its forecast and a retry all add time
- # left; the fraction waits for the run to catch up with it.
- fraction = self.shown = max(self.shown, fraction)
- overall.update(fraction=fraction, phase_start_fraction=start,
- phase_weight=weight / total,
- phase_elapsed_seconds=elapsed, phase_seconds=seconds,
- remaining_seconds=remaining,
- remaining_floor_seconds=max(future, self.decode_tail),
- sampling_time_weighted=self.sampling_time_weighted)
- message['overall'] = overall
- return message
+ self.finished = message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete'
+ if self.finished:
+ self.measured[self.phase] = max(.001, now - self.phase_started)
+ self.weights[self.phase] = self.measured[self.phase]
+ self.total = sum(self.weights.values())
+ self.completed.update(self.weights)
+ self.remaining_target = self.remaining_shown = 0.
+ self.fraction = 1.
+ if message.keys() - {'report_id'}:
+ self.last_message = {k: v for k, v in message.items() if k not in ('overall', 'reset', 'new_request', 'prediction', 'sampling_plan')}
+ else:
+ message = dict(self.last_message, **message)
+ return self._render(message, now)
+
+ def observe(self, event):
+ """Use an engine event that has no progress message, without publishing."""
+ published, window = self.last_publish, self.rise_window.copy()
+ self.annotate({}, event=event)
+ self.last_publish, self.rise_window = published, window
+
+ def heartbeat(self):
+ now = self.clock()
+ if self.finished or now - self.last_publish < 1.:
+ return None
+ if (self.phase in self.steps and self.count is not None
+ and self.count[0] >= self.count[1] > 0):
+ # Between passes no pending step time is being spent.
+ self.last_tick = now
+ elif self.silence == 'stage':
+ # Admit only the part of this interval inside the predicted stage.
+ # Crossing its end freezes at the boundary, even with sparse polls.
+ end = self.phase_started + self.weights.get(self.phase, 0.)
+ dt = max(0., min(now, end) - max(self.last_tick, self.phase_started))
+ if dt:
+ self.remaining_target = max(0., self.remaining_target - dt)
+ self._smooth(dt)
+ self.last_tick = max(self.last_tick, min(now, end))
+ # Silent time is not counted down until the next engine event shows
+ # the engine is still working; that event then counts all of it.
+ return self._render(self.last_message, now, heartbeat=True)
def source_root():
@@ -550,10 +1020,7 @@ def progress_message(event):
'lora_prepare_start': 'Loading LoRAs',
'resource_retry': 'Adjusting memory placement and retrying'}
if name == 'decode_phase':
- message = {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'}
- if isinstance(event.get('phase'), str) and event['phase'] in _DECODE_PARTS:
- message['decode_part'] = _DECODE_PARTS[event['phase']]
- return message
+ return {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'}
if name == 'decode_progress':
return dict(label='Decoding the video', timing_phase='decode', phase='decode', stage='video',
done=event.get('done'), total=event.get('total'), unit='frames',
@@ -782,12 +1249,22 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
state['encoder_prewarm'] = prewarm_summary(encoder_prewarm)
save(run / 'comfy-request.json', state)
started = time.monotonic()
- whole_progress = WholeVideoProgress()
+ try:
+ whole_progress = WholeVideoProgress.for_request(machine, canvas, output_directory,
+ base_steps=base_steps, refine_steps=refine_steps, two_pass=two_pass,
+ preview=preview, upscale=bool(upscale), encoder_prewarm=state['encoder_prewarm'])
+ except Exception:
+ # A progress estimate cannot prevent generation, even without its parameters.
+ try:
+ whole_progress = WholeVideoProgress()
+ except Exception:
+ whole_progress = _PlainProgress()
+ whole_progress = _SafeProgress(whole_progress)
from .comfy_progress import REPORTS
report_id = REPORTS.register(output)
- def send_progress(message):
+ def send_progress(message, event=None):
if progress:
- progress(whole_progress.annotate(dict(message, report_id=report_id)))
+ progress(whole_progress.annotate(dict(message, report_id=report_id), event=event))
process = None
from .result_cache import ResultCache, request_key, inspect_bounded
cache = ResultCache(output_directory)
@@ -802,8 +1279,14 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
tails[2].reset()
message = progress_message(event)
if message:
- send_progress(message)
+ send_progress(message, event)
+ else:
+ whole_progress.observe(event)
+ heartbeat = whole_progress.heartbeat()
+ if progress and heartbeat:
+ progress(heartbeat)
try:
+ send_progress({'label': 'Preparing video'})
if interrupted:
interrupted()
extra = {}
@@ -889,10 +1372,9 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
state['sampling_cache_install'] = preparation
if preparation['seconds']:
started += preparation['seconds']
- whole_progress = WholeVideoProgress()
send_progress({'reset': True, 'label': 'Preparing video'})
- whole_progress.sampling_plan = planned
- whole_progress.forecast(progress_history_forecast(root, machine, dict(sampled, sampling_plan=planned)))
+ whole_progress.configure(plan=planned, media=extra.get('media', {}),
+ conditioning=bool(extra.get('conditioning')), sampled=sampled)
send_progress({'label': 'Preparing %.3f s video + audio · %d × %d' %
(canvas['seconds'], width, height)})
if release_models:
diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py
index fb0cd9b..32126c3 100644
--- a/freevideo_engine/comfy_progress.py
+++ b/freevideo_engine/comfy_progress.py
@@ -83,7 +83,18 @@ class ProgressState:
old = self.rows.get(node)
metadata_only = ('compatibility' in value and not value.get('new_request')
and not any(key in value for key in ('phase', 'stage', 'timing_phase', 'label', 'reset')))
- if old and metadata_only:
+ # A heartbeat carries a newer estimate, not engine progress (Harper's
+ # bridge repeats the last message's fields with a fresh overall).
+ heartbeat = isinstance(value.get('overall'), dict) and value['overall'].get('heartbeat') is True
+ overall_measured = now
+ if old and heartbeat:
+ # Replace only overall: the row and its age keep describing the
+ # last engine message, so a reopened page still sees a quiet engine.
+ value = dict(old['message'], overall=value['overall'])
+ value.pop('new_request', None)
+ value.pop('reset', None)
+ measured = old['measured']
+ elif old and metadata_only:
# A settings notice is not engine progress. Keep the current
# activity and its timestamp, including in reopened snapshots.
value = dict(old['message'], compatibility=value['compatibility'])
@@ -91,11 +102,11 @@ class ProgressState:
value.pop('reset', None)
if isinstance(value.get('overall'), dict) and 'reset' in value['overall']:
value['overall'] = {key: item for key, item in value['overall'].items() if key != 'reset'}
- measured = old['measured']
+ measured, overall_measured = old['measured'], old.get('overall_measured', old['measured'])
elif old and value.get('warning'):
# A memory warning must not replace the current sampling step.
value = dict(old['message'], warning=value['warning'])
- measured = old['measured']
+ measured, overall_measured = old['measured'], old.get('overall_measured', old['measured'])
else:
measured = now
# The page starts a new request on these too and clears its notes.
@@ -134,16 +145,22 @@ class ProgressState:
self._record_timeline(node, value, message, now, measured)
self.sequence += 1
value.update(node=node, stream_id=self.stream_id, sequence=self.sequence)
- self.rows[node] = {'message': value, 'measured': measured}
+ self.rows[node] = {'message': value, 'measured': measured, 'overall_measured': overall_measured}
while len(self.rows) > 32:
oldest = min(self.rows, key=lambda key: self.rows[key]['message']['sequence'])
del self.rows[oldest]
- return dict(value, age_seconds=max(0., now - measured))
+ return self._aged(value, now, measured, overall_measured)
+
+ @staticmethod
+ def _aged(message, now, measured, overall_measured):
+ # age_seconds: since the last engine message; overall_age_seconds: since
+ # the estimate in `overall` (a heartbeat refreshes only that one).
+ return dict(message, age_seconds=max(0., now - measured), overall_age_seconds=max(0., now - overall_measured))
def snapshot(self):
with self.lock:
now = self.clock()
- return [dict(deepcopy(row['message']), age_seconds=max(0., now-row['measured']))
+ return [self._aged(deepcopy(row['message']), now, row['measured'], row.get('overall_measured', row['measured']))
for row in self.rows.values()]
diff --git a/freevideo_engine/effort_forecast.py b/freevideo_engine/effort_forecast.py
index a94108c..71d959b 100644
--- a/freevideo_engine/effort_forecast.py
+++ b/freevideo_engine/effort_forecast.py
@@ -10,6 +10,7 @@ import math
import platform
from pathlib import Path
from statistics import median
+import threading
import time
@@ -48,7 +49,8 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):
if len(attempts) > 1:
return None
passes = video.get('sampling_passes') if plan.get('enabled') else [video]
- if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') else 1):
+ preview = bool(report.get('preview') or plan.get('preview'))
+ if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') and not preview else 1):
return None
measured = []
for part in passes:
@@ -61,7 +63,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):
if sum(p['seconds'] for p in measured) > total * 1.01:
return None
base = plan.get('base_steps', video.get('config', {}).get('steps', 8))
- refine = plan.get('refine_steps', 0) if plan.get('enabled') else 0
+ refine = plan.get('refine_steps', 0) if plan.get('enabled') and not preview else 0
if measured[0]['steps'] != base or (refine and measured[1]['steps'] != refine):
return None
encoding = report.get('encoding') or {}
@@ -118,7 +120,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):
version=version if isinstance(version, str) and len(version) <= 64 else None,
in_use=in_use,
reference_rows=reference_rows,
- flow='preview' if report.get('preview') or plan.get('preview') else 'full', stages=stages,
+ flow='preview' if preview else 'full', stages=stages,
sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)},
references=task_has_references(video.get('config', {}).get('task', 't2va')),
model_resident=video.get('resident_engine_cache_hit'))
@@ -130,7 +132,7 @@ def estimate(records, canvas, *, base_steps=8, refine_steps=3, two_pass=True, ta
validate_steps(base_steps, refine_steps, two_pass)
candidates = []
for row in records:
- if (row['task'] != task or row['adapters'] != adapters
+ if (row.get('flow', 'full') != 'full' or row['task'] != task or row['adapters'] != adapters
or row['shape'] != {k: canvas[k] for k in ('width', 'height', 'frames')}):
continue
# Both passes retain their own resolution costs. Never multiply the
@@ -319,6 +321,76 @@ def _stage_forecast(records, request, cold_flags, prior):
return result
+_START_CONTEXT = {}
+# Guards the entries above: only one background read per key, and readers
+# always see a complete hardware/records pair. File IO stays outside it.
+_START_LOCK = threading.Lock()
+
+
+def _context_key(output_directory, machine):
+ device = machine.get('device_identity') if machine.get('device_backend') == 'mps' else machine.get('gpu_uuid')
+ system = machine.get('system') or platform.system()
+ return (str(output_directory), json.dumps(device, sort_keys=True), system, machine.get('setup_run')), device, system
+
+
+def start_context(output_directory, machine, *, timeout=.035):
+ """Bound foreground IO to ~35 ms, even on slow/unavailable output disks.
+
+ One daemon per key reads in the background. Its last result stays in use
+ while a newer read runs (at most one every 15 s), so a request that starts
+ minutes after the previous one still sees that run. A request that started
+ before a read finished can pick it up later (history_read_at).
+ No GPU calls, resident RPC, model scans, hashing or directory-wide cache
+ inspection. Installation inventory describes identity, not live free RAM.
+ """
+ from threading import Event, Thread
+ key, device, system = _context_key(output_directory, machine)
+ start = False
+ with _START_LOCK:
+ entry = _START_CONTEXT.get(key)
+ if entry is None:
+ entry = dict(ready=Event(), started=None, read_at=None, hardware={}, records=[], reading=False)
+ _START_CONTEXT[key] = entry
+ now = time.monotonic()
+ if not entry['reading'] and (entry['started'] is None or now - entry['started'] > 15):
+ entry['reading'], entry['started'], start = True, now, True
+ if start:
+ def read():
+ hardware, records = dict(entry['hardware']), list(entry['records'])
+ try:
+ if machine.get('setup_run'):
+ with (Path(machine['setup_run']) / 'plan.json').open('rb') as stream:
+ raw = stream.read(1024 * 1024 + 1)
+ if len(raw) <= 1024 * 1024:
+ found = json.loads(raw).get('inventory', {}).get('hardware', {})
+ if same_device(found, device):
+ hardware = {k: found[k] for k in
+ ('gpu_name', 'capability', 'ram_total', 'vram_total', 'system') if k in found}
+ except (OSError, ValueError, TypeError, AttributeError):
+ pass
+ try:
+ records = local_records(output_directory, device, system=system, stage_details=True)
+ except (OSError, ValueError, TypeError, AttributeError):
+ pass
+ finally:
+ with _START_LOCK:
+ entry['hardware'], entry['records'] = hardware, records
+ entry['read_at'], entry['reading'] = time.monotonic(), False
+ entry['ready'].set()
+ Thread(target=read, name='freevideo-progress-history', daemon=True).start()
+ if timeout and not entry['ready'].is_set():
+ entry['ready'].wait(timeout)
+ with _START_LOCK:
+ return dict(entry['hardware']), list(entry['records'])
+
+
+def history_read_at(output_directory, machine):
+ """Monotonic time the last background read finished, or None."""
+ with _START_LOCK:
+ entry = _START_CONTEXT.get(_context_key(output_directory, machine)[0])
+ return entry and entry['read_at']
+
+
def local_records(output_directory, gpu_uuid, *, system=None, stage_details=False, refresh=False):
"""Bounded recent JSON reads, cached between slider changes; never open MP4s."""
system = system or platform.system()
@@ -338,7 +410,10 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals
except OSError:
continue
records = []
- for _, path in sorted(paths, reverse=True)[:48]:
+ # Slider estimates keep their original window of the 48 newest files; the
+ # progress history reads on until it has 48 usable rows.
+ candidates = sorted(paths, reverse=True)
+ for _, path in (candidates if stage_details else candidates[:48]):
try:
with path.open('rb') as stream:
data = stream.read(4 * 1024 * 1024 + 1)
@@ -347,6 +422,8 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals
row = timing_record(json.loads(data), gpu_uuid, system, stage_details=stage_details)
if row:
records.append(row)
+ if stage_details and len(records) == 48:
+ break
except (OSError, ValueError, TypeError, AttributeError, KeyError):
continue
if not stage_details:
diff --git a/freevideo_engine/progress_timeline.py b/freevideo_engine/progress_timeline.py
index bb2bab8..7fb3870 100644
--- a/freevideo_engine/progress_timeline.py
+++ b/freevideo_engine/progress_timeline.py
@@ -290,7 +290,7 @@ WARNINGS = frozenset((
SEGMENT_KEYS = ('phase', 'stage', 'timing_phase', 'label_key')
ENUMS = dict(phase=PHASES, stage=STAGES, timing_phase=TIMING_PHASES,
label_key=LABEL_KEYS, unit=UNITS, warning=WARNINGS)
-OVERALL_ENUMS = dict(status=STATUSES)
+OVERALL_ENUMS = dict(status=STATUSES, remaining_source=frozenset(('local', 'similar', 'measured', 'other')))
RETRY_ENUMS = dict(kind=RETRY_KINDS, failed_phase=PHASES)
COUNTERS = frozenset(('done', 'total', 'block', 'blocks', 'reference_trimmed'))
FLAGS = frozenset(('low_memory', 'kernel_cache_note', 'estimated', 'new_request', 'reset', 'result'))
@@ -349,7 +349,7 @@ def _fields(value, *, strict=False, nested=None):
(nested is None and (key in COUNTERS or key in FLAGS or key in SECONDS or key in FRACTIONS
or key in ('t', 'bytes_per_second', 'overall', 'retry'))) or
(nested == 'overall' and (key in OVERALL_SECONDS or key in OVERALL_FRACTIONS
- or key in ('estimated', 'sampling_time_weighted'))) or
+ or key in ('estimated', 'sampling_time_weighted', 'at_ceiling'))) or
(nested == 'retry' and key in ('attempt', 'max_attempts', 'reuse_sampling')))
if not known:
# Heartbeat and the bridge's extra overall fields are intentionally omitted.
@@ -370,8 +370,9 @@ def _fields(value, *, strict=False, nested=None):
raise ValueError('Invalid progress mapping')
result[key], count = _fields(raw, strict=strict, nested=key)
errors += count
- elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted'):
- if (strict or key == 'sampling_time_weighted') and type(raw) is not bool:
+ elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted',
+ 'at_ceiling'):
+ if (strict or key in ('sampling_time_weighted', 'at_ceiling')) and type(raw) is not bool:
raise ValueError('Invalid progress flag')
result[key] = bool(raw)
elif (nested is None and key in COUNTERS) or key in ('attempt', 'max_attempts'):
diff --git a/freevideo_engine/stage_estimate.py b/freevideo_engine/stage_estimate.py
index c06f1cb..5b5dded 100644
--- a/freevideo_engine/stage_estimate.py
+++ b/freevideo_engine/stage_estimate.py
@@ -124,6 +124,11 @@ def canvases(r):
first = dict(full, width=32*uw*m, height=32*uh*m)
else:
first = dict(full)
+ # two_pass.plan() runs a first pass under SMALL_FIRST_PASS_PIXELS (138,240) as one pass at the
+ # target size, except when the first pass itself is wanted (a preview or the upscale of one).
+ if (not sg.get('first') and two and r.get('flow') not in ('preview', 'upscale')
+ and first != full and first['width']*first['height'] < 138240):
+ return full, full, full
second = dict(full, **(sg.get('second') or {}))
decode = first if r.get('flow') == 'preview' else full
return first, second, decode
diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js
index 4f04116..09f68fb 100644
--- a/web/autopilot_chat.js
+++ b/web/autopilot_chat.js
@@ -1,7 +1,7 @@
// The conversation with the connected agent: what the user says, what the agent answers, and the agent's jobs as
// cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page.
import {agentGet, agentPost} from './agent_session.js';
-import { estimatedFraction, remainingShown } from './generation_progress.js';
+import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js';
const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css);
@@ -34,6 +34,28 @@ export function jobEstimate(job, age) {
// again: the rest of the way is drawn shorter, so the level neither waits nor goes back, and is full at the end.
const DRIFT = .25, RISE = .0005;
export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1});
+// The time left a running job shows, by the creation panel's rule: once measured never raised, and gone once its last
+// minute has passed. `row` is what the job showed before (none yet: it starts at the engine's number); `seconds` is null
+// from the moment the time has run out.
+export function jobLeft(row, sent, source, time) {
+ if (!row) {
+ if (!(sent > 0)) return null;
+ row = {seconds: sent, at: time, minute: null, last: 0, source: null, above: 0};
+ }
+ if (row.seconds === null) return row;
+ // Before the run is measured the engine's number is an estimate from earlier runs: follow it up as well as down.
+ const named = typeof source === 'string', initial = source === 'similar';
+ // The first number that is not that estimate replaces it once, up or down, and its last minute starts over.
+ if (named && !initial && row.source === 'similar' && sent > 0) { row.seconds = sent; row.minute = null; }
+ if (named) row.source = source;
+ const target = sent > 0 ? sent : null; row.above = remainingAbove(row.above, row.seconds, target, time - row.at);
+ row.seconds = remainingShown(row.seconds, target, time - row.at, initial, row.above); row.at = time;
+ if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; }
+ // An estimate that rises out of its last minute starts that minute over.
+ else if (row.seconds >= 60 && initial) row.minute = null;
+ if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null;
+ return row;
+}
export function jobLevel(row, base, left, time) {
if (row.top === null || base > row.top + RISE) {
if (row.top !== null && row.top < 1) row.scale = (1 - row.value) / (1 - row.top);
@@ -140,21 +162,13 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = ()
if (job.status === 'running') jobLevel(row.level, jobEstimate(row.job, time - row.at), row.job.progress?.remaining_seconds, time);
return row.level.value;
}
- // The time left each job shows, by the creation panel's rule: never raised, and gone once its last minute has passed.
const times = new Map();
function leftOf(job) {
- const time = performance.now() / 1000, sent = job.progress?.remaining_seconds;
if (job.status !== 'running') { times.delete(job.id); return null; }
- let row = times.get(job.id);
- if (!row) {
- if (!(sent > 0)) return null;
- times.set(job.id, row = {seconds: sent, at: time, minute: null, last: 0});
- }
- if (row.seconds === null) return null;
- row.seconds = remainingShown(row.seconds, sent > 0 ? sent : null, time - row.at); row.at = time;
- if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; }
- if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null;
- return row.seconds;
+ const row = jobLeft(times.get(job.id), job.progress?.remaining_seconds,
+ job.progress?.estimate?.overall?.remaining_source, performance.now() / 1000);
+ if (row) times.set(job.id, row);
+ return row ? row.seconds : null;
}
const active = () => !stopped && element.isConnected && !document.hidden && shown();
diff --git a/web/generation_progress.js b/web/generation_progress.js
index 2fee372..bb281d4 100644
--- a/web/generation_progress.js
+++ b/web/generation_progress.js
@@ -216,17 +216,24 @@ export function referenceTrimText(row, t) {
`${name} 比生成的视频长,已截取前 ${used} 秒${rule}`);
}
-// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Never raised:
-// an estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute"
-// early. A lower one is eased into.
-export function remainingShown(shown, target, dt) {
+// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Once measured, an
+// estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute"
+// early; one that has stayed well above it for a minute (`above` seconds so far) raises it gently, or a run
+// measured much slower would leave a low number standing for minutes. A lower one is eased into. An initial
+// estimate (made before this run was measured) is followed both ways: a slow machine must not keep a low number.
+export function remainingShown(shown, target, dt, initial = false, above = 0) {
if (target === null) return Math.max(0, shown - dt);
+ if (initial) return shown + (target - shown) * (1 - Math.exp(-dt / 15));
if (target < shown - dt) {
const counted = Math.max(0, shown - dt);
return counted + (target - counted) * (1 - Math.exp(-dt / 15));
}
- return target < shown ? target : shown;
+ if (target < shown) return target;
+ return above >= 60 ? Math.min(target, shown + Math.max(1, shown * .003) * dt) : shown;
}
+// The seconds the engine's estimate has stayed well above the shown time (a quarter more), `dt` seconds on.
+export const remainingAbove = (above, shown, target, dt) =>
+ target !== null && target > Math.max(shown * 1.25, shown + 30) ? above + dt : 0;
// The whole-video fraction the engine's last message stands for now: its estimate moved on by the seconds since the
// step sample (stateAge) and the whole-video sample (overallAge) arrived. Null when the message carries none.
export function estimatedFraction(state, overall, stateAge, overallAge) {
@@ -294,6 +301,13 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
// Mac: the unified-memory notice of this request, and whether its first pass has ended.
let memoryTight = null, firstPassOver = false;
let state = {}, overall = {}, started = 0, sampledAt = 0, overallAt = 0, timer = null, elapsedShown = 0;
+ // Where the bridge's remaining time comes from: 'similar' (no record of these
+ // settings here yet), 'local' (this computer's record) or 'measured' (this run).
+ // Absent from older bridges, which keep the two-steps rule.
+ let estimateSource = null, etaChangedAt = -Infinity;
+ // The page shows its own "preparing" row at the click; the engine's first
+ // message for that request follows and must not restart the elapsed time.
+ let placeholder = false, elapsedLead = 0;
let lastProgressAt = 0, previousProgress = null, segmentStartedAt = 0, segmentKey = '';
let pauseAnnounced = false, lastExplanation = '', announcedMessages = new Set();
let announcementRecord = null;
@@ -301,7 +315,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
let completedSteps = 0, previousRemaining = null, remainingAt = 0, lastActivity = null;
// Once the shown time reaches zero before the video is saved, it stays hidden for this request.
// The last minute runs on real time even while an estimate holds the number.
- let remainingRetired = false, lastMinuteAt = null, lastMinute = 0;
+ let remainingRetired = false, lastMinuteAt = null, lastMinute = 0, aboveSeconds = 0;
let shownFraction = null;
// The visible percentage counts toward its value with the bar instead of
// jumping. Accessible values are set exactly and immediately elsewhere.
@@ -600,7 +614,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
}
// Preserve the server's heartbeat/floor policy for the raw target only.
- // The displayed estimate has its own clock and never increases.
+ // The displayed estimate has its own clock; once it is no longer initial it never increases.
function rawRemaining() {
if (!valid(overall.remaining_seconds)) return null;
return hasHeartbeat ? overall.remaining_seconds : Math.max(
@@ -608,8 +622,25 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
overall.remaining_seconds - Math.max(0, (now() - overallAt) / 1000));
}
function observeRemaining(message) {
- if (completedSteps < 2 || remainingSeconds !== null || remainingRetired || !message.overall) return;
+ if (!message.overall) return;
+ const source = typeof message.overall.remaining_source === 'string' ? message.overall.remaining_source : null;
const raw = valid(message.overall.remaining_seconds) ? rawRemaining() : null;
+ if (source) {
+ // The bridge estimates the whole request from the first message:
+ // show it at once. Leaving the initial (similar-settings) estimate replaces
+ // the number once, up or down; after that it only falls.
+ const leaving = estimateSource === 'similar' && source !== 'similar';
+ estimateSource = source;
+ // The correction starts the last minute over. Once the shown time has run
+ // out it stays hidden for this attempt, as without an initial estimate.
+ if (leaving && raw !== null && !remainingRetired) { lastMinuteAt = null; lastMinute = 0; }
+ if (raw === null || remainingRetired) return;
+ // A retry's fresh estimate still waits for the 5 s spacing of the shown
+ // minutes; only a new request starts the line over (in update()).
+ if (remainingSeconds === null || leaving) { remainingSeconds = raw; remainingAt = now(); }
+ return;
+ }
+ if (completedSteps < 2 || remainingSeconds !== null || remainingRetired) return;
// Compare distinct received messages, never repeated timer ticks.
// Zero is stable only against another zero.
if (raw !== null && previousRemaining !== null
@@ -620,7 +651,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
}
function resetRemaining() {
completedSteps = 0; previousRemaining = null; remainingSeconds = null; remainingAt = now();
- remainingRetired = false; lastMinuteAt = null; lastMinute = 0;
+ remainingRetired = false; lastMinuteAt = null; lastMinute = 0; aboveSeconds = 0;
}
function tick() {
if (element.hidden) { drawStall(0, true); return; }
@@ -631,12 +662,16 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
// The page counts from its first message until the engine's own elapsed time
// arrives; that switch never moves the shown time back within a request.
const seconds = elapsedShown = Math.max(elapsedShown,
- valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge : (now()-started)/1000);
+ valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge + elapsedLead : (now()-started)/1000);
elapsed.textContent = `${t('Elapsed', '已用时')} ${duration(Math.max(0, seconds))}`;
if (ended) remainingSeconds = null;
if (remainingSeconds !== null) {
- remainingSeconds = remainingShown(remainingSeconds, rawRemaining(), Math.max(0, (now() - remainingAt) / 1000));
+ const sent = rawRemaining(), passed = Math.max(0, (now() - remainingAt) / 1000);
+ aboveSeconds = remainingAbove(aboveSeconds, remainingSeconds, sent, passed);
+ remainingSeconds = remainingShown(remainingSeconds, sent, passed, estimateSource === 'similar', aboveSeconds);
if (remainingSeconds < 60 && lastMinuteAt === null) { lastMinuteAt = now(); lastMinute = remainingSeconds; }
+ // An initial estimate may rise again: its last minute starts over.
+ else if (remainingSeconds >= 60 && estimateSource === 'similar') lastMinuteAt = null;
// A run slower than estimated would leave "less than a minute" up for minutes:
// once that minute has passed, or the time reaches zero, stop showing it.
if (!ended && (remainingSeconds <= 0
@@ -646,9 +681,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
}
remainingAt = now();
eta.hidden = remainingSeconds === null;
- eta.textContent = remainingSeconds === null ? '' : remainingSeconds >= 60
+ const etaText = remainingSeconds === null ? '' : remainingSeconds >= 60
? t(`About ${Math.ceil(remainingSeconds / 60)}\u00a0min left`, `预计还需约 ${Math.ceil(remainingSeconds / 60)}\u00a0分钟`)
: t('Less than a minute left', '预计还需不到 1\u00a0分钟');
+ const initial = remainingSeconds !== null && estimateSource === 'similar';
+ // While it follows the bridge, the shown minutes change at most every 5 s.
+ if (eta.textContent !== etaText && (!initial || !eta.textContent || now() - etaChangedAt >= 5000)) {
+ eta.textContent = etaText; etaChangedAt = now();
+ }
elapsed.hidden = false; times.hidden = elapsed.hidden && eta.hidden;
drawStall(age, ended);
drawActivity(age, ended);
@@ -662,8 +702,21 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
// not a stage change or something the user needs to act on.
if (message.warning === 'ram_budget_warning' && !message.phase) return;
const reset = newRequest(message);
- if (element.hidden || reset) {
- state = {}; overall = {}; started = now(); overallAt = now(); shownFraction = null; elapsedShown = 0;
+ const placeholderReset = element.hidden || reset;
+ const terminal = ['complete', 'failed', 'cancelled'].includes(message.phase);
+ if (placeholderReset) {
+ // The engine's first message after the page's own placeholder is the
+ // same click: keep counting from it. The placeholder lasts from this
+ // node's own start until any engine message, its end, or hide().
+ const sameClick = placeholder && message.stream_id !== undefined;
+ // From then on the engine's elapsed time is offset by the wait before it began.
+ const firstAt = valid(message.received_at) ? message.received_at : now();
+ elapsedLead = sameClick ? Math.max(0, (firstAt - started) / 1000
+ - (valid(message.overall?.elapsed_seconds) ? message.overall.elapsed_seconds : 0)) : 0;
+ if (!sameClick) { started = now(); elapsedShown = 0; }
+ placeholder = message.stream_id === undefined && !terminal;
+ state = {}; overall = {}; overallAt = now(); shownFraction = null; estimateSource = null;
+ eta.textContent = ''; etaChangedAt = -Infinity;
lastProgressAt = valid(message.received_at) ? message.received_at : now();
previousProgress = null; segmentKey = ''; segmentStartedAt = lastProgressAt;
pauseAnnounced = false; lastExplanation = ''; announcedMessages.clear();
@@ -674,21 +727,32 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
samplingPlan = {}; secondPass = false; hasHeartbeat = false;
announcementRecord = null;
}
+ if (!placeholderReset && (message.stream_id !== undefined || terminal)) placeholder = false;
const record = announcementsFor(message, reset);
if (record) {
announcementRecord = record;
for (const text of record.messages) announcedMessages.add(text);
}
const receivedAt = valid(message.received_at) ? message.received_at : now();
- const heartbeat = message.overall?.heartbeat === true;
+ // The estimate in `overall` can be newer than the engine message it rides
+ // on: a heartbeat refreshes only that (overall_received_at, from publish).
+ const overallReceivedAt = valid(message.overall_received_at) ? message.overall_received_at : receivedAt;
+ const beat = message.overall?.heartbeat === true;
+ // A heartbeat repeats the last engine message. A page that has not seen
+ // that message yet (reopened, or opened mid-run) takes it as the message.
+ const heartbeat = beat && previousProgress !== null;
if (message.reset === true) {
- hasHeartbeat = false; resetRemaining(); lastActivity = null;
+ hasHeartbeat = false; lastActivity = null;
+ // A stage restarting its own count (the sampler's start) leaves the
+ // bridge's whole-request estimate standing; a retry starts afresh.
+ if (estimateSource && !message.retry && message.phase !== 'recovery') { completedSteps = 0; previousRemaining = null; }
+ else resetRemaining();
// A reset that would move a shown share back starts its stage afresh
// (hidden, 5 s again). One before anything was counted, such as the
// sampler's own start, keeps the stage's clock and the 2 s rule.
if (miniFraction > 0) resetMini();
}
- if (heartbeat) hasHeartbeat = true;
+ if (beat) hasHeartbeat = true;
if (!heartbeat && message.phase === 'sampling' && valid(message.done))
completedSteps = Math.max(completedSteps, message.done);
if (message.low_memory && typeof message.low_memory === 'object') lowMemory = true;
@@ -714,8 +778,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
samplingPlan = {...message.plan_context};
// A heartbeat carries a newer estimate, never a newer engine event.
// Keep sampledAt paired with the sampling counters/elapsed sample.
- if (heartbeat && previousProgress !== null) {
- overall = {...overall, ...message.overall}; overallAt = receivedAt;
+ if (heartbeat) {
+ overall = {...overall, ...message.overall}; overallAt = overallReceivedAt;
observeRemaining(message);
tick(); return;
}
@@ -724,7 +788,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
|| message.prediction || message.sampling_plan || reset;
if (!activityEvent && previousProgress !== null) {
if (message.overall && typeof message.overall === 'object') {
- overall = {...message.overall}; overallAt = receivedAt; observeRemaining(message);
+ overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message);
}
tick(); return;
}
@@ -759,14 +823,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
}
element.dataset.phase = message.phase || message.timing_phase || '';
if (message.overall && typeof message.overall === 'object') {
- overall = {...message.overall}; overallAt = sampledAt;
+ overall = {...message.overall}; overallAt = overallReceivedAt;
observeRemaining(message);
}
element.hidden = false;
tick(); if (timer === null) timer = setInterval(tick, 250);
}
function hide() {
- element.hidden = true; status.hidden = true; recovery = ''; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false;
+ element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false;
compatibilityNote.hidden = true; compatibilityNote.textContent = '';
memoryTight = null; firstPassOver = false;
elapsedShown = 0;
diff --git a/web/progress_connection.js b/web/progress_connection.js
index 078c3ee..04e6ad1 100644
--- a/web/progress_connection.js
+++ b/web/progress_connection.js
@@ -10,7 +10,9 @@ export function createProgressConnection(api, consume, now = () => Date.now()) {
// The graph may still be loading. Do not acknowledge an undelivered
// snapshot; the next poll restores it once the node exists.
const age = Number.isFinite(message.age_seconds) ? Math.max(0, message.age_seconds) : 0;
- if (consume({...message, received_at: now() - age * 1000}) !== false)
+ // A heartbeat refreshes only the estimate: it is younger than the engine message.
+ const overallAge = Number.isFinite(message.overall_age_seconds) ? Math.max(0, message.overall_age_seconds) : age;
+ if (consume({...message, received_at: now() - age * 1000, overall_received_at: now() - overallAge * 1000}) !== false)
seen.set(key, message);
}
async function refresh() {