Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Whole-video progress and the time left are estimated from this computer's earlier generations
Bowen Xue978791238596
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.pyindex 143f3b1..0d7cfce 100644--- a/freevideo_engine/agent/jobs.py+++ b/freevideo_engine/agent/jobs.py@@ -992,7 +992,7 @@ class Jobs:'step_elapsed_seconds', 'estimated_step_seconds') if message.get(key) is not None}overall = message.get('overall') if isinstance(message.get('overall'), dict) else {}overall = {key: overall[key] for key in ('fraction', 'estimated', 'status', 'phase_start_fraction', 'phase_weight',- 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted')+ 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted', 'remaining_source')if overall.get(key) is not None}if overall:value['overall'] = overalldiff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.pyindex be11145..954a74c 100644--- a/freevideo_engine/comfy_bridge.py+++ b/freevideo_engine/comfy_bridge.py@@ -11,6 +11,7 @@ import subprocessimport sysimport timeimport uuid+from collections import dequefrom .long_paths import io_path, resolved, plain, plain_data, plain_errorsfrom . import processes@@ -75,260 +76,729 @@ def progress_history_forecast(root, machine, canvas):return {} # A progress estimate cannot prevent generation.-# The decoder's phase events, as decode.py and macos_decode.py print them.+class _SafeProgress:+ """Any failure of the estimate falls back to plain progress messages."""+ def __init__(self, inner):+ self.inner = inner++ def _call(self, name, *args, **kwargs):+ try:+ return getattr(self.inner, name)(*args, **kwargs)+ except Exception:+ self.inner = _PlainProgress() # A progress estimate cannot prevent generation.+ return getattr(self.inner, name)(*args, **kwargs)++ def annotate(self, message, *, event=None):+ return self._call('annotate', message, event=event)++ def observe(self, event):+ return self._call('observe', event)++ def heartbeat(self):+ return self._call('heartbeat')++ def configure(self, **facts):+ return self._call('configure', **facts)+++class _PlainProgress:+ """Pass engine messages through unchanged when no estimate can be built."""+ def annotate(self, message, *, event=None):+ return dict(message)++ def observe(self, event):+ pass++ def heartbeat(self):+ return None++ def configure(self, **facts):+ pass+++def _engine_version():+ from . import __version__+ return __version__+++# Names emitted by both decoders; forecast keys use the same suffixes._DECODE_PARTS = {'Loading video VAE': 'vae_load', 'Decoding the video': 'video_decode','Loading and decoding audio': 'audio_load_decode', 'Saving MP4 and audio': 'encode'}class WholeVideoProgress:- """Annotate phase events with one report-weighted whole-video progress.+ """One elapsed/remaining clock, bounded by evidence of completed work.- Local reports take precedence. Without history, live sampling supplies a- rough decoder estimate which real decoded-frame timings then replace.- These presentation estimates never change resource admission or placement.+ This display estimator never changes the inference plan. Stage predictions+ set the initial ruler; only real events release its count ceiling. Replay+ options control smoothing, silent countdown and the ceiling's time shares."""- ORDER = ('encoding', 'load', 'sampling', 'decode')- # The decoder's own steps, in the order it runs them, by the name of each one's forecast.- DECODE_PARTS = ('vae_load', 'video_decode', 'audio_load_decode', 'encode')-- def __init__(self, clock=time.perf_counter):- self.clock = clock- self.started = clock()- self.phase_started = self.started- self.phase = None- self.completed = set()- self.weights = {}+ DECODE_PARTS = tuple(_DECODE_PARTS.values())++ def __init__(self, clock=time.perf_counter, *, hardware=None, request=None,+ cold_flags=None, records=(), rate=.005, smoothing='track',+ silence='stage', ceiling='stretch', tau=10., max_rise=.019, final=.98, slow_start=True, started=None):+ from . import stage_estimate+ from .effort_forecast import stage_forecast+ if not math.isfinite(rate) or rate < 0:+ raise ValueError('Progress smoothing rate must be finite and nonnegative')+ for name, value, choices in (('smoothing', smoothing, ('countdown', 'hold', 'track')),+ ('silence', silence, ('freeze', 'stage')),+ ('ceiling', ceiling, ('prior', 'current', 'stretch'))):+ if value not in choices:+ raise ValueError(f'Invalid progress {name}: {value!r}')+ self.clock, self.rate = clock, rate+ self.smoothing, self.silence, self.ceiling = smoothing, silence, ceiling+ # tau: time constant of a downward correction ('track'); max_rise: the+ # most the shown fraction may rise in any one second (None: no cap).+ self.tau, self.max_rise = tau, max_rise+ # final: once every frame is decoded, the bar approaches this fraction+ # over the predicted audio and save time (None: no approach).+ self.final = final+ # slow_start: on a 16 GB machine, a load much slower than predicted+ # scales the later stages until history or measured speed takes over.+ self.slow_start = slow_start+ self.started = self.phase_started = self.last_tick = self.last_event = clock() if started is None else started+ self.last_publish = self.started+ self.rise_window = deque()+ self.hardware = dict(hardware or {})+ self.request = dict(width=1344, height=768, frames=243, base_steps=8,+ refine_steps=3, two_pass=True, flow='full', task='t2va')+ self.request.update({k: v for k, v in (request or {}).items() if v is not None})+ self.cold_flags = dict(cold_flags or {})+ self.records = records+ self.params = stage_estimate._params(None)+ self.prior = stage_estimate.predict_prior(self.hardware, self.request, self.cold_flags, self.params)+ self.weights = {k: self.prior['stages'][k]['seconds'] for k in stage_estimate.STAGES}+ local = stage_forecast(records, self.request, self.cold_flags, self.weights, version=_engine_version())+ self.remaining_source = 'local' if local.get('stages') else 'similar'+ self.typical = local['typical']+ self.local = local+ self.weights.update(local.get('stages', {}))+ self.completed = {k for k, v in self.weights.items() if v == 0}self.measured = {}- # Seconds each phase has run over every stretch of it, the forecast's- # own weights, phases the forecast closed and phases a retry does not repeat.self.spent = {}- self.forecasts = {}- self.closed = set()+ self.forecasts = dict(self.weights)+ self.phase_counts = {}self.reused = set()+ self.phase = 'encode'+ self.completed.discard(self.phase)+ self.sampling_plan = {}+ self.steps = {'pass1': {}, 'pass2': {}}+ self.live_calibration = True+ self.count = None+ self.fraction = 0.+ self.ceiling_base = 0.+ self.last_ceiling = 0.+ self.ceiling_weights = dict(self.weights)+ self.remaining_target = self.remaining_shown = sum(self.weights.values())+ self.total = self.remaining_target+ self.last_message = {}+ self.finished = False+ self.decode_remaining = Noneself.decode_parts = {}+ self.decode_part = Noneself.decode_rest = None- self.total = 0.- self.forecast_ready = False- self.failed = False- self.sampling_plan = {}- self.decode_from_sampling = False- self.sampling_time_weighted = False- self.sampling_rest = None- self.decode_tail = 0.- self.shown = 0.-- def _sampling_estimate(self, message, now):- if self._step_estimate(message, now) or not self.sampling_rest or self.sampling_rest[0] != message.get('done'):- return- # No new step time from this message. Time that passes outside a step, as between the two passes, is not- # taken out of the steps still to run.- inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0.- self.weights['sampling'] = max(self.weights.get('sampling', 0.),- now - self.phase_started + max(0., self.sampling_rest[1] - inside))-- def _step_estimate(self, message, now):- estimate = _timing_seconds(message.get('estimated_step_seconds'))- done, total = message.get('done'), message.get('total')- if estimate is None or type(done) is not int or type(total) is not int or not 0 <= done <= total:+ self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds'])+ self.retry_decode_only = False+ self.retry_reuse_first = False+ self.history = None # (output directory, machine, opened): read late by _late_history.+ self.shown_source = self.remaining_source++ @classmethod+ def for_request(cls, machine, canvas, output_directory, *, base_steps=8,+ refine_steps=3, two_pass=True, preview=False, upscale=False,+ encoder_prewarm=None, options=None):+ import platform+ from .effort_forecast import start_context+ started, opened = time.perf_counter(), time.monotonic()+ observed, records = start_context(output_directory, machine)+ hardware = cls._hardware(observed, machine)+ # machine.vram_gib/ram_gib can be user policy caps, not physical totals.+ # A worker process being alive is not proof that a model is resident.+ warm = encoder_prewarm or {}+ flags = {'encode_cached': True if upscale else None}+ if warm.get('gpu_preload') is True and warm.get('state') in ('ready', 'complete') and not warm.get('worker_released'):+ flags['encoder_cold'] = False+ tracker = cls(hardware=hardware, records=records, cold_flags=flags, started=started,+ request=dict(canvas, base_steps=base_steps, refine_steps=refine_steps,+ two_pass=two_pass, flow='upscale' if upscale else 'preview' if preview else 'full'),+ **(options or {}))+ tracker.history = (output_directory, machine, opened)+ return tracker++ @staticmethod+ def _hardware(observed, machine):+ import platform+ observed = dict(observed)+ if isinstance(machine.get('hardware'), dict):+ observed.update(machine['hardware'])+ device = machine.get('device_identity')+ device = device if isinstance(device, dict) else {}+ hardware = dict(gpu_name=observed.get('gpu_name') or machine.get('gpu_name') or device.get('name'),+ compute_capability=observed.get('capability'),+ os=observed.get('system') or machine.get('system') or platform.system())+ for target, source in (('ram_total_gib', 'ram_total'), ('vram_total_gib', 'vram_total')):+ value = observed.get(source) or device.get('unified_ram_bytes')+ if _timing_seconds(value):+ hardware[target] = value / 2**30+ return hardware++ def _late_history(self):+ """Use history the background read finished after this request began."""+ from .effort_forecast import start_context, history_read_at+ if self.history is None or self.remaining_source == 'measured':return- # Within a pass's first step, a layer event extrapolates the whole step- # from the first layers, which can include compilation and read-ahead- # (one slow layer once became a 142-minute estimate). With an estimate- # already in hand, use layer-based step times only after the pass has- # completed a step; a first run without one still gets its first number.- base = self.sampling_plan.get('base_steps')- if (self.forecast_ready and message.get('stage') == 'layers'- and (done == 0 or type(base) is int and 0 < base == done < total)):+ output_directory, machine, opened = self.history+ if (history_read_at(output_directory, machine) or 0.) < opened:return- full_step = estimate- remaining = (total - done) * estimate- if message.get('uniform_remaining_steps') is False:- plan = self.sampling_plan- first, second = plan.get('first', {}), plan.get('second') or {}- area = first.get('width', 0) * first.get('height', 0)- if not area or not second.get('width') or not second.get('height'):- return # Do not treat the two larger refinement NFEs as small NFEs.- full_step *= second['width'] * second['height'] / area- remaining = max(0, plan['base_steps'] - done) * estimate + plan['refine_steps'] * full_step- self.sampling_rest = done, remaining- remaining = max(0., remaining - (_timing_seconds(message.get('step_elapsed_seconds')) or 0.))- self.weights['sampling'] = max(.001, now - self.phase_started + remaining)- self.sampling_time_weighted = True- if 'decode' not in self.weights or self.decode_from_sampling:- # Windows 1344x768x243 measurements: decode/save was 1.71, 1.90- # and 2.01 final-resolution warm NFEs on 4060 Ti / 5060 Ti / 5090.- # This is an approximate first-run ruler, not a GPU speed claim.- # It scales with this request's work, then yields to frame timing.- self.weights['decode'] = 1.9 * full_step- self.decode_from_sampling = True- self.forecast_ready = True- return True+ self.history = None # One late update per request.+ # Work on the live state, but restore all of it if anything fails.+ saved = {key: dict(value) if isinstance(value, dict) else value for key, value in vars(self).items()}+ try:+ observed, rows = start_context(output_directory, machine, timeout=0.)+ for key, value in self._hardware(observed, machine).items():+ if value is not None and self.hardware.get(key) is None:+ self.hardware[key] = value+ self.records = rows+ self._reforecast()+ self.remaining_target = self._target(self.clock())+ except Exception:+ # Damaged history keeps the estimate the request already has.+ vars(self).clear()+ vars(self).update(saved)- @staticmethod- def _forecast_weights(forecast):- stages = forecast.get('stages') if isinstance(forecast, dict) else None- if not isinstance(stages, dict):- return {}- values = {- 'encoding': _timing_seconds(stages.get('text_encode_seconds')),- 'load': _timing_seconds(stages.get('load_seconds')),- 'sampling': _timing_seconds(stages.get('sample_seconds')),- 'decode': _timing_seconds(stages.get('decode_save_seconds')),- }- values = {key: value for key, value in values.items() if value is not None}- # Sparse phase forecasts expose work_seconds plus its independently- # predicted components. Keep the measured residual as the final- # save/cleanup tail instead of silently dropping it or double-counting- # the worker total.- work = _timing_seconds(stages.get('work_seconds'))- residual = work - sum(values.get(key, 0.) for key in ('load', 'sampling', 'decode')) if work is not None else 0.- if residual > 0:- values['decode'] = values.get('decode', 0.) + residual- return values+ def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None):+ """Use facts already obtained by the bridge; never inspect media/models."""+ self._late_history()+ from .media_request import task_for+ if sampled:+ # The engine samples this canvas, which can differ from the requested size.+ self.request.update({k: sampled[k] for k in ('width', 'height', 'frames') if sampled.get(k) is not None})+ if plan:+ self.sampling_plan = plan+ self.request.update(two_pass=bool(plan.get('enabled')), base_steps=plan.get('base_steps', 8),+ refine_steps=plan.get('refine_steps', 0),+ sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)})+ for key in ('pass1_resident_blocks', 'pass2_resident_blocks'):+ value = plan.get(key)+ if type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 50:+ self.request[key] = value+ if placement:+ # generate.py's video_start/decode_resume carry the selected full+ # canvas engine/decoder plan. Pass 1 can admit extra temporary+ # cache, so its residency stays unknown for a two-pass request.+ engine, decoder = placement.get('engine'), placement.get('decoder')+ engine = engine if isinstance(engine, dict) else {}+ decoder = decoder if isinstance(decoder, dict) else {}+ blocks = engine.get('resident_blocks')+ if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 50:+ key = 'pass2_resident_blocks' if self.request['two_pass'] else 'pass1_resident_blocks'+ self.request[key] = blocks+ blocks = decoder.get('resident_blocks')+ if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 36:+ self.request['decoder_resident_blocks'] = blocks+ if type(decoder.get('offload')) is bool:+ self.request['decoder_offload'] = decoder['offload']+ if media is not None:+ self.request.update(task=task_for(media), lora_count=len(media.get('loras') or []),+ reference_count=len(media.get('references') or []) ++ sum(bool(media.get(k)) for k in ('first', 'last')))+ if conditioning:+ self.cold_flags['encode_cached'] = True+ self._reforecast()- @classmethod- def _forecast_parts(cls, forecast):- stages = forecast.get('stages') if isinstance(forecast, dict) else None- values = {name: _timing_seconds(stages.get(name + '_seconds')) for name in cls.DECODE_PARTS} \- if isinstance(stages, dict) else {}- return {name: value for name, value in values.items() if value is not None}+ def _reforecast(self, *, reset_source=False):+ from .stage_estimate import predict_prior, STAGES+ from .effort_forecast import stage_forecast+ self.prior = predict_prior(self.hardware, self.request, self.cold_flags, self.params)+ weights = {k: self.prior['stages'][k]['seconds'] for k in STAGES}+ local = stage_forecast(self.records, self.request, self.cold_flags, weights, version=_engine_version())+ weights.update(local.get('stages', {}))+ self.typical = local['typical']+ self.local = local+ if reset_source or self.remaining_source != 'measured':+ self.remaining_source = 'local' if local.get('stages') else 'similar'+ for key, value in weights.items():+ if key not in self.completed:+ self.weights[key] = value+ self.forecasts[key] = value+ self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds'])def forecast(self, value):- value = value.get('forecast') if isinstance(value, dict) and 'forecast' in value else value- weights = self._forecast_weights(value)- if weights:- # The prediction is emitted after prompt encoding has completed.- # Close that phase before the first video-model event so its- # measured weight is credited instead of starting a second timer.- if self.phase == 'encoding':- self._close(self.clock())- self.closed.add('encoding')- self.phase = None- self.forecasts.update(weights)- self.decode_parts = self._forecast_parts(value)- self.weights.update(weights)- if 'decode' in weights:- self.decode_from_sampling = False- self.forecast_ready = all(name in self.weights for name in ('sampling', 'decode'))- self._settle()- self.total = sum(self.weights.values())-- def _base(self):- return sum(self.weights.get(name, 0.) for name in self.completed)+ """Consume older worker forecasts without closing or crediting a phase."""+ value = value.get('forecast', value) if isinstance(value, dict) else {}+ if not isinstance(value, dict):+ return+ stages = value.get('stages') or {}+ if not isinstance(stages, dict):+ return+ # Decoder substeps remain useful after sampling speed is measured.+ for part in self.DECODE_PARTS:+ seconds = _timing_seconds(stages.get(part + '_seconds'))+ if seconds is not None:+ self.decode_parts[part] = seconds+ # Worker predictions lack the bridge's two-record similarity evidence;+ # they must never create a local typical or replace live measurements.+ if self.remaining_source == 'measured':+ return+ mapping = {'encode': 'text_encode_seconds', 'load': 'load_seconds', 'decode_save': 'decode_save_seconds'}+ for key, source in mapping.items():+ seconds = _timing_seconds(stages.get(source))+ if seconds is not None and key not in self.completed:+ self.weights[key] = seconds+ sample = _timing_seconds(stages.get('sample_seconds'))+ if sample:+ keys = [k for k in ('pass1', 'upscale', 'pass2') if k not in self.completed]+ total = sum(self.weights[k] for k in keys)+ if total:+ for key in keys:+ self.weights[key] *= sample / total+ work = _timing_seconds(stages.get('work_seconds'))+ parts = [_timing_seconds(stages.get(k)) for k in ('load_seconds', 'sample_seconds', 'decode_save_seconds')]+ if work and all(v is not None for v in parts) and 'decode_save' not in self.completed:+ self.weights['decode_save'] += max(0., work - sum(parts))+ self.forecasts.update({k: v for k, v in self.weights.items() if k not in self.completed})- def _close(self, now):+ def _enter(self, phase, now):+ from .stage_estimate import STAGES+ if phase == self.phase:+ return+ self._late_history()+ if phase == 'load' and self.phase == 'upscale':+ # The model handed over between the passes belongs to the gap+ # between them, which is a stage of its own; a retry reloads+ # through _retry instead.+ returnif self.phase is not None:self.spent[self.phase] = max(0., now - self.phase_started)-- def _enter(self, phase, now):- """Phases do not always come one after the other: a preset check- runs for a moment while the prompt is encoded, and an encoder event- can follow the forecast. A phase that comes back goes on from the- time it already has, and only the phases before the one now running- are finished; a later one that ran for a moment keeps its forecast."""- self._close(now)+ self.phase_counts[self.phase] = self.count+ self.forecasts[self.phase] = self.weights[self.phase]+ # A brief model check can interrupt encoding, and loading can return+ # between passes. Only predecessors of the active phase are finished.+ self.completed = (set(STAGES[:STAGES.index(phase)]) | self.reused) - {phase}+ for key in STAGES:+ if key in self.completed and key in self.spent:+ self.measured[key] = self.weights[key] = max(.001, self.spent[key])+ elif key in self.measured:+ self.measured.pop(key)+ self.weights[key] = max(self.forecasts.get(key, 0.), self.spent.get(key, 0.))self.phase, self.phase_started = phase, now - self.spent.get(phase, 0.)- self.closed.discard(phase)- self._settle()-- def _settle(self):- before = self.ORDER[:self.ORDER.index(self.phase)] if self.phase else ()- self.completed = (set(before) | self.closed | self.reused) - {self.phase}- for name in self.ORDER:- seconds = self.spent.get(name)- if name in self.completed:- if seconds is not None:- self.measured[name] = self.weights[name] = max(.001, seconds)- elif name not in self.reused:- self.weights.pop(name, None) # Passed without running: nothing of it is left to wait for.- elif name in self.measured:- del self.measured[name]- self.weights[name] = max(self.forecasts.get(name, 0.), seconds or 0., .001)-- def _message_phase(self, message):- if message.get('timing_phase') in ('encoding', 'load', 'sampling', 'decode'):- return message['timing_phase']- phase = message.get('phase')- return {'sampling': 'sampling', 'sample_finalize': 'decode',- 'complete': 'decode'}.get(phase) if isinstance(phase, str) else None-- def annotate(self, message):- message = dict(message)- if isinstance(message.get('sampling_plan'), dict):- self.sampling_plan = message['sampling_plan']+ self.count = self.phase_counts.get(phase, self._initial_count())+ self.decode_remaining = None++ def _initial_count(self):+ from .stage_estimate import canvases, step_counts+ if self.phase == 'load':+ return (0, 50, 1)+ if self.phase in ('pass1', 'pass2'):+ count = step_counts(self.request)[0 if self.phase == 'pass1' else 1]+ return (0, count * 50, 1)+ if self.phase == 'decode_save':+ return (0, canvases(self.request)[2]['frames'], 1)+ return None++ def _set_count(self, done, total, margin=1):+ # Duplicate/delayed logs cannot undo already witnessed counts. Using+ # the smallest known unit avoids lowering the ceiling on the first+ # counted event after an uncounted stage-start notification.+ candidate = (max(0, done), total, margin)+ old = self.count+ if old is None or (candidate[0] + margin) / max(total, 1) >= (old[0] + old[2]) / max(old[1], 1):+ self.count = candidate++ def _phase_for(self, message, event):+ name = event.get('event')+ if self.retry_decode_only:+ return 'decode_save'+ if name == 'decode_resume' or message.get('phase') == 'sample_finalize':+ return 'decode_save'+ if message.get('stage') in ('latent_upscale', 'first_pass_reused'):+ return 'upscale'+ phase = message.get('timing_phase') or message.get('phase')+ if phase == 'encoding' and any(self.steps.values()):+ # After synchronized sampling steps, encoder-file lines are late+ # notifications from the finished prompt, not a new encoding run.+ return None+ if phase == 'sampling':+ # `loaded` falsely looks like sampling 0/N in the legacy converter.+ if name == 'loaded':+ return 'load'+ done = message.get('done', 0)+ base = self.request['base_steps']+ if self.request['flow'] == 'upscale' or self.retry_reuse_first:+ return 'pass2'+ if self.request['flow'] != 'preview' and self.request['two_pass']:+ if done > base or (done == base and (name == 'sampling_start' or message.get('stage') in ('layers', 'inputs'))):+ return 'pass2'+ return 'pass1'+ return {'encoding': 'encode', 'load': 'load', 'decode': 'decode_save'}.get(phase) if isinstance(phase, str) else None++ def _retry(self, retry, now):+ # A retry changes the plan; the earlier load no longer says how fast+ # this machine is, and measured steps arrive again soon.+ self.slow_start = False+ self.retry_decode_only = retry.get('reuse_sampling') is True+ self.retry_reuse_first = retry.get('reuse_first_pass') is True+ start = 'decode_save' if self.retry_decode_only else 'load'+ if not self.retry_decode_only:+ self.steps['pass2'].clear()+ if not retry.get('reuse_first_pass'):+ self.steps['pass1'].clear()+ from .stage_estimate import STAGES+ self.spent = {k: v for k, v in self.spent.items() if STAGES.index(k) < STAGES.index(start)}+ self.phase_counts = {}+ self.decode_part, self.decode_rest = None, None+ self.reused = set(STAGES[:STAGES.index(start)])+ self.completed = set(self.reused)+ if self.retry_reuse_first and not self.retry_decode_only:+ self.completed.add('pass1')+ self.reused.add('pass1')+ self.phase, self.phase_started = start, now+ self.count, self.decode_remaining = self._initial_count(), None+ self.live_calibration = False+ # Recovery can change placement. Wait for the new attempt's plan;+ # retained step durations are not new measurements of pending work.+ for key in ('pass1_resident_blocks', 'pass2_resident_blocks',+ 'decoder_resident_blocks', 'decoder_offload'):+ self.request.pop(key, None)+ self._reforecast(reset_source=True)+ self.ceiling_base = self.fraction+ self.ceiling_weights = {k: v for k, v in self.weights.items() if k not in self.completed}++ def _counts_and_live(self, message, event, now):+ from .stage_estimate import predict_live, step_counts+ phase = self.phase+ done, total = message.get('done'), message.get('total')+ if phase in ('pass1', 'pass2'):+ index = 0 if phase == 'pass1' else 1+ count = step_counts(self.request)[index]+ offset = self.request['base_steps'] if index else 0+ if type(done) is int:+ local_done = max(0, min(count, done - offset))+ blocks, block = message.get('blocks'), message.get('block')+ if type(blocks) is int and blocks > 0 and type(block) is int:+ # One step of margin: layer events are dispatch estimates,+ # steps are the synchronized counts (a first step may compile).+ self._set_count(local_done * blocks + max(0, block - 1), count * blocks,+ blocks if self.ceiling == 'stretch' else 1)+ else:+ # Keep the same one-layer margin across step boundaries;+ # changing from one step to one layer could lower a ceiling.+ self._set_count(local_done * 50, count * 50, 50 if self.ceiling == 'stretch' else 1)+ seconds = _timing_seconds(message.get('last_step_seconds'))+ if seconds and local_done > 0:+ self.steps[phase][local_done] = seconds+ if 1 in self.steps[phase] and 2 in self.steps[phase]:+ self.live_calibration = True+ elif phase == 'load':+ if event.get('event') == 'loaded':+ done, total = 50, 50+ elif event.get('event') == 'prepared_blocks':+ done, total = event.get('blocks'), 50+ elif event.get('event') == 'model_load_phase':+ done, total = event.get('done'), event.get('total')+ # LoRA groups describe a substage, not completion of the whole load.+ if type(done) is int and type(total) is int and total > 0 and message.get('unit') != 'bytes':+ self._set_count(done, total)+ elif phase == 'decode_save' and message.get('stage') == 'video':+ elapsed = _timing_seconds(message.get('elapsed_seconds'))+ if type(done) is int and type(total) is int and 0 <= done <= total and total > 0:+ self._set_count(done, total)+ if elapsed and done > 0:+ self.decode_remaining = elapsed * (total - done) / done + self._after_frames(total)+ self.weights[phase] = now - self.phase_started + self.decode_remaining+ self.remaining_source = 'measured'+ elif phase == 'decode_save' and (event.get('event') == 'decode_phase' or message.get('decode_part')):+ # These real transitions finish video frames even for a decoder+ # that does not stream progress callbacks.+ part = message.get('decode_part') or _DECODE_PARTS.get(event.get('phase') if isinstance(event.get('phase'), str) else '')+ if part in self.DECODE_PARTS and part != self.decode_part:+ rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):]+ self.decode_part = part+ if all(k in self.decode_parts for k in rest):+ self.decode_rest = sum(self.decode_parts[k] for k in rest[1:])+ self.decode_remaining = self.decode_parts[part] + self.decode_rest+ self.weights[phase] = now - self.phase_started + self.decode_remaining+ if part in ('audio_load_decode', 'encode'):+ total = self.count[1] if self.count else self.request['frames']+ self._set_count(total, total)+ if self.decode_remaining is None:+ # Without frame callbacks this is the first sign that the+ # video is decoded; only the measured tail remains.+ self.decode_remaining = self._after_frames(total)+ self.weights[phase] = now - self.phase_started + self.decode_remaining+ self.remaining_source = 'measured'+ # Only contiguous, actually completed steps calibrate the model.+ steps = {}+ for key, values in self.steps.items():+ steps[key] = []+ for i in range(1, len(values) + 1):+ if i not in values:+ break+ steps[key].append(values[i])+ if self.live_calibration and any(len(v) >= 2 for v in steps.values()):+ done_stages = [('decode' if k == 'decode_save' else k) for k in self.completed]+ live = predict_live(dict(hardware=self.hardware, request=self.request, cold_flags=self.cold_flags,+ pass1_steps=steps['pass1'], pass2_steps=steps['pass2'], completed_stages=done_stages,+ current_stage=phase, step_elapsed_seconds=message.get('step_elapsed_seconds', 0.),+ pass_wall_overhead_done=[k for k, v in steps.items() if v]), self.params)+ # With matching history on this machine, scale its own stage times+ # by the measured warm step instead of the general model.+ from statistics import median+ history, warm, ratio = self.local.get('stages') or {}, self.local.get('warm') or {}, None+ for key in ('pass2', 'pass1'):+ if len(steps[key]) >= 2 and warm.get(key):+ ratio = min(2., max(.5, median(steps[key][1:]) / warm[key]))+ break+ for key in ('pass1', 'upscale', 'pass2', 'decode_save'):+ if key in self.completed or (key == 'decode_save' and self.decode_remaining is not None):+ continue+ if ratio is not None and key in history:+ self.weights[key] = history[key] * (1. if key == 'decode_save' else ratio)+ continue+ remaining = live['stages'][key]['seconds']+ if key == phase:+ if key == 'decode_save':+ # decode/save prediction is a full duration until frames arrive.+ self.weights[key] = remaining+ else:+ self.weights[key] = now - self.phase_started + remaining+ else:+ self.weights[key] = remaining+ self.decode_tail = max(0., live['stages']['decode_save']['seconds'] - live['stages']['video_decode']['seconds'])+ self.remaining_source = 'measured'++ def _after_frames(self, frames):+ # Audio, MP4 and saving after the last decoded frame: about 2 s plus+ # 0.03 s per frame (measured, slightly below the median, so the bar+ # rarely jumps at saving), never more than the model's tail.+ if self.decode_rest is not None and self.decode_part == 'video_decode':+ return self.decode_rest+ return min(self.decode_tail, 2. + .03 * frames)++ def _slow_factor(self, now):+ # A disk-bound 16 GB machine that loads much slower than predicted is+ # slower in every later stage too (reports: about the square root of+ # the load ratio). Larger memory or a slow encode does not predict it.+ if not self.slow_start or self.remaining_source != 'similar':+ return 1.+ ram = self.hardware.get('ram_total_gib')+ if ram is None or ram > 16.5:+ return 1.+ predicted = self.prior['stages']['load']['seconds']+ actual = self.measured.get('load') or (now - self.phase_started if self.phase == 'load' else None)+ if not actual or predicted <= 1 or actual / predicted <= 2:+ return 1.+ return min(4., math.sqrt(actual / predicted))++ def _target(self, now):+ factor = self._slow_factor(now)+ later = ('pass1', 'upscale', 'pass2', 'decode_save')+ remaining = sum(v * (factor if k in later else 1.)+ for k, v in self.weights.items() if k not in self.completed and k != self.phase)+ elapsed = max(0., now - self.phase_started)+ # The stage under way is scaled too until its speed is measured.+ current = self.weights.get(self.phase, 0.) * (factor if self.phase in later else 1.) - elapsed+ if current < 0:+ current = max(3., .1 * elapsed)+ return max(0., remaining + current)++ def _ceiling(self, now):+ weights = (self.ceiling_weights if self.ceiling == 'prior' else+ {k: self.weights[k] for k in self.ceiling_weights})+ stretch = self.ceiling == 'stretch' and (self.count is None or self.count[0] <= 0)+ if stretch and self.phase in weights:+ # An overrunning stage without counted work keeps its end ahead of+ # it, by the same remainder the remaining-time target assumes.+ elapsed = max(0., now - self.phase_started)+ weights[self.phase] = max(weights[self.phase], elapsed + max(3., .1 * elapsed))+ total = sum(weights.values())+ if total <= 0:+ if self.ceiling in ('current', 'stretch'):+ self.last_ceiling = .99+ return .99+ credit = sum(v for k, v in weights.items() if k in self.completed)+ # An uncounted stage earns its allowance over time, not at its entry.+ # This also avoids releasing all of upscale's share at a boundary.+ duration = weights.get(self.phase, 0.)+ fraction = min(1., max(0., now - self.phase_started) / max(duration, .001))+ if self.count is not None and not stretch:+ done, count, margin = self.count+ fraction = min(1., max(0., (done + margin) / max(count, 1)))+ credit += duration * fraction+ value = min(.99, self.ceiling_base + (1. - self.ceiling_base) * credit / total)+ if self.ceiling in ('current', 'stretch'):+ self.last_ceiling = max(self.last_ceiling, value)+ value = self.last_ceiling+ return value++ def _smooth(self, dt):+ if self.smoothing == 'track':+ shown, target = self.remaining_shown, self.remaining_target+ if target > shown:+ # Slower than predicted: stop counting down, then rise gently.+ # A time that ran out starts again from the remainder an+ # overrunning stage keeps; from zero it could never rise.+ shown = max(shown, min(target, 3.))+ self.remaining_shown = min(target, shown + self.rate * shown * dt)+ else:+ shown = max(target, shown - dt)+ self.remaining_shown = max(0., shown - (shown - target) * (1. - math.exp(-dt / max(self.tau, .001))))+ return+ # Hold suppresses countdown only for an upward target correction.+ # Both policies approach the target without crossing it by correction;+ # countdown itself can pass below it. Keep a positive running remainder.+ if self.smoothing == 'hold' and self.remaining_target > self.remaining_shown:+ shown = self.remaining_shown+ else:+ floor = 3. if self.smoothing == 'countdown' else min(3., self.remaining_shown)+ shown = max(floor, self.remaining_shown - dt)+ limit = self.rate * shown * dt+ self.remaining_shown = max(0., shown + max(-limit, min(limit, self.remaining_target - shown)))++ def _render(self, message, now, *, heartbeat=False):+ if self.remaining_source == 'local' and self.shown_source == 'similar':+ # History read after the start replaces the estimate from similar+ # settings: this computer's record corrects it once, so do not smooth.+ # (A first measurement keeps smoothing; two steps are still noisy.)+ self.remaining_shown = self.remaining_target+ self.shown_source = self.remaining_source+ elapsed = max(0., now - self.started)+ candidate = elapsed / max(elapsed + self.remaining_shown, .001)+ if (self.final is not None and self.phase == 'decode_save' and self.count+ and self.count[0] >= self.count[1] > 0):+ # Every frame is decoded; only audio, MP4 and saving remain, often+ # for a few seconds only. Move there as fast as the rise limit allows.+ candidate = max(candidate, self.final)+ ceiling = self._ceiling(now)+ limit = .99+ # Retain the value in force one second ago and all newer changes.+ while len(self.rise_window) > 1 and self.rise_window[1][0] <= now - 1.:+ self.rise_window.popleft()+ if self.max_rise is not None:+ base = self.rise_window[0][1] if self.rise_window else self.fraction+ limit = min(limit, base + self.max_rise)+ self.fraction = max(self.fraction, min(limit, candidate, ceiling))+ if not self.rise_window or self.rise_window[-1][1] != self.fraction:+ if len(self.rise_window) > 1 and self.rise_window[-1][0] == now:+ self.rise_window.pop()+ self.rise_window.append((now, self.fraction))+ overall = dict(status='complete' if self.finished else 'running', estimated=True,+ fraction=1. if self.finished else self.fraction,+ remaining_seconds=0. if self.finished else self.remaining_shown,+ elapsed_seconds=elapsed, remaining_source=self.remaining_source,+ phase_start_fraction=self.fraction, phase_weight=0., remaining_floor_seconds=0.,+ at_ceiling=not self.finished and candidate >= ceiling,+ sampling_time_weighted=True)+ key = self.phase+ if key in self.steps:+ key += '_first' if not self.steps[key] else '_warm'+ if key in self.typical:+ overall.update(typical_seconds=self.typical[key], typical_source='local')+ if heartbeat:+ overall['heartbeat'] = True+ self.last_publish = now+ return dict(message, overall=overall)++ def annotate(self, message, *, event=None):+ message, event = dict(message), event or {}+ now = self.clock()+ if self.finished:+ return self._render(message, now)+ dt = max(0., now - self.last_tick)+ old_phase, old_part = self.phase, self.decode_part+ waiting_for_pass = (self.phase in self.steps and self.count is not None+ and self.count[0] >= self.count[1] > 0)+ plan = event if event.get('event') == 'sampling_plan' else message.get('sampling_plan')+ if isinstance(plan, dict):+ self.configure(plan=plan)if isinstance(message.get('prediction'), dict):self.forecast(message['prediction'])- retry = message.get('retry') if isinstance(message.get('retry'), dict) else {}- if retry.get('reuse_sampling') is True:- self.reused.add('sampling')- self.completed.add('sampling')- phase = self._message_phase(message)- now = self.clock()- if phase and phase != self.phase:+ retried = isinstance(message.get('retry'), dict) or message.get('stage') == 'preview_mismatch'+ if isinstance(message.get('retry'), dict):+ self._retry(message['retry'], now)+ if message.get('stage') == 'preview_mismatch':+ self.request['flow'] = 'full'+ self._retry({}, now)+ if event.get('event') in ('video_start', 'decode_resume'):+ self.configure(placement=event)+ phase = self._phase_for(message, event)+ if phase:self._enter(phase, now)- if phase == 'sampling':- self._sampling_estimate(message, now)- if phase == 'decode' and message.get('stage') == 'video':- done, total = message.get('done'), message.get('total')- elapsed = _timing_seconds(message.get('elapsed_seconds'))- if elapsed and type(done) is int and type(total) is int and 0 < done <= total:- video_seconds = elapsed * total / done- # Reports put audio/MP4/post-decode cleanup at about 6-8% of- # video decode time. Retain this tail until audio/save starts.- self.decode_tail = max(video_seconds * .08, self.decode_rest or 0.)- self.weights['decode'] = now - self.phase_started + video_seconds - elapsed + self.decode_tail- self.forecast_ready = True- elif phase == 'decode':- self.decode_tail = 0.- part = message.get('decode_part')- rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] if part in self.DECODE_PARTS else ()- if rest and all(name in self.decode_parts for name in rest):- # The decoder says which of its steps begins: what is left of- # the phase is this step and the ones after it, not the whole- # forecast made before the run.- self.decode_rest = sum(self.decode_parts[name] for name in rest[1:])- self.weights['decode'] = now - self.phase_started + self.decode_parts[part] + self.decode_rest+ if phase in self.steps and phase != old_phase and phase not in self.spent:+ # A first layer reports time inside its step, not the model+ # handover before it. Backdate only that witnessed work.+ inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0.+ self.phase_started = now - inside+ if waiting_for_pass:+ dt = min(dt, inside)+ if event.get('event') == 'conditioning_cache_hit':+ self.cold_flags['encode_cached'] = True+ if event.get('event') == 'conditioning_reused':+ self.cold_flags['encode_cached'] = True+ if event.get('event') == 'encoder_device_reuse':+ self.cold_flags['encoder_cold'] = False+ if event.get('event') == 'compute_device':+ self.hardware.update(gpu_name=event.get('name') or self.hardware.get('gpu_name'))+ if _timing_seconds(event.get('vram_total_bytes')):+ self.hardware['vram_total_gib'] = event['vram_total_bytes'] / 2**30+ self._reforecast()+ if not retried:+ self._counts_and_live(message, event, now)+ self.remaining_target = self._target(now)+ self._smooth(dt)+ if retried:+ # A new attempt is estimated afresh: the time left of the attempt+ # that failed says nothing about it, and may already have run out.+ self.remaining_shown = self.remaining_target+ if self.phase == 'decode_save' and self.decode_rest is not None and (+ self.decode_part != old_part or message.get('stage') == 'video'):+ # A decoder boundary gives the pending substeps directly; the+ # fraction still obeys its monotonic, per-second rise limit.+ self.remaining_shown = self.remaining_target+ self.last_tick = self.last_event = nowself.total = sum(self.weights.values())- if message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete':- message['overall'] = {'status': 'complete', 'fraction': 1., 'estimated': False,- 'elapsed_seconds': now - self.started, 'remaining_seconds': 0.}- return message- overall = {'status': 'running', 'estimated': self.forecast_ready,- 'elapsed_seconds': now - self.started}- if self.forecast_ready and self.total > 0:- weight = self.weights.get(self.phase, 0.)- elapsed = max(0., now - self.phase_started)- seconds = self.weights.get(self.phase)- remaining = max(0., self.total - self._base() - min(elapsed, seconds or 0.))- future = sum(value for name, value in self.weights.items()- if name not in self.completed and name != self.phase)- total = self.total- if weight and elapsed > .95 * weight and .95 * future > .05 * self._base():- # A phase that outruns its forecast (a model read from a cold- # disk) is given the time it has taken, so the fraction keeps- # rising instead of standing at the forecast's end; it does so- # only while enough of the run is still ahead for that to- # raise it. The time left is unchanged: what follows is.- seconds = elapsed / .95- total += seconds - weight- weight = seconds- start = self._base() / total- fraction = start- if weight:- fraction += weight / total * min(.95, elapsed / max(seconds, 1e-9))- # What has been given out is not taken back. A phase that- # returns (a model loaded again between the passes), a decoder- # step that ran past its forecast and a retry all add time- # left; the fraction waits for the run to catch up with it.- fraction = self.shown = max(self.shown, fraction)- overall.update(fraction=fraction, phase_start_fraction=start,- phase_weight=weight / total,- phase_elapsed_seconds=elapsed, phase_seconds=seconds,- remaining_seconds=remaining,- remaining_floor_seconds=max(future, self.decode_tail),- sampling_time_weighted=self.sampling_time_weighted)- message['overall'] = overall- return message+ self.finished = message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete'+ if self.finished:+ self.measured[self.phase] = max(.001, now - self.phase_started)+ self.weights[self.phase] = self.measured[self.phase]+ self.total = sum(self.weights.values())+ self.completed.update(self.weights)+ self.remaining_target = self.remaining_shown = 0.+ self.fraction = 1.+ if message.keys() - {'report_id'}:+ self.last_message = {k: v for k, v in message.items() if k not in ('overall', 'reset', 'new_request', 'prediction', 'sampling_plan')}+ else:+ message = dict(self.last_message, **message)+ return self._render(message, now)++ def observe(self, event):+ """Use an engine event that has no progress message, without publishing."""+ published, window = self.last_publish, self.rise_window.copy()+ self.annotate({}, event=event)+ self.last_publish, self.rise_window = published, window++ def heartbeat(self):+ now = self.clock()+ if self.finished or now - self.last_publish < 1.:+ return None+ if (self.phase in self.steps and self.count is not None+ and self.count[0] >= self.count[1] > 0):+ # Between passes no pending step time is being spent.+ self.last_tick = now+ elif self.silence == 'stage':+ # Admit only the part of this interval inside the predicted stage.+ # Crossing its end freezes at the boundary, even with sparse polls.+ end = self.phase_started + self.weights.get(self.phase, 0.)+ dt = max(0., min(now, end) - max(self.last_tick, self.phase_started))+ if dt:+ self.remaining_target = max(0., self.remaining_target - dt)+ self._smooth(dt)+ self.last_tick = max(self.last_tick, min(now, end))+ # Silent time is not counted down until the next engine event shows+ # the engine is still working; that event then counts all of it.+ return self._render(self.last_message, now, heartbeat=True)def source_root():@@ -550,10 +1020,7 @@ def progress_message(event):'lora_prepare_start': 'Loading LoRAs','resource_retry': 'Adjusting memory placement and retrying'}if name == 'decode_phase':- message = {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'}- if isinstance(event.get('phase'), str) and event['phase'] in _DECODE_PARTS:- message['decode_part'] = _DECODE_PARTS[event['phase']]- return message+ return {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'}if name == 'decode_progress':return dict(label='Decoding the video', timing_phase='decode', phase='decode', stage='video',done=event.get('done'), total=event.get('total'), unit='frames',@@ -782,12 +1249,22 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,state['encoder_prewarm'] = prewarm_summary(encoder_prewarm)save(run / 'comfy-request.json', state)started = time.monotonic()- whole_progress = WholeVideoProgress()+ try:+ whole_progress = WholeVideoProgress.for_request(machine, canvas, output_directory,+ base_steps=base_steps, refine_steps=refine_steps, two_pass=two_pass,+ preview=preview, upscale=bool(upscale), encoder_prewarm=state['encoder_prewarm'])+ except Exception:+ # A progress estimate cannot prevent generation, even without its parameters.+ try:+ whole_progress = WholeVideoProgress()+ except Exception:+ whole_progress = _PlainProgress()+ whole_progress = _SafeProgress(whole_progress)from .comfy_progress import REPORTSreport_id = REPORTS.register(output)- def send_progress(message):+ def send_progress(message, event=None):if progress:- progress(whole_progress.annotate(dict(message, report_id=report_id)))+ progress(whole_progress.annotate(dict(message, report_id=report_id), event=event))process = Nonefrom .result_cache import ResultCache, request_key, inspect_boundedcache = ResultCache(output_directory)@@ -802,8 +1279,14 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,tails[2].reset()message = progress_message(event)if message:- send_progress(message)+ send_progress(message, event)+ else:+ whole_progress.observe(event)+ heartbeat = whole_progress.heartbeat()+ if progress and heartbeat:+ progress(heartbeat)try:+ send_progress({'label': 'Preparing video'})if interrupted:interrupted()extra = {}@@ -889,10 +1372,9 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,state['sampling_cache_install'] = preparationif preparation['seconds']:started += preparation['seconds']- whole_progress = WholeVideoProgress()send_progress({'reset': True, 'label': 'Preparing video'})- whole_progress.sampling_plan = planned- whole_progress.forecast(progress_history_forecast(root, machine, dict(sampled, sampling_plan=planned)))+ whole_progress.configure(plan=planned, media=extra.get('media', {}),+ conditioning=bool(extra.get('conditioning')), sampled=sampled)send_progress({'label': 'Preparing %.3f s video + audio · %d × %d' %(canvas['seconds'], width, height)})if release_models:diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.pyindex fb0cd9b..32126c3 100644--- a/freevideo_engine/comfy_progress.py+++ b/freevideo_engine/comfy_progress.py@@ -83,7 +83,18 @@ class ProgressState:old = self.rows.get(node)metadata_only = ('compatibility' in value and not value.get('new_request')and not any(key in value for key in ('phase', 'stage', 'timing_phase', 'label', 'reset')))- if old and metadata_only:+ # A heartbeat carries a newer estimate, not engine progress (Harper's+ # bridge repeats the last message's fields with a fresh overall).+ heartbeat = isinstance(value.get('overall'), dict) and value['overall'].get('heartbeat') is True+ overall_measured = now+ if old and heartbeat:+ # Replace only overall: the row and its age keep describing the+ # last engine message, so a reopened page still sees a quiet engine.+ value = dict(old['message'], overall=value['overall'])+ value.pop('new_request', None)+ value.pop('reset', None)+ measured = old['measured']+ elif old and metadata_only:# A settings notice is not engine progress. Keep the current# activity and its timestamp, including in reopened snapshots.value = dict(old['message'], compatibility=value['compatibility'])@@ -91,11 +102,11 @@ class ProgressState:value.pop('reset', None)if isinstance(value.get('overall'), dict) and 'reset' in value['overall']:value['overall'] = {key: item for key, item in value['overall'].items() if key != 'reset'}- measured = old['measured']+ measured, overall_measured = old['measured'], old.get('overall_measured', old['measured'])elif old and value.get('warning'):# A memory warning must not replace the current sampling step.value = dict(old['message'], warning=value['warning'])- measured = old['measured']+ measured, overall_measured = old['measured'], old.get('overall_measured', old['measured'])else:measured = now# The page starts a new request on these too and clears its notes.@@ -134,16 +145,22 @@ class ProgressState:self._record_timeline(node, value, message, now, measured)self.sequence += 1value.update(node=node, stream_id=self.stream_id, sequence=self.sequence)- self.rows[node] = {'message': value, 'measured': measured}+ self.rows[node] = {'message': value, 'measured': measured, 'overall_measured': overall_measured}while len(self.rows) > 32:oldest = min(self.rows, key=lambda key: self.rows[key]['message']['sequence'])del self.rows[oldest]- return dict(value, age_seconds=max(0., now - measured))+ return self._aged(value, now, measured, overall_measured)++ @staticmethod+ def _aged(message, now, measured, overall_measured):+ # age_seconds: since the last engine message; overall_age_seconds: since+ # the estimate in `overall` (a heartbeat refreshes only that one).+ return dict(message, age_seconds=max(0., now - measured), overall_age_seconds=max(0., now - overall_measured))def snapshot(self):with self.lock:now = self.clock()- return [dict(deepcopy(row['message']), age_seconds=max(0., now-row['measured']))+ return [self._aged(deepcopy(row['message']), now, row['measured'], row.get('overall_measured', row['measured']))for row in self.rows.values()]diff --git a/freevideo_engine/effort_forecast.py b/freevideo_engine/effort_forecast.pyindex a94108c..71d959b 100644--- a/freevideo_engine/effort_forecast.py+++ b/freevideo_engine/effort_forecast.py@@ -10,6 +10,7 @@ import mathimport platformfrom pathlib import Pathfrom statistics import median+import threadingimport time@@ -48,7 +49,8 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):if len(attempts) > 1:return Nonepasses = video.get('sampling_passes') if plan.get('enabled') else [video]- if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') else 1):+ preview = bool(report.get('preview') or plan.get('preview'))+ if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') and not preview else 1):return Nonemeasured = []for part in passes:@@ -61,7 +63,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):if sum(p['seconds'] for p in measured) > total * 1.01:return Nonebase = plan.get('base_steps', video.get('config', {}).get('steps', 8))- refine = plan.get('refine_steps', 0) if plan.get('enabled') else 0+ refine = plan.get('refine_steps', 0) if plan.get('enabled') and not preview else 0if measured[0]['steps'] != base or (refine and measured[1]['steps'] != refine):return Noneencoding = report.get('encoding') or {}@@ -118,7 +120,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True):version=version if isinstance(version, str) and len(version) <= 64 else None,in_use=in_use,reference_rows=reference_rows,- flow='preview' if report.get('preview') or plan.get('preview') else 'full', stages=stages,+ flow='preview' if preview else 'full', stages=stages,sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)},references=task_has_references(video.get('config', {}).get('task', 't2va')),model_resident=video.get('resident_engine_cache_hit'))@@ -130,7 +132,7 @@ def estimate(records, canvas, *, base_steps=8, refine_steps=3, two_pass=True, tavalidate_steps(base_steps, refine_steps, two_pass)candidates = []for row in records:- if (row['task'] != task or row['adapters'] != adapters+ if (row.get('flow', 'full') != 'full' or row['task'] != task or row['adapters'] != adaptersor row['shape'] != {k: canvas[k] for k in ('width', 'height', 'frames')}):continue# Both passes retain their own resolution costs. Never multiply the@@ -319,6 +321,76 @@ def _stage_forecast(records, request, cold_flags, prior):return result+_START_CONTEXT = {}+# Guards the entries above: only one background read per key, and readers+# always see a complete hardware/records pair. File IO stays outside it.+_START_LOCK = threading.Lock()+++def _context_key(output_directory, machine):+ device = machine.get('device_identity') if machine.get('device_backend') == 'mps' else machine.get('gpu_uuid')+ system = machine.get('system') or platform.system()+ return (str(output_directory), json.dumps(device, sort_keys=True), system, machine.get('setup_run')), device, system+++def start_context(output_directory, machine, *, timeout=.035):+ """Bound foreground IO to ~35 ms, even on slow/unavailable output disks.++ One daemon per key reads in the background. Its last result stays in use+ while a newer read runs (at most one every 15 s), so a request that starts+ minutes after the previous one still sees that run. A request that started+ before a read finished can pick it up later (history_read_at).+ No GPU calls, resident RPC, model scans, hashing or directory-wide cache+ inspection. Installation inventory describes identity, not live free RAM.+ """+ from threading import Event, Thread+ key, device, system = _context_key(output_directory, machine)+ start = False+ with _START_LOCK:+ entry = _START_CONTEXT.get(key)+ if entry is None:+ entry = dict(ready=Event(), started=None, read_at=None, hardware={}, records=[], reading=False)+ _START_CONTEXT[key] = entry+ now = time.monotonic()+ if not entry['reading'] and (entry['started'] is None or now - entry['started'] > 15):+ entry['reading'], entry['started'], start = True, now, True+ if start:+ def read():+ hardware, records = dict(entry['hardware']), list(entry['records'])+ try:+ if machine.get('setup_run'):+ with (Path(machine['setup_run']) / 'plan.json').open('rb') as stream:+ raw = stream.read(1024 * 1024 + 1)+ if len(raw) <= 1024 * 1024:+ found = json.loads(raw).get('inventory', {}).get('hardware', {})+ if same_device(found, device):+ hardware = {k: found[k] for k in+ ('gpu_name', 'capability', 'ram_total', 'vram_total', 'system') if k in found}+ except (OSError, ValueError, TypeError, AttributeError):+ pass+ try:+ records = local_records(output_directory, device, system=system, stage_details=True)+ except (OSError, ValueError, TypeError, AttributeError):+ pass+ finally:+ with _START_LOCK:+ entry['hardware'], entry['records'] = hardware, records+ entry['read_at'], entry['reading'] = time.monotonic(), False+ entry['ready'].set()+ Thread(target=read, name='freevideo-progress-history', daemon=True).start()+ if timeout and not entry['ready'].is_set():+ entry['ready'].wait(timeout)+ with _START_LOCK:+ return dict(entry['hardware']), list(entry['records'])+++def history_read_at(output_directory, machine):+ """Monotonic time the last background read finished, or None."""+ with _START_LOCK:+ entry = _START_CONTEXT.get(_context_key(output_directory, machine)[0])+ return entry and entry['read_at']++def local_records(output_directory, gpu_uuid, *, system=None, stage_details=False, refresh=False):"""Bounded recent JSON reads, cached between slider changes; never open MP4s."""system = system or platform.system()@@ -338,7 +410,10 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Falsexcept OSError:continuerecords = []- for _, path in sorted(paths, reverse=True)[:48]:+ # Slider estimates keep their original window of the 48 newest files; the+ # progress history reads on until it has 48 usable rows.+ candidates = sorted(paths, reverse=True)+ for _, path in (candidates if stage_details else candidates[:48]):try:with path.open('rb') as stream:data = stream.read(4 * 1024 * 1024 + 1)@@ -347,6 +422,8 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Falsrow = timing_record(json.loads(data), gpu_uuid, system, stage_details=stage_details)if row:records.append(row)+ if stage_details and len(records) == 48:+ breakexcept (OSError, ValueError, TypeError, AttributeError, KeyError):continueif not stage_details:diff --git a/freevideo_engine/progress_timeline.py b/freevideo_engine/progress_timeline.pyindex bb2bab8..7fb3870 100644--- a/freevideo_engine/progress_timeline.py+++ b/freevideo_engine/progress_timeline.py@@ -290,7 +290,7 @@ WARNINGS = frozenset((SEGMENT_KEYS = ('phase', 'stage', 'timing_phase', 'label_key')ENUMS = dict(phase=PHASES, stage=STAGES, timing_phase=TIMING_PHASES,label_key=LABEL_KEYS, unit=UNITS, warning=WARNINGS)-OVERALL_ENUMS = dict(status=STATUSES)+OVERALL_ENUMS = dict(status=STATUSES, remaining_source=frozenset(('local', 'similar', 'measured', 'other')))RETRY_ENUMS = dict(kind=RETRY_KINDS, failed_phase=PHASES)COUNTERS = frozenset(('done', 'total', 'block', 'blocks', 'reference_trimmed'))FLAGS = frozenset(('low_memory', 'kernel_cache_note', 'estimated', 'new_request', 'reset', 'result'))@@ -349,7 +349,7 @@ def _fields(value, *, strict=False, nested=None):(nested is None and (key in COUNTERS or key in FLAGS or key in SECONDS or key in FRACTIONSor key in ('t', 'bytes_per_second', 'overall', 'retry'))) or(nested == 'overall' and (key in OVERALL_SECONDS or key in OVERALL_FRACTIONS- or key in ('estimated', 'sampling_time_weighted'))) or+ or key in ('estimated', 'sampling_time_weighted', 'at_ceiling'))) or(nested == 'retry' and key in ('attempt', 'max_attempts', 'reuse_sampling')))if not known:# Heartbeat and the bridge's extra overall fields are intentionally omitted.@@ -370,8 +370,9 @@ def _fields(value, *, strict=False, nested=None):raise ValueError('Invalid progress mapping')result[key], count = _fields(raw, strict=strict, nested=key)errors += count- elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted'):- if (strict or key == 'sampling_time_weighted') and type(raw) is not bool:+ elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted',+ 'at_ceiling'):+ if (strict or key in ('sampling_time_weighted', 'at_ceiling')) and type(raw) is not bool:raise ValueError('Invalid progress flag')result[key] = bool(raw)elif (nested is None and key in COUNTERS) or key in ('attempt', 'max_attempts'):diff --git a/freevideo_engine/stage_estimate.py b/freevideo_engine/stage_estimate.pyindex c06f1cb..5b5dded 100644--- a/freevideo_engine/stage_estimate.py+++ b/freevideo_engine/stage_estimate.py@@ -124,6 +124,11 @@ def canvases(r):first = dict(full, width=32*uw*m, height=32*uh*m)else:first = dict(full)+ # two_pass.plan() runs a first pass under SMALL_FIRST_PASS_PIXELS (138,240) as one pass at the+ # target size, except when the first pass itself is wanted (a preview or the upscale of one).+ if (not sg.get('first') and two and r.get('flow') not in ('preview', 'upscale')+ and first != full and first['width']*first['height'] < 138240):+ return full, full, fullsecond = dict(full, **(sg.get('second') or {}))decode = first if r.get('flow') == 'preview' else fullreturn first, second, decodediff --git a/web/autopilot_chat.js b/web/autopilot_chat.jsindex 4f04116..09f68fb 100644--- a/web/autopilot_chat.js+++ b/web/autopilot_chat.js@@ -1,7 +1,7 @@// The conversation with the connected agent: what the user says, what the agent answers, and the agent's jobs as// cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page.import {agentGet, agentPost} from './agent_session.js';-import { estimatedFraction, remainingShown } from './generation_progress.js';+import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js';const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css);@@ -34,6 +34,28 @@ export function jobEstimate(job, age) {// again: the rest of the way is drawn shorter, so the level neither waits nor goes back, and is full at the end.const DRIFT = .25, RISE = .0005;export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1});+// The time left a running job shows, by the creation panel's rule: once measured never raised, and gone once its last+// minute has passed. `row` is what the job showed before (none yet: it starts at the engine's number); `seconds` is null+// from the moment the time has run out.+export function jobLeft(row, sent, source, time) {+ if (!row) {+ if (!(sent > 0)) return null;+ row = {seconds: sent, at: time, minute: null, last: 0, source: null, above: 0};+ }+ if (row.seconds === null) return row;+ // Before the run is measured the engine's number is an estimate from earlier runs: follow it up as well as down.+ const named = typeof source === 'string', initial = source === 'similar';+ // The first number that is not that estimate replaces it once, up or down, and its last minute starts over.+ if (named && !initial && row.source === 'similar' && sent > 0) { row.seconds = sent; row.minute = null; }+ if (named) row.source = source;+ const target = sent > 0 ? sent : null; row.above = remainingAbove(row.above, row.seconds, target, time - row.at);+ row.seconds = remainingShown(row.seconds, target, time - row.at, initial, row.above); row.at = time;+ if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; }+ // An estimate that rises out of its last minute starts that minute over.+ else if (row.seconds >= 60 && initial) row.minute = null;+ if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null;+ return row;+}export function jobLevel(row, base, left, time) {if (row.top === null || base > row.top + RISE) {if (row.top !== null && row.top < 1) row.scale = (1 - row.value) / (1 - row.top);@@ -140,21 +162,13 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = ()if (job.status === 'running') jobLevel(row.level, jobEstimate(row.job, time - row.at), row.job.progress?.remaining_seconds, time);return row.level.value;}- // The time left each job shows, by the creation panel's rule: never raised, and gone once its last minute has passed.const times = new Map();function leftOf(job) {- const time = performance.now() / 1000, sent = job.progress?.remaining_seconds;if (job.status !== 'running') { times.delete(job.id); return null; }- let row = times.get(job.id);- if (!row) {- if (!(sent > 0)) return null;- times.set(job.id, row = {seconds: sent, at: time, minute: null, last: 0});- }- if (row.seconds === null) return null;- row.seconds = remainingShown(row.seconds, sent > 0 ? sent : null, time - row.at); row.at = time;- if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; }- if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null;- return row.seconds;+ const row = jobLeft(times.get(job.id), job.progress?.remaining_seconds,+ job.progress?.estimate?.overall?.remaining_source, performance.now() / 1000);+ if (row) times.set(job.id, row);+ return row ? row.seconds : null;}const active = () => !stopped && element.isConnected && !document.hidden && shown();diff --git a/web/generation_progress.js b/web/generation_progress.jsindex 2fee372..bb281d4 100644--- a/web/generation_progress.js+++ b/web/generation_progress.js@@ -216,17 +216,24 @@ export function referenceTrimText(row, t) {`${name} 比生成的视频长,已截取前 ${used} 秒${rule}`);}-// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Never raised:-// an estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute"-// early. A lower one is eased into.-export function remainingShown(shown, target, dt) {+// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Once measured, an+// estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute"+// early; one that has stayed well above it for a minute (`above` seconds so far) raises it gently, or a run+// measured much slower would leave a low number standing for minutes. A lower one is eased into. An initial+// estimate (made before this run was measured) is followed both ways: a slow machine must not keep a low number.+export function remainingShown(shown, target, dt, initial = false, above = 0) {if (target === null) return Math.max(0, shown - dt);+ if (initial) return shown + (target - shown) * (1 - Math.exp(-dt / 15));if (target < shown - dt) {const counted = Math.max(0, shown - dt);return counted + (target - counted) * (1 - Math.exp(-dt / 15));}- return target < shown ? target : shown;+ if (target < shown) return target;+ return above >= 60 ? Math.min(target, shown + Math.max(1, shown * .003) * dt) : shown;}+// The seconds the engine's estimate has stayed well above the shown time (a quarter more), `dt` seconds on.+export const remainingAbove = (above, shown, target, dt) =>+ target !== null && target > Math.max(shown * 1.25, shown + 30) ? above + dt : 0;// The whole-video fraction the engine's last message stands for now: its estimate moved on by the seconds since the// step sample (stateAge) and the whole-video sample (overallAge) arrived. Null when the message carries none.export function estimatedFraction(state, overall, stateAge, overallAge) {@@ -294,6 +301,13 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f// Mac: the unified-memory notice of this request, and whether its first pass has ended.let memoryTight = null, firstPassOver = false;let state = {}, overall = {}, started = 0, sampledAt = 0, overallAt = 0, timer = null, elapsedShown = 0;+ // Where the bridge's remaining time comes from: 'similar' (no record of these+ // settings here yet), 'local' (this computer's record) or 'measured' (this run).+ // Absent from older bridges, which keep the two-steps rule.+ let estimateSource = null, etaChangedAt = -Infinity;+ // The page shows its own "preparing" row at the click; the engine's first+ // message for that request follows and must not restart the elapsed time.+ let placeholder = false, elapsedLead = 0;let lastProgressAt = 0, previousProgress = null, segmentStartedAt = 0, segmentKey = '';let pauseAnnounced = false, lastExplanation = '', announcedMessages = new Set();let announcementRecord = null;@@ -301,7 +315,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = flet completedSteps = 0, previousRemaining = null, remainingAt = 0, lastActivity = null;// Once the shown time reaches zero before the video is saved, it stays hidden for this request.// The last minute runs on real time even while an estimate holds the number.- let remainingRetired = false, lastMinuteAt = null, lastMinute = 0;+ let remainingRetired = false, lastMinuteAt = null, lastMinute = 0, aboveSeconds = 0;let shownFraction = null;// The visible percentage counts toward its value with the bar instead of// jumping. Accessible values are set exactly and immediately elsewhere.@@ -600,7 +614,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f}// Preserve the server's heartbeat/floor policy for the raw target only.- // The displayed estimate has its own clock and never increases.+ // The displayed estimate has its own clock; once it is no longer initial it never increases.function rawRemaining() {if (!valid(overall.remaining_seconds)) return null;return hasHeartbeat ? overall.remaining_seconds : Math.max(@@ -608,8 +622,25 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = foverall.remaining_seconds - Math.max(0, (now() - overallAt) / 1000));}function observeRemaining(message) {- if (completedSteps < 2 || remainingSeconds !== null || remainingRetired || !message.overall) return;+ if (!message.overall) return;+ const source = typeof message.overall.remaining_source === 'string' ? message.overall.remaining_source : null;const raw = valid(message.overall.remaining_seconds) ? rawRemaining() : null;+ if (source) {+ // The bridge estimates the whole request from the first message:+ // show it at once. Leaving the initial (similar-settings) estimate replaces+ // the number once, up or down; after that it only falls.+ const leaving = estimateSource === 'similar' && source !== 'similar';+ estimateSource = source;+ // The correction starts the last minute over. Once the shown time has run+ // out it stays hidden for this attempt, as without an initial estimate.+ if (leaving && raw !== null && !remainingRetired) { lastMinuteAt = null; lastMinute = 0; }+ if (raw === null || remainingRetired) return;+ // A retry's fresh estimate still waits for the 5 s spacing of the shown+ // minutes; only a new request starts the line over (in update()).+ if (remainingSeconds === null || leaving) { remainingSeconds = raw; remainingAt = now(); }+ return;+ }+ if (completedSteps < 2 || remainingSeconds !== null || remainingRetired) return;// Compare distinct received messages, never repeated timer ticks.// Zero is stable only against another zero.if (raw !== null && previousRemaining !== null@@ -620,7 +651,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f}function resetRemaining() {completedSteps = 0; previousRemaining = null; remainingSeconds = null; remainingAt = now();- remainingRetired = false; lastMinuteAt = null; lastMinute = 0;+ remainingRetired = false; lastMinuteAt = null; lastMinute = 0; aboveSeconds = 0;}function tick() {if (element.hidden) { drawStall(0, true); return; }@@ -631,12 +662,16 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f// The page counts from its first message until the engine's own elapsed time// arrives; that switch never moves the shown time back within a request.const seconds = elapsedShown = Math.max(elapsedShown,- valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge : (now()-started)/1000);+ valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge + elapsedLead : (now()-started)/1000);elapsed.textContent = `${t('Elapsed', '已用时')} ${duration(Math.max(0, seconds))}`;if (ended) remainingSeconds = null;if (remainingSeconds !== null) {- remainingSeconds = remainingShown(remainingSeconds, rawRemaining(), Math.max(0, (now() - remainingAt) / 1000));+ const sent = rawRemaining(), passed = Math.max(0, (now() - remainingAt) / 1000);+ aboveSeconds = remainingAbove(aboveSeconds, remainingSeconds, sent, passed);+ remainingSeconds = remainingShown(remainingSeconds, sent, passed, estimateSource === 'similar', aboveSeconds);if (remainingSeconds < 60 && lastMinuteAt === null) { lastMinuteAt = now(); lastMinute = remainingSeconds; }+ // An initial estimate may rise again: its last minute starts over.+ else if (remainingSeconds >= 60 && estimateSource === 'similar') lastMinuteAt = null;// A run slower than estimated would leave "less than a minute" up for minutes:// once that minute has passed, or the time reaches zero, stop showing it.if (!ended && (remainingSeconds <= 0@@ -646,9 +681,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f}remainingAt = now();eta.hidden = remainingSeconds === null;- eta.textContent = remainingSeconds === null ? '' : remainingSeconds >= 60+ const etaText = remainingSeconds === null ? '' : remainingSeconds >= 60? t(`About ${Math.ceil(remainingSeconds / 60)}\u00a0min left`, `预计还需约 ${Math.ceil(remainingSeconds / 60)}\u00a0分钟`): t('Less than a minute left', '预计还需不到 1\u00a0分钟');+ const initial = remainingSeconds !== null && estimateSource === 'similar';+ // While it follows the bridge, the shown minutes change at most every 5 s.+ if (eta.textContent !== etaText && (!initial || !eta.textContent || now() - etaChangedAt >= 5000)) {+ eta.textContent = etaText; etaChangedAt = now();+ }elapsed.hidden = false; times.hidden = elapsed.hidden && eta.hidden;drawStall(age, ended);drawActivity(age, ended);@@ -662,8 +702,21 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f// not a stage change or something the user needs to act on.if (message.warning === 'ram_budget_warning' && !message.phase) return;const reset = newRequest(message);- if (element.hidden || reset) {- state = {}; overall = {}; started = now(); overallAt = now(); shownFraction = null; elapsedShown = 0;+ const placeholderReset = element.hidden || reset;+ const terminal = ['complete', 'failed', 'cancelled'].includes(message.phase);+ if (placeholderReset) {+ // The engine's first message after the page's own placeholder is the+ // same click: keep counting from it. The placeholder lasts from this+ // node's own start until any engine message, its end, or hide().+ const sameClick = placeholder && message.stream_id !== undefined;+ // From then on the engine's elapsed time is offset by the wait before it began.+ const firstAt = valid(message.received_at) ? message.received_at : now();+ elapsedLead = sameClick ? Math.max(0, (firstAt - started) / 1000+ - (valid(message.overall?.elapsed_seconds) ? message.overall.elapsed_seconds : 0)) : 0;+ if (!sameClick) { started = now(); elapsedShown = 0; }+ placeholder = message.stream_id === undefined && !terminal;+ state = {}; overall = {}; overallAt = now(); shownFraction = null; estimateSource = null;+ eta.textContent = ''; etaChangedAt = -Infinity;lastProgressAt = valid(message.received_at) ? message.received_at : now();previousProgress = null; segmentKey = ''; segmentStartedAt = lastProgressAt;pauseAnnounced = false; lastExplanation = ''; announcedMessages.clear();@@ -674,21 +727,32 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = fsamplingPlan = {}; secondPass = false; hasHeartbeat = false;announcementRecord = null;}+ if (!placeholderReset && (message.stream_id !== undefined || terminal)) placeholder = false;const record = announcementsFor(message, reset);if (record) {announcementRecord = record;for (const text of record.messages) announcedMessages.add(text);}const receivedAt = valid(message.received_at) ? message.received_at : now();- const heartbeat = message.overall?.heartbeat === true;+ // The estimate in `overall` can be newer than the engine message it rides+ // on: a heartbeat refreshes only that (overall_received_at, from publish).+ const overallReceivedAt = valid(message.overall_received_at) ? message.overall_received_at : receivedAt;+ const beat = message.overall?.heartbeat === true;+ // A heartbeat repeats the last engine message. A page that has not seen+ // that message yet (reopened, or opened mid-run) takes it as the message.+ const heartbeat = beat && previousProgress !== null;if (message.reset === true) {- hasHeartbeat = false; resetRemaining(); lastActivity = null;+ hasHeartbeat = false; lastActivity = null;+ // A stage restarting its own count (the sampler's start) leaves the+ // bridge's whole-request estimate standing; a retry starts afresh.+ if (estimateSource && !message.retry && message.phase !== 'recovery') { completedSteps = 0; previousRemaining = null; }+ else resetRemaining();// A reset that would move a shown share back starts its stage afresh// (hidden, 5 s again). One before anything was counted, such as the// sampler's own start, keeps the stage's clock and the 2 s rule.if (miniFraction > 0) resetMini();}- if (heartbeat) hasHeartbeat = true;+ if (beat) hasHeartbeat = true;if (!heartbeat && message.phase === 'sampling' && valid(message.done))completedSteps = Math.max(completedSteps, message.done);if (message.low_memory && typeof message.low_memory === 'object') lowMemory = true;@@ -714,8 +778,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = fsamplingPlan = {...message.plan_context};// A heartbeat carries a newer estimate, never a newer engine event.// Keep sampledAt paired with the sampling counters/elapsed sample.- if (heartbeat && previousProgress !== null) {- overall = {...overall, ...message.overall}; overallAt = receivedAt;+ if (heartbeat) {+ overall = {...overall, ...message.overall}; overallAt = overallReceivedAt;observeRemaining(message);tick(); return;}@@ -724,7 +788,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f|| message.prediction || message.sampling_plan || reset;if (!activityEvent && previousProgress !== null) {if (message.overall && typeof message.overall === 'object') {- overall = {...message.overall}; overallAt = receivedAt; observeRemaining(message);+ overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message);}tick(); return;}@@ -759,14 +823,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f}element.dataset.phase = message.phase || message.timing_phase || '';if (message.overall && typeof message.overall === 'object') {- overall = {...message.overall}; overallAt = sampledAt;+ overall = {...message.overall}; overallAt = overallReceivedAt;observeRemaining(message);}element.hidden = false;tick(); if (timer === null) timer = setInterval(tick, 250);}function hide() {- element.hidden = true; status.hidden = true; recovery = ''; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false;+ element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false;compatibilityNote.hidden = true; compatibilityNote.textContent = '';memoryTight = null; firstPassOver = false;elapsedShown = 0;diff --git a/web/progress_connection.js b/web/progress_connection.jsindex 078c3ee..04e6ad1 100644--- a/web/progress_connection.js+++ b/web/progress_connection.js@@ -10,7 +10,9 @@ export function createProgressConnection(api, consume, now = () => Date.now()) {// The graph may still be loading. Do not acknowledge an undelivered// snapshot; the next poll restores it once the node exists.const age = Number.isFinite(message.age_seconds) ? Math.max(0, message.age_seconds) : 0;- if (consume({...message, received_at: now() - age * 1000}) !== false)+ // A heartbeat refreshes only the estimate: it is younger than the engine message.+ const overallAge = Number.isFinite(message.overall_age_seconds) ? Math.max(0, message.overall_age_seconds) : age;+ if (consume({...message, received_at: now() - age * 1000, overall_received_at: now() - overallAge * 1000}) !== false)seen.set(key, message);}async function refresh() {