Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Prompt rewrites fit cards with less free graphics memory, wait for memory other programs hold...
Bowen Xue6ff1aecce687
diff --git a/freevideo_engine/gpu_holders.py b/freevideo_engine/gpu_holders.pynew file mode 100644index 0000000..04408ad--- /dev/null+++ b/freevideo_engine/gpu_holders.py@@ -0,0 +1,144 @@+"""Which other programs hold memory on the GPU, so the page can say what to close.++Bytes come from NVML; Windows (WDDM) usually reports none per process, so+there they come from the "GPU Process Memory" performance counters that Task+Manager shows. Those are summed over every adapter the process uses. FreeVideo's+own processes and the desktop's system components are left out: the user+cannot close them. Names are shown on the page only, never written to reports.+"""+import ctypes as C+import os+from pathlib import Path+import re+import sys++MiB = 2**20+# Smaller holders do not free enough to matter.+FLOOR = 128 * MiB+LIMIT = 3+# Desktop compositors and shells on Linux; Windows ones are found by location.+LINUX_DESKTOP = {'Xorg', 'Xwayland', 'gnome-shell', 'kwin_x11', 'kwin_wayland', 'plasmashell', 'cinnamon', 'mutter'}+++def counter_bytes():+ """{pid: dedicated GPU bytes} from Windows performance counters, or None."""+ from ctypes import wintypes as W++ class Value(C.Structure):+ _fields_ = [('status', W.DWORD), ('value', C.c_longlong)]++ class Item(C.Structure):+ _fields_ = [('name', C.c_wchar_p), ('value', Value)]++ pdh = C.WinDLL('pdh')+ query, counter = C.c_void_p(), C.c_void_p()+ if pdh.PdhOpenQueryW(None, None, C.byref(query)):+ return None+ try:+ if pdh.PdhAddEnglishCounterW(query, r'\GPU Process Memory(*)\Dedicated Usage', None, C.byref(counter)):+ return None+ if pdh.PdhCollectQueryData(query):+ return None+ large = 0x400+ for _ in range(3):+ # Instances come and go between asking the size and reading them.+ size, count = W.DWORD(0), W.DWORD(0)+ pdh.PdhGetFormattedCounterArrayW(counter, large, C.byref(size), C.byref(count), None)+ buffer = C.create_string_buffer(size.value + 4096)+ size = W.DWORD(len(buffer))+ if pdh.PdhGetFormattedCounterArrayW(counter, large, C.byref(size), C.byref(count), buffer) == 0:+ break+ else:+ return None+ found = {}+ for item in C.cast(buffer, C.POINTER(Item))[:count.value]:+ match = re.match(r'pid_(\d+)_', item.name or '')+ if match and item.value.status == 0 and item.value.value > 0:+ pid = int(match.group(1))+ found[pid] = found.get(pid, 0) + item.value.value+ return found+ finally:+ pdh.PdhCloseQuery(query)+++def description(path):+ """The executable's own product description (what Task Manager lists), or None."""+ from ctypes import wintypes as W+ version = C.WinDLL('version')+ size = version.GetFileVersionInfoSizeW(path, None)+ if not size:+ return None+ data = C.create_string_buffer(size)+ if not version.GetFileVersionInfoW(path, 0, size, data):+ return None+ pointer, length = C.c_void_p(), W.UINT()+ if not version.VerQueryValueW(data, r'\VarFileInfo\Translation', C.byref(pointer), C.byref(length)) or length.value < 4:+ return None+ language, page = C.cast(pointer, C.POINTER(W.WORD * 2)).contents+ key = r'\StringFileInfo\%04x%04x\FileDescription' % (language, page)+ if not version.VerQueryValueW(data, key, C.byref(pointer), C.byref(length)) or length.value < 2:+ return None+ return C.wstring_at(pointer, length.value).rstrip('\0').strip() or None+++def label(process):+ """The name the user knows the program by, or None for a system component."""+ import psutil+ try:+ name = process.name()+ except psutil.Error:+ return None+ if sys.platform != 'win32':+ return None if name in LINUX_DESKTOP else name+ try:+ path = process.exe()+ except psutil.Error:+ return None # Protected system processes do not show their location.+ system = Path(os.environ.get('SystemRoot', r'C:\Windows')).resolve()+ if not path or Path(path).resolve().is_relative_to(system):+ return None+ try:+ return description(path) or Path(name).stem+ except OSError:+ return Path(name).stem+++def holders(device):+ """[{'name', 'bytes'}] of other programs on the GPU, largest first; None when they cannot be listed.++ Programs of unknown size are left out: on Windows every desktop program+ draws on the GPU, and naming them all would not say what to close."""+ import psutil+ from .background_room import Gpu, family+ gpu = Gpu(device)+ try:+ listed = gpu.processes()+ finally:+ gpu.close()+ if listed is None:+ return None+ if sys.platform == 'win32' and all(size is None for size in listed.values()):+ try:+ measured = counter_bytes()+ except (OSError, AttributeError):+ measured = None+ if measured:+ listed = {pid: measured.get(pid) for pid in set(listed) | set(measured)}+ try:+ own = family(os.getpid())+ except OSError:+ own = {os.getpid()}+ found = {}+ for pid, size in listed.items():+ if pid in own or size is None:+ continue+ try:+ name = label(psutil.Process(pid))+ except psutil.Error:+ continue+ if name:+ # One program often runs as several processes; list it once.+ found[name] = found.get(name, 0) + size+ rows = sorted((dict(name=name, bytes=size) for name, size in found.items() if size >= FLOOR),+ key=lambda row: -row['bytes'])+ return rows[:LIMIT]diff --git a/freevideo_engine/prompt_vlm/load.py b/freevideo_engine/prompt_vlm/load.pyindex 9119c96..9f66d57 100644--- a/freevideo_engine/prompt_vlm/load.py+++ b/freevideo_engine/prompt_vlm/load.py@@ -36,11 +36,12 @@ def stored(folder):return rows-def resident(folder, torch, device, precision, keep=None):+def resident(folder, torch, device, precision, keep=None, vision_on_host=False):"""Returns the model on `device`, with int8 projections if precision == 'int8'.With `keep`, layers from that index on hold their int8 rows in host memory- and are streamed to the GPU as they run."""+ and are streamed to the GPU as they run. With vision_on_host, the vision+ tower stays in host memory and runs on the CPU."""from accelerate import init_empty_weightsfrom torch import nnfrom transformers import AutoConfig, GenerationConfig, Qwen3VLForConditionalGeneration@@ -65,6 +66,7 @@ def resident(folder, torch, device, precision, keep=None):readers = {}def load(prefix):+ target = 'cpu' if vision_on_host and prefix.startswith('model.visual.') else devicegroups = {}for name, shard in shards.items():if name.startswith(prefix):@@ -77,7 +79,7 @@ def resident(folder, torch, device, precision, keep=None):module = model.get_submodule(parent)# The dtype Transformers would load it in (the skeleton's).dtype = getattr(module, leaf).dtype- setattr(module, leaf, nn.Parameter(tensor.to(device=device, dtype=dtype), requires_grad=False))+ setattr(module, leaf, nn.Parameter(tensor.to(device=target, dtype=dtype), requires_grad=False))try:layers = 'model.language_model.layers.'@@ -101,7 +103,17 @@ def resident(folder, torch, device, precision, keep=None):reader.close()if tied:model.lm_head.weight = model.model.language_model.embed_tokens.weight+ if vision_on_host:+ visual = model.model.visual+ del model.model.visualmodel = model.to(device)+ if vision_on_host:+ # Registered last, so the model's first parameter (its device) stays on the GPU.+ model.model.visual = visual+ move = lambda value: value.to('cpu') if torch.is_tensor(value) else value+ visual.register_forward_pre_hook(+ lambda module, args, kwargs: (tuple(map(move, args)), {k: move(v) for k, v in kwargs.items()}),+ with_kwargs=True)if keep is not None:model.streamer = int8.Streamer(torch, list(model.model.language_model.layers)[keep:])torch.cuda.empty_cache() # Merge and quantize temporaries; KV space comes next.diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.pyindex 430b1b4..d2cccc3 100644--- a/freevideo_engine/prompt_vlm/service.py+++ b/freevideo_engine/prompt_vlm/service.py@@ -14,6 +14,10 @@ import uuidfrom . import assets, rules+# How long a rewrite that does not fit waits for other programs to free the+# GPU (the page names them) before it fails; the user can cancel meanwhile.+GPU_WAIT_SECONDS = 300+class Worker:"""One pipe owner across idle imports and the subsequent rewrite."""@@ -70,6 +74,30 @@ class Worker:stream.close()+def release_resident_models(env, descriptor):+ """Release the generation worker's idle models before the rewrite model loads.++ Returns the readings for the rewrite report: a state, how long the release took and the+ card's free memory around it (numbers and states only, never paths)."""+ from .. import resident_process+ release = dict(idle_release='no_resident_process')+ if not (env.get(resident_process.ENV) and Path(env[resident_process.ENV]).is_file()):+ return release+ before, releasing = resident_process.gpu_memory(None), time.monotonic()+ release['idle_release'] = 'released'+ try:+ resident_process.release_idle_cache(env, descriptor)+ resident_process.OWNER.record(action='released', reason='prompt_model')+ except resident_process.SessionRestarted:+ release['idle_release'] = 'already_exited' # That worker already exited, which is the release we need.+ env.pop(resident_process.ENV, None)+ after = resident_process.gpu_memory(None)+ release.update(idle_release_seconds=time.monotonic() - releasing,+ idle_release_gpu_free_before_bytes=before[1] - before[0] if before else None,+ idle_release_gpu_free_after_bytes=after[1] - after[0] if after else None)+ return release++class Service:def __init__(self, installation, input_directory):self.installation = installation@@ -91,6 +119,7 @@ class Service:self.closed = Falseself.request_epoch = Noneself.active_worker = None+ self.release_readings = Noneatexit.register(self.close)def prestart(self, root, machine, *, generating=False):@@ -420,7 +449,7 @@ class Service:else:with tempfile.TemporaryDirectory(prefix='freevideo-prompt-', ignore_cleanup_errors=True) as frames:self.video_frames(value, frames)- self.infer(root, machine, descriptor, value)+ self.infer(root, machine, descriptor, value, wait=action == 'rewrite')except InterruptedError:self.progress(dict(phase='cancelled'))except BlockingIOError:@@ -444,7 +473,7 @@ class Service:# Output rejections contain condition IDs, counts, typed media labels and retry flags only.report = {key: item for key, item in self.state.items()if key in ('phase', 'error', 'exception', 'error_details', 'timings', 'diagnostics',- 'resources', 'cleanup', 'worker_returncode', 'output_rejection')}+ 'resources', 'cleanup', 'worker_returncode', 'output_rejection', 'gpu_wait')}try:spec = assets.catalog()except (OSError, ValueError):@@ -465,20 +494,90 @@ class Service:except OSError:pass- def infer(self, root, machine, descriptor, value):- # A claimed idle worker has no inherited lease. Keep the parent's lease- # through bounded cleanup, including failures before request submission.+ def infer(self, root, machine, descriptor, value, *, wait=False):+ """Run the worker; when the model does not fit the GPU, name the programs+ holding it and, for a rewrite, wait for them to free enough and run again."""+ until = time.monotonic() + GPU_WAIT_SECONDS+ waited = dict(waits=0, seconds=0., freed=0)+ self.release_readings = None+ while True:+ # A claimed idle worker has no inherited lease. Keep the parent's lease+ # through bounded cleanup, including failures before request submission.+ try:+ self._infer(root, machine, descriptor, value)+ finally:+ session = self.active_worker+ if session is not None:+ from .. import processes+ processes.stop(session.process, grace=5)+ session.process.wait()+ session.close()+ self.active_worker = None+ self.process = None+ if self.state.get('phase') != 'failed' or self.state.get('error') != 'out_of_memory':+ return+ stats = self.state.get('diagnostics') or {}+ needed, free = stats.get('gpu_free_needed_bytes'), stats.get('gpu_allocator_budget_bytes')+ short = needed - free if type(needed) is int and type(free) is int and needed > free else None+ freed = False+ if wait and short is not None:+ waiting = time.monotonic()+ try:+ freed = self.wait_for_gpu(machine, short, until)+ finally:+ # Counts and seconds only: the programs' names stay on the page.+ waited.update(waits=waited['waits'] + 1, seconds=waited['seconds'] + time.monotonic() - waiting,+ freed=waited['freed'] + freed)+ self.progress(dict(gpu_wait=dict(waited)))+ if not freed:+ self.progress(dict(phase='failed', error='out_of_memory', gpu_holders=self.gpu_holders(machine)))+ return+ with self.mutex:+ for key in ('error', 'exception', 'error_details', 'gpu_holders', 'gpu_short_bytes'):+ self.state.pop(key, None)+ self.state['phase'] = 'loading'++ @staticmethod+ def gpu_holders(machine):+ """Other programs holding GPU memory, for the page only; None when unknown."""+ from ..gpu_holders import holderstry:- self._infer(root, machine, descriptor, value)+ return holders(machine.get('gpu_uuid'))+ except Exception:+ return None++ def wait_for_gpu(self, machine, short, until):+ """True once the GPU has `short` more bytes free than now; False at `until`+ or when free memory cannot be read. Raises InterruptedError on cancel."""+ from ..background_room import Gpu+ try:+ gpu = Gpu(machine.get('gpu_uuid'))+ except Exception:+ return False+ try:+ reading = gpu.memory()+ if reading is None:+ return False+ # The worker measured its shortfall against what it could use; ask+ # for that much more than is free now, after it has exited.+ target = reading[1] - reading[0] + short+ listed = 0.+ while time.monotonic() < until:+ self.check()+ reading = gpu.memory()+ if reading is None:+ return False+ free = reading[1] - reading[0]+ if free >= target:+ return True+ if time.monotonic() - listed >= 2:+ listed = time.monotonic()+ self.progress(dict(phase='waiting_gpu', gpu_short_bytes=target - free,+ gpu_holders=self.gpu_holders(machine)))+ self.stop.wait(1)+ return Falsefinally:- session = self.active_worker- if session is not None:- from .. import processes- processes.stop(session.process, grace=5)- session.process.wait()- session.close()- self.active_worker = None- self.process = None+ gpu.close()def _infer(self, root, machine, descriptor, value):from .. import processes@@ -490,13 +589,12 @@ class Service:owner = resident_process.OWNERif owner.process is not None and owner.process.poll() is None and owner.endpoint:env[resident_process.ENV] = str(owner.endpoint)- if env.get(resident_process.ENV) and Path(env[resident_process.ENV]).is_file():- try:- resident_process.release_idle_cache(env, descriptor)- resident_process.OWNER.record(action='released', reason='prompt_model')- except resident_process.SessionRestarted:- pass # That worker already exited, which is the release we need.- env.pop(resident_process.ENV, None)+ release = release_resident_models(env, descriptor)+ # A run again after waiting has nothing left to release; the report keeps the first attempt's.+ if release['idle_release'] != 'no_resident_process':+ self.release_readings = release+ elif self.release_readings:+ release = self.release_readingsexpired = Nonewith self.mutex:session, self.prewarm = self.prewarm, None@@ -517,7 +615,7 @@ class Service:diagnostics = dict(prewarm=prewarm, prewarm_reason=reason if prewarm == 'cold' else 'none',prewarm_age_seconds=time.monotonic() - session.started if session else None,prewarm_import_seconds=session.import_seconds if session else None,- request_to_first_output_seconds=None)+ request_to_first_output_seconds=None, **release)footprint = dict(prewarm_rss_bytes=None, prewarm_private_bytes=None)if session is not None:try:diff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.pyindex 078078e..64542cc 100644--- a/freevideo_engine/prompt_vlm/worker.py+++ b/freevideo_engine/prompt_vlm/worker.py@@ -137,9 +137,10 @@ def gpu_snapshot(stats):torch = sys.modules.get('torch')if torch is not None and torch.cuda.is_initialized():try:+ free, total = torch.cuda.mem_get_info()stats.update(gpu_peak_allocated_bytes=torch.cuda.max_memory_allocated(),gpu_peak_reserved_bytes=torch.cuda.max_memory_reserved(),- gpu_free_bytes=torch.cuda.mem_get_info()[0],+ gpu_free_bytes=free, gpu_total_bytes=total,gpu_allocator_ooms=torch.cuda.memory_stats().get('num_ooms'))except (RuntimeError, OSError):stats['gpu_sample_unavailable'] = True@@ -220,10 +221,13 @@ WORKING_VRAM = 3 * 2**28OUTPUT_TOKENS = 1800-def kv_positions(input_tokens):+def kv_positions(input_tokens, answer_tokens=None):"""Cache positions reserved for a request: the input and its rewrite, then acorrection retry that resends both with a short instruction and writes another.- A retry that would need more is skipped when the model is held on the GPU."""+ A retry that would need more is skipped when the model is held on the GPU.+ With answer_tokens, only one answer of that length is reserved (no retry)."""+ if answer_tokens is not None:+ return input_tokens + answer_tokens + 256return input_tokens + 2 * OUTPUT_TOKENS + 256@@ -235,10 +239,10 @@ def reserved_tokens(stats):return tokens-def kv_bytes(stored, input_tokens):+def kv_bytes(stored, input_tokens, answer_tokens=None):keys = [row for name, row in stored.items() if name.endswith('self_attn.k_proj.weight')]per_token = 2 * len(keys) * (keys[0]['shape'][0] if keys else 0) * 2 # K and V, bf16- return per_token * kv_positions(input_tokens)+ return per_token * kv_positions(input_tokens, answer_tokens)def retry_allowed(tokens, stats):@@ -260,6 +264,61 @@ def placement(weights, int8_saving, kv, allocator_budget):return 'bf16' if weights <= room else 'int8' if weights - int8_saving <= room else None+def gpu_plan(weights, stored, free, input_tokens, answer_tokens, text_only):+ """The GPU placement for `free` allocator bytes, or None when none fits.++ Best first: bf16 or int8 resident, then int8 with the later layers streamed+ from host memory. A text-only request keeps the vision tower in host memory+ when that gives a better tier, and when nothing else fits reserves the cache+ for one answer (no correction retry). A request with images runs the vision+ tower on the CPU only when nothing fits otherwise, and keeps the retry: a+ rewrite that misses a picture reference fails without it."""+ from . import int8+ vision = sum(row['bytes'] for name, row in stored.items() if name.startswith('model.visual.'))+ saving = int8.saved_bytes(stored)+ def fit(answer, on_host):+ kv = kv_bytes(stored, input_tokens, answer)+ size = weights - vision if on_host else weights+ plan = dict(kv=kv, positions=kv_positions(input_tokens, answer), vision_on_host=on_host,+ vision_bytes=vision if on_host else 0, keep=None, host_bytes=0)+ precision = placement(size, saving, kv, free)+ if precision:+ return dict(plan, precision=precision)+ streamed = int8.resident_layers(size, stored, free - kv - WORKING_VRAM)+ if streamed:+ return dict(plan, precision='int8', keep=streamed[0], host_bytes=streamed[1])+ return None+ if kv_positions(input_tokens, answer_tokens) >= kv_positions(input_tokens):+ answer_tokens = None+ if text_only:+ # Host memory for the vision tower only when that buys a better tier.+ rank = lambda plan: (plan['precision'] == 'bf16', plan['keep'] is None, plan['keep'] or 0)+ whole, apart = fit(None, False), fit(None, True)+ if whole and (not apart or rank(whole) >= rank(apart)):+ return whole+ options = ((None, True), (answer_tokens, True))+ else:+ options = ((None, False), (None, True))+ for answer, on_host in dict.fromkeys(options):+ plan = fit(answer, on_host)+ if plan:+ return plan+ return None+++def gpu_needed(weights, stored, free, input_tokens, answer_tokens, text_only):+ """The fewest allocator bytes, to 32 MiB, for which gpu_plan finds a placement."""+ step = 32 * 2**20+ low, high = free // step, -(-(weights + 2 * kv_bytes(stored, input_tokens) + WORKING_VRAM) // step)+ while low + 1 < high:+ middle = (low + high) // 2+ if gpu_plan(weights, stored, middle * step, input_tokens, answer_tokens, text_only):+ high = middle+ else:+ low = middle+ return high * step++def load_model(folder, torch, stats, checkpoint):"""Weights, FreeToken operators and GPU placement; returns (model, device)."""from . import core, int8@@ -272,32 +331,43 @@ def load_model(folder, torch, stats, checkpoint):keep = Nonehost_bytes = 0weight_budget = 0+ vision_bytes = 0if torch.cuda.is_available():from .. import gpu_budget- free, _ = torch.cuda.mem_get_info()- stats.update(gpu_name=torch.cuda.get_device_name(), gpu_free_before_load_bytes=free,+ free, total = torch.cuda.mem_get_info()+ stats.update(gpu_name=torch.cuda.get_device_name(), gpu_free_before_load_bytes=free, gpu_total_bytes=total,gpu_compute_capability=list(torch.cuda.get_device_capability()))admission = gpu_budget.configure(torch, free, reserve_bytes=256 * 2**20)free = min(free, admission.get('effective_allocator_limit_bytes') or free)# When the layers do not all fit, keep activation/KV space and the# desktop outside the budget for the layers that stay on the GPU.weight_budget = min(9 * 2**30, max(0, free - 2 * 2**30))- kv = kv_bytes(shapes, reserved_tokens(stats))- stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget, gpu_kv_reserve_bytes=kv)- resident = placement(weights, int8.saved_bytes(shapes), kv, free)- if not resident:- # Too big even as int8: keep what fits on the GPU and stream the other- # layers' int8 rows from host memory, a layer ahead of use.- plan = int8.resident_layers(weights, shapes, free - kv - WORKING_VRAM)- if plan:- resident, (keep, host_bytes) = 'int8', plan+ tokens = reserved_tokens(stats)+ plan = gpu_plan(weights, shapes, free, tokens, stats.get('answer_tokens', OUTPUT_TOKENS),+ not stats.get('image_count'))+ positions = plan['positions'] if plan else kv_positions(tokens)+ stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget,+ gpu_kv_reserve_bytes=plan['kv'] if plan else kv_bytes(shapes, tokens))+ if plan:+ resident, keep, host_bytes = plan['precision'], plan['keep'], plan['host_bytes']+ vision_bytes = plan['vision_bytes']+ stats.update(vision_device='cpu' if plan['vision_on_host'] else 'gpu',+ kv_reserve='one_answer' if positions < kv_positions(tokens) else 'with_retry')stats.update(streamed_layers=len(int8.layer_bytes(shapes)) - keep if keep is not None else 0,streamed_host_bytes=host_bytes)if not resident and weight_budget <= 2 * 2**30:# Not even the streaming tier fits: the model would run on the CPU at# about 0.2 tokens/s and hit the request deadline, so say so at once.+ # The service waits for this much to become free before trying again.+ answer = stats.get('answer_tokens', OUTPUT_TOKENS)+ stats['gpu_free_needed_bytes'] = gpu_needed(weights, shapes, free, tokens, answer, not stats.get('image_count'))+ # The report names the figure for a text-only request and for one with images; this+ # request's own kind may fit at the free memory shown, so these start from zero.+ stats.update(placement_refused='gpu_free_below_floor',+ gpu_free_needed_text_only_bytes=gpu_needed(weights, shapes, 0, tokens, answer, True),+ gpu_free_needed_with_images_bytes=gpu_needed(weights, shapes, 0, tokens, answer, False))raise ValueError('out_of_memory')- stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes) if resident else MAPPED_RAM+ stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes + vision_bytes) if resident else MAPPED_RAMavailable = stats['available_memory_at_start_bytes' if resident else 'available_memory_before_load_bytes']if available < stats['ram_required_bytes']:raise ValueError('ram_space')@@ -307,9 +377,9 @@ def load_model(folder, torch, stats, checkpoint):loading = time.monotonic()if resident:from .load import resident as load_resident- model = load_resident(folder, torch, 'cuda:%d' % card, resident, keep)+ model = load_resident(folder, torch, 'cuda:%d' % card, resident, keep, vision_on_host=bool(vision_bytes))mapping, device = {'': card}, 'cuda:%d' % card- stats.update(weights=resident, kv_positions=kv_positions(reserved_tokens(stats)))+ stats.update(weights=resident, kv_positions=positions)else:import psutilfrom transformers import Qwen3VLForConditionalGeneration@@ -509,7 +579,7 @@ def split(folder, value, stats):from .rules import SPLIT_RETRY, split_instruction, split_user_text, validate_splitcount = len(value['seconds'])limit = min(2 * OUTPUT_TOKENS - 200, 600 + 300 * count) # Within the cache reserved for a rewrite and its retry.- stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, dtype='bfloat16',+ stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, answer_tokens=limit, dtype='bfloat16',attention='sdpa', kv_cache='dynamic', backend='transformers', accelerations=[],persistent_model=False)torch, checkpoint = begin(stats)@@ -574,7 +644,7 @@ def ask(folder, value, stats):questions = value['questions']stats.update(stage='imports', pid=os.getpid(), mode='ask', question_count=len(questions),image_count=sum(len(q['images']) for q in questions), format=value['format'],- max_tokens=value['max_tokens'], custom_system=bool(value.get('system')),+ max_tokens=value['max_tokens'], answer_tokens=value['max_tokens'], custom_system=bool(value.get('system')),dtype='bfloat16', attention='sdpa', kv_cache='dynamic', backend='transformers',accelerations=[], persistent_model=False)torch, checkpoint = begin(stats)@@ -636,7 +706,8 @@ def ask(folder, value, stats):{'role': 'user', 'content': ASK_JSON_RETRY}]answer['retried'] = Trueretry_inputs = tokenize(processor, retry, pixels)- if retry_inputs.input_ids.shape[-1] > 6500:+ if retry_inputs.input_ids.shape[-1] > 6500 or (stats.get('kv_reserve') == 'one_answer' and+ retry_inputs.input_ids.shape[-1] + value['max_tokens'] > stats['kv_positions']):answer['error'] = 'invalid_json'else:ids, _ = generate(torch, model, retry_inputs.to(device), value['max_tokens'],diff --git a/freevideo_engine/resident_process.py b/freevideo_engine/resident_process.pyindex a6420fd..60497fa 100644--- a/freevideo_engine/resident_process.py+++ b/freevideo_engine/resident_process.py@@ -628,19 +628,24 @@ def held_summary(decisions):return {'held': sorted(held), 'evicted': evicted}-def gpu_used(gpu):+def gpu_memory(gpu):+ """(used, total) bytes of the whole device, other programs included; None when unreadable."""try:from .background_room import Gpudevice = Gpu(gpu)try:- memory = device.memory()+ return device.memory() or Nonefinally:device.close()- return memory[0] if memory else Noneexcept (OSError, RuntimeError, AttributeError, ValueError, ImportError):return None+def gpu_used(gpu):+ memory = gpu_memory(gpu)+ return memory[0] if memory else None++def gpu_own(gpu):"""Card bytes held by this process tree (the kept worker and its leases), or None if not listed."""try:diff --git a/web/issue_button.js b/web/issue_button.jsindex e43af07..ecaedb6 100644--- a/web/issue_button.js+++ b/web/issue_button.js@@ -9,9 +9,10 @@ const ISSUE_FORM = 'https://github.com/FlashML-org/FreeVideo/issues/new';/*** Title, text and summary for the prefilled form. `where` is [English, Chinese] words for the place- * ("Export", "导出"); `message` is the sentence the user sees.+ * ("Export", "导出"); `message` is the sentence the user sees; `facts` is an optional flat object of numbers and+ * states (never text or paths) that travels in the summary.*/-export function issueDraft({where, message, kind = null}, t = en => en) {+export function issueDraft({where, message, kind = null, facts = null}, t = en => en) {const place = Array.isArray(where) ? where : [String(where || 'FreeVideo'), String(where || 'FreeVideo')];const clean = createErrorReport({exception_message: String(message || '')}).message;const lines = clean.split(/\r?\n/).filter(line => line.trim());@@ -20,7 +21,7 @@ export function issueDraft({where, message, kind = null}, t = en => en) {const text = [t('FreeVideo showed an error.', 'FreeVideo 显示了一个错误。'),`Where: ${place[0]}`, `Error: ${first.trim().slice(0, MESSAGE)}`, ...(kind ? [`Kind: ${kind}`] : [])].join('\n');const tail = clean.length <= TAIL ? clean : clean.slice(0, 500) + '\n...\n' + clean.slice(-(TAIL - 505));- return {title: `[Feedback] ${t(place[0], place[1])}`, text, summary: JSON.stringify({where: place[0], kind, tail})};+ return {title: `[Feedback] ${t(place[0], place[1])}`, text, summary: JSON.stringify(facts && Object.keys(facts).length ? {where: place[0], kind, tail, facts} : {where: place[0], kind, tail})};}// Browser-only modules, loaded as the button appears so the click opens the page at oncediff --git a/web/prompt_enhance.css b/web/prompt_enhance.cssindex 4c24391..419baff 100644--- a/web/prompt_enhance.css+++ b/web/prompt_enhance.css@@ -18,6 +18,7 @@.fv-enhance-status:is([data-state=progress],[data-state=error],[data-state=note]){display:block}.fv-enhance-status[data-state=error]{color:var(--fv-danger,#ff8a80);background:color-mix(in srgb,var(--fv-danger,#ff8a80) 7%,transparent)}.fv-enhance-status .fv-issue-button{min-height:26px;margin-top:7px;padding:3px 9px;font-size:11px}+.fv-enhance-status .fv-issue-button:not([hidden]){display:flex;width:fit-content}.fv-enhance-meter{position:relative;height:3px;margin-top:7px;border-radius:2px;background:#ffffff12;overflow:hidden}.fv-enhance-meter>i{position:absolute;inset:0 auto 0 0;width:0;border-radius:inherit;background:var(--fv-accent,#6db8fa);transition:width .4s ease-out}.fv-enhance-meter[data-indeterminate=true]>i{width:32%;animation:fv-enhance-sweep 1.4s ease-in-out infinite}diff --git a/web/prompt_enhance.js b/web/prompt_enhance.jsindex 6e433f7..db3c536 100644--- a/web/prompt_enhance.js+++ b/web/prompt_enhance.js@@ -113,13 +113,16 @@ export function createPromptEnhancer({node, input, editor, api, context, setTextconst meter = el('div', null, 'fv-enhance-meter'), fill = el('i');meter.setAttribute('role', 'progressbar'); meter.setAttribute('aria-valuemin', '0'); meter.setAttribute('aria-valuemax', '100');// The shared "Report a problem" button, handed in by the page.- const report = issueButton ? issueButton(t, () => ({where: ['Prompt enhancement', '提示词增强'], message: statusText.textContent})) : document.createElement('span');+ // Feedback quotes the line without the names of the user's other programs.+ let quoted = '';+ const report = issueButton ? issueButton(t, () => ({where: ['Prompt enhancement', '提示词增强'], message: quoted, facts: memoryFacts})) : document.createElement('span');report.hidden = true;meter.append(fill); status.append(statusText, meter, report);// One place sets what the status line says and how it looks.- function show(text, state = '', fraction = null) {+ function show(text, state = '', fraction = null, quote = text) {if (downloadOnly) { publishModel({text: text || ''}); return; }statusText.textContent = text || '';+ quoted = quote || '';status.dataset.state = text ? state : '';report.hidden = !text || state !== 'error';meter.hidden = state !== 'progress';@@ -163,6 +166,13 @@ export function createPromptEnhancer({node, input, editor, api, context, setTextif (versions.value.enabled) void prestart.request();// Available RAM the model needs; the server knows the usual case for this card.let ramBytes = 11 * 2 ** 30;+ // Memory readings of a failed rewrite for the feedback form: numbers and states only.+ let memoryFacts = null;+ const MEMORY_FACTS = ['gpu_total_bytes', 'gpu_free_before_load_bytes', 'gpu_allocator_budget_bytes', 'gpu_free_needed_text_only_bytes', 'gpu_free_needed_with_images_bytes',+ 'placement_refused', 'gpu_free_bytes', 'gpu_allocator_ooms', 'idle_release', 'idle_release_seconds',+ 'idle_release_gpu_free_before_bytes', 'idle_release_gpu_free_after_bytes'];+ const factsOf = diagnostics => Object.fromEntries(MEMORY_FACTS.filter(key => ['number', 'string'].includes(typeof diagnostics?.[key]))+ .map(key => [key, diagnostics[key]]));const gib = value => String(Math.round(value / 2 ** 30));// Mentions are counted as the character descriptions they expand to, so the count can exceed what the box shows.const lengthOf = (error, english) => {@@ -171,6 +181,9 @@ export function createPromptEnhancer({node, input, editor, api, context, setText: `提示词${error.expanded ? '连同角色描述' : ''}共 ${number} 字符`;};// Unknown codes keep the server's own reason, without paths, so the user can quote it.+ // Other programs holding graphics memory, as the server measured them; no-break spaces keep name, size and unit on one line.+ const holderNames = holders => (Array.isArray(holders) ? holders : []).slice(0, 3)+ .map(row => `${String(row?.name || '').slice(0, 60)}\u00a0${((row?.bytes || 0) / 2 ** 30).toFixed(1)}\u00a0GiB`).join(t(', ', '、'));const reasonOf = error => String(error?.message || '').replace(/[A-Za-z]:[\\/][^\s'"]*|\/[^\s'"]+\/[^\s'"]*/g, '').replace(/\s+/g, ' ').trim().slice(0, 120);// A dropped request names a browser extension only when the error carries evidence of one.const networkText = error => {@@ -195,7 +208,10 @@ export function createPromptEnhancer({node, input, editor, api, context, setText: t('Enter a prompt of up to 12,000 characters.', '请输入不超过 12,000 字符的提示词。'),invalid_output: t('The rewrite was incomplete. Retry or use the original.', '改写结果不完整,请重试或使用原文。'),context_length: t('The scene is too long for this model. Shorten it or use the original.', '内容超出改写模型容量,请精简或使用原文。'),- out_of_memory: t('Not enough memory to rewrite. Close other GPU apps or use the original.', '改写时内存不足,请关闭其他显卡程序或使用原文。'),+ out_of_memory: holderNames(error?.holders)+ ? t(`Not enough graphics memory to rewrite. In use: ${holderNames(error.holders)}. Close the listed programs and rewrite again, or use the original.`,+ `显存不足,无法改写。正在占用:${holderNames(error.holders)}。关闭上述程序后可重新改写,或使用原文。`)+ : t('Not enough memory to rewrite. Close other GPU apps or use the original.', '改写时内存不足,请关闭其他显卡程序或使用原文。'),timeout: t('Rewriting took too long. Retry or use the original.', '改写超时,请重试或使用原文。'),stale: t('The prompt or references changed. Rewrite again to use the current scene.', '原文或素材已更改,可按当前内容重新改写。'),cancelled: t('Cancelled. Your original prompt is retained.', '已取消,原文已保留。'),@@ -215,7 +231,10 @@ export function createPromptEnhancer({node, input, editor, api, context, setText: t('Rewriting supports prompts of up to 12,000 characters.', '改写只支持不超过 12,000 字符的提示词。'),invalid_output: t('The rewrite was incomplete.', '改写结果不完整。'),context_length: t('The scene is too long for the rewrite model.', '内容超出改写模型容量。'),- out_of_memory: t('Not enough memory to rewrite. Close other GPU apps before rewriting again.', '改写时内存不足,关闭其他显卡程序后可再改写。'),+ out_of_memory: holderNames(error?.holders)+ ? t(`Not enough graphics memory to rewrite. In use: ${holderNames(error.holders)}. Close the listed programs before rewriting again.`,+ `显存不足,无法改写。正在占用:${holderNames(error.holders)}。关闭上述程序后可重新改写。`)+ : t('Not enough memory to rewrite. Close other GPU apps before rewriting again.', '改写时内存不足,关闭其他显卡程序后可再改写。'),timeout: t('Rewriting took too long.', '改写超时。'),unsupported_media: errorText(error), too_many_images: errorText(error), stale: errorText(error),connected_media: errorText(error), audio_media: errorText(error), unreadable_video: errorText(error),@@ -295,11 +314,19 @@ export function createPromptEnhancer({node, input, editor, api, context, setTextonReport(row);}if (row.phase === 'complete') return row;- if (row.phase === 'failed') throw Object.assign(new Error(row.error || 'rewrite_failed'), {ramBytes: row.diagnostics?.ram_required_bytes});+ if (row.phase === 'failed') {+ memoryFacts = row.error === 'out_of_memory' ? factsOf(row.diagnostics) : null;+ throw Object.assign(new Error(row.error || 'rewrite_failed'), {ramBytes: row.diagnostics?.ram_required_bytes, holders: row.gpu_holders});+ }if (row.phase === 'cancelled') throw new Error(action === 'install' ? 'download_failed' : 'cancelled');if (disposed) throw new Error('cancelled');if (row.phase === 'download') {showDownload(row.done || 0, row.total || 0);+ } else if (row.phase === 'waiting_gpu') {+ // The rewrite starts by itself once enough graphics memory is free.+ const short = Math.max(0.1, (row.gpu_short_bytes || 0) / 2 ** 30).toFixed(1), names = holderNames(row.gpu_holders);+ show(names ? t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite. In use: ${names}.`, `显存不足,再空出 ${short}\u00a0GiB 后自动改写。正在占用:${names}。`)+ : t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite.`, `显存不足,再空出 ${short}\u00a0GiB 后自动改写。`), 'progress');} else if (action !== 'install') {// The ellipsis says "still going" until the seconds start counting.const [en, zh] = row.phase === 'rewriting' ? ['Enhancing the prompt', '正在改写提示词'] : ['Preparing enhancement', '正在准备提示词增强'];@@ -369,7 +396,7 @@ export function createPromptEnhancer({node, input, editor, api, context, setTextwaiting = true; show(errorText({message: 'prompt_required'}), 'note'); input.focus();throw new Error('prompt_required');}- cancelling = false;+ cancelling = false; memoryFacts = null;const execute = async () => {if (disposed || cancelling) throw new Error('cancelled');versions.value.useOriginal = false;@@ -391,7 +418,8 @@ export function createPromptEnhancer({node, input, editor, api, context, setTextapply(); show('');};running = Promise.resolve().then(execute).catch(error => {- if (!disposed) show(errorText(error), error?.message === 'cancelled' ? '' : error?.message === 'stale' ? 'note' : 'error');+ if (!disposed) show(errorText(error), error?.message === 'cancelled' ? '' : error?.message === 'stale' ? 'note' : 'error', null,+ errorText({name: error?.name, message: error?.message, ramBytes: error?.ramBytes, characters: error?.characters, expanded: error?.expanded}));throw error;}).finally(() => { running = null; cancelling = false; install.hidden = true; render(); });render();