Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Add LoRA support to Qwen-Image generation, editing and character generation
Bowen Xue993060009664
diff --git a/freevideo_engine/agent/cast.py b/freevideo_engine/agent/cast.pyindex 948f2f3..51fa615 100644--- a/freevideo_engine/agent/cast.py+++ b/freevideo_engine/agent/cast.py@@ -679,7 +679,11 @@ class Cast:if value['face'] and card['kind'] != 'character':raise spec.SpecError([schema.diagnostic('/face', 'false for a setting card', True)])ready = value.get('source') != 'video' and getattr(self.jobs.port, 'image_ready', lambda: False)()- source = value.get('source', 'image' if ready else 'video')+ from ..image_gen.loras import validate_selection+ loras = validate_selection(value.get('loras', []))+ source = value.get('source', 'image' if ready or loras else 'video')+ if loras and source != 'image':+ spec.fail('/loras', 'LoRAs for reference pictures require source=image')# An explicit image request goes through the image service even when# its pack is absent: that failure owns a diagnostic report and a# durable job/card, just like a picture requested directly.@@ -689,7 +693,7 @@ class Cast:from ..image_gen.cast_views import VIEWS, SETTING_VIEWS# A picture takes seconds, so a character always gets every view, the face close-up included.views = list(SETTING_VIEWS if card['kind'] == 'setting' else VIEWS)- items = self._image_items(card, views[:1], seed)+ items = self._image_items(card, views[:1], seed, loras=loras)else:parts = [('turn', SCENE if card['kind'] == 'setting' else TURN, False)]if value['face']:@@ -708,12 +712,14 @@ class Cast:card['_generating'].update(source=source, stage='front', views=views[1:],job_views={jobs[0]: views[0]}, name=card['name'],description=card['description'])+ if loras:+ card['_generating']['loras'] = lorasresult = dict(job=jobs[0], jobs=jobs, card=view(card))self._remember(record, ref, digest, result, client)self._save(record)return result- def _image_items(self, card, views, seed, front=None, from_photo=False):+ def _image_items(self, card, views, seed, front=None, from_photo=False, loras=None):from ..image_gen.cast_views import size, uses_front, view_prompt, photo_prompt# A picture somebody put on the card (the user, or an agent passing on the user's photo) is the person.photo = next((row['media'] for row in card.get('images', []) if not row.get('generated')), None)@@ -726,7 +732,8 @@ class Cast:view_prompt(dict(card, photo=True) if from_photo else card, part))items.append(dict(kind='image', prompt=prompt, width=width, height=height, seed=seed,images=images, label=('Cast · %s · %s' % (card['name'], part))[:120],- metadata=dict(cast_card=card['id'], cast_part=part)))+ metadata=dict(cast_card=card['id'], cast_part=part),+ **({'loras': loras} if loras else {})))return itemsdef _image_views(self, identity, generating, hidden, rows):@@ -742,7 +749,7 @@ class Cast:card = record['card']basis = dict(id=identity, kind=hidden['kind'], name=hidden['name'], description=hidden['description'])items = self._image_items(basis, hidden['views'], hidden['seed'], dict(file=front['result']['file']),- from_photo=bool(front['spec'].get('images')))+ from_photo=bool(front['spec'].get('images')), loras=hidden.get('loras'))jobs, _ = self.jobs.submit_request(dict(jobs=items, ref='cast-' + hidden['ref'] + '-views'),client='cast:' + identity, parent=dict(cast=identity, ref=hidden['ref']))card['generating']['jobs'].extend(row['id'] for row in jobs)diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.pyindex 77fbddd..d9afb86 100644--- a/freevideo_engine/agent/comfy.py+++ b/freevideo_engine/agent/comfy.py@@ -322,7 +322,11 @@ class ComfyPort:# Local image generation ----------------------------------------------------------------------------def _image(self):from ..image_gen.service import get_service- return get_service(self.server)+ service = get_service(self.server)+ if (service is not None and getattr(service, 'folders', None) is None+ and getattr(self, 'folders', None) is not None):+ service.folders = self.folders+ return service@propertydef image(self):diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.pyindex 2ddc5bf..28ff01d 100644--- a/freevideo_engine/agent/errors.py+++ b/freevideo_engine/agent/errors.py@@ -7,6 +7,7 @@ from contextvars import ContextVarimport mathimport reimport uuid+from ..image_gen.loras import ERRORS as LORA_ERRORS, MESSAGE_EN as LORA_MESSAGE_EN, MESSAGE_ZH as LORA_MESSAGE_ZHREQUEST_ID = ContextVar('agent_request_id', default=None)# The work session a token request belongs to (sessions.py); jobs and productions record it.@@ -155,13 +156,14 @@ def image_failure(code, report_id=None):en, zh = IMAGE_REASONS.get(code, IMAGE_REASONS['invalid_request'] if code.startswith('invalid_')or code == 'too_many_images' else IMAGE_REASONS['generation_failed'])valid = isinstance(report_id, str) and re.fullmatch(r'[0-9a-f]{32}', report_id)- return dict(reason=dict(en=en, zh=zh), report_id=report_id if valid and code not in (+ return dict(reason=dict(en=en, zh=zh), report_id=report_id if valid and code not in LORA_ERRORS and code not in ('busy', 'cancelled', 'invalid_request', 'invalid_media', 'invalid_prompt', 'invalid_dimensions','invalid_seed', 'too_many_images', 'disk_full') else None)# Recovery is a function of the committed task and phase, not just an error code.IMAGE_ERRORS = {+ **LORA_MESSAGE_EN,'invalid_request': 'The picture request contains invalid fields.','invalid_prompt': 'The picture prompt must contain text and be at most 2000 characters.','invalid_dimensions': 'Picture width and height must be integers from 256 to 2048.',@@ -183,6 +185,7 @@ IMAGE_ERRORS = {'disk_full': IMAGE_REASONS['disk_full'][0],'cancelled': 'The picture job was cancelled.',}+IMAGE_REASONS.update({code: (LORA_MESSAGE_EN[code], LORA_MESSAGE_ZH[code]) for code in LORA_ERRORS})for _code, _message in IMAGE_ERRORS.items():if _code not in ERRORS:ERRORS[_code] = (503 if _code in ('pack_missing', 'closed', 'image_unavailable') else 422,@@ -267,6 +270,7 @@ def safe_details(value):# One public sentence shared by conversation cards, HTTP and MCP. Backend text# is classified, never echoed: it can contain prompts, filenames or credentials.FAILURE_REASONS_ZH = {+ **LORA_MESSAGE_ZH,'lan_unavailable': '局域网设置需要启动器安装记录','out_of_memory': '生成时显存或内存不足','insufficient_memory': '可用显存或内存不够生成这段内容',@@ -316,6 +320,7 @@ FAILURE_REASONS_ZH = {'internal_error': '没有完成',}FAILURE_REASONS_EN = {+ **LORA_MESSAGE_EN,'lan_unavailable': 'Phone access requires FreeVideo installed with the launcher','out_of_memory': 'Graphics or system memory ran out during generation','insufficient_memory': 'There is not enough free graphics or system memory for this generation',diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.pyindex 1243662..7513077 100644--- a/freevideo_engine/agent/jobs.py+++ b/freevideo_engine/agent/jobs.py@@ -1999,7 +1999,7 @@ class Jobs:def start_image(self, row):spec = row['spec']row['_phase'] = 'preparing images'- value = {k: spec[k] for k in ('width', 'height', 'seed') if k in spec}+ value = {k: spec[k] for k in ('width', 'height', 'seed', 'loras') if k in spec}if spec.get('people'):# A first frame: the people's pictures at the asked places, in the scene shown or described.action = 'compose'diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.pyindex 8f37c4f..ca82a58 100644--- a/freevideo_engine/agent/schema.py+++ b/freevideo_engine/agent/schema.py@@ -245,12 +245,21 @@ def vlm(*, local=False):('kind', 'questions'), description='x-job-kind also checks the sum of images and sampled video frames across questions (at most 24).', **{'x-job-kind': 'vlm'})+def image_loras():+ from ..image_gen.loras import MAX_LORAS, MAX_NAME, MIN_STRENGTH, MAX_STRENGTH+ # The name's rules (extension, folders, control characters) are checked by validate_selection with its own messages.+ return array(obj(dict(name=dict(type='string', minLength=1, maxLength=MAX_NAME), strength=number(MIN_STRENGTH, MAX_STRENGTH, default=1.),+ enabled=dict(type='boolean', default=True)), ('name',)), high=MAX_LORAS,+ **{'x-image-loras': True})++def image(*, local=False):return obj(dict(kind={'const': 'image', 'type': 'string'},prompt=dict(type='string', minLength=1, maxLength=2000),width=number(256, 2048, integer=True), height=number(256, 2048, integer=True),aspect=enum(IMAGE_ASPECTS, description='Instead of width and height; 1:1 when all three are omitted.'),seed=nullable(number(-2**63, 2**64 - 1, integer=True)),+ loras=image_loras(),images=array({'oneOf': [FILE, obj(dict(file=FILE), ('file',))]}, high=4, description='Pictures to change ''or combine: input files, such as result.file of an earlier picture job. The prompt calls them image 1, image 2.'),people=array(obj(dict(image=FILE, cast=CAST, name=string(40), x=number(0, 1), height=number(.05, 1),@@ -613,7 +622,8 @@ def cast_schema(name, *, local=False):return obj(dict(revision=number(1, integer=True), ref=REF))if name == 'generate':return obj(dict(face=dict(type='boolean', default=False, description='Also make one face close-up (video; the image model always does).'),- source=enum(('image', 'video'), description='Defaults to image when the local image model is ready, otherwise video.'),+ loras=image_loras(),+ source=enum(('image', 'video'), description='Defaults to image when LoRAs are selected or the local image model is ready, otherwise video.'),replace=dict(type='boolean', default=False,description='Replace pictures the user did not add; default fills empty slots.'),revision=number(1, integer=True), ref=REF))@@ -1227,6 +1237,12 @@ def cross_issues(schema, value, path):errors = []def bad(field, allowed, got=None):errors.append(diagnostic(path + '/' + field, allowed, got))+ if schema.get('x-image-loras'):+ from ..image_gen.loras import MESSAGE_ZH, validate_selection+ try:+ validate_selection(value)+ except ValueError as error:+ errors.append(diagnostic(path, MESSAGE_ZH[str(error)], None))flags = schema.get('x-media')if flags is not None:item = {'file': value} if isinstance(value, str) else value@@ -1551,7 +1567,7 @@ SESSION_ARGUMENT = dict(type='string', pattern='^s[0-9a-f]{12}$',description='Work session; omit for latest.')-SHAPE_MIN_BYTES = 400+SHAPE_MIN_BYTES = 200RECIPE_TOOL = 'freevideo_edit_image'diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.pyindex e70c8b9..46846f8 100644--- a/freevideo_engine/agent/spec.py+++ b/freevideo_engine/agent/spec.py@@ -271,6 +271,10 @@ def normalize(value, *, port=None):result['questions'].append(item)result.update({k: value[k] for k in ('format', 'max_tokens', 'system') if k in value})elif kind == 'image':+ from ..image_gen.loras import validate_selection+ loras = validate_selection(original.get('loras', []))+ if loras:+ result['loras'] = loraswidth, height = ((value['width'], value['height']) if 'width' in value elsecontract.IMAGE_ASPECTS[value.get('aspect', '1:1')])result.update(prompt=value.get('prompt', ''), width=width, height=height,diff --git a/freevideo_engine/comfy_upload.py b/freevideo_engine/comfy_upload.pyindex 5a724af..aaaf089 100644--- a/freevideo_engine/comfy_upload.py+++ b/freevideo_engine/comfy_upload.py@@ -110,10 +110,18 @@ def register():from .lora_check import check_installed_filetry:name = request.query.get('name')- path = resolve_loras(json.dumps([dict(name=name)]), folder_paths.get_filename_list('loras'),- lambda n: folder_paths.get_full_path_or_raise('loras', n))[0]['path']- result = await asyncio.to_thread(check_installed_file, path)+ if request.query.get('kind') == 'image':+ from .image_gen.loras import check_named_file+ result = await asyncio.to_thread(check_named_file, name, folders=folder_paths)+ else:+ path = resolve_loras(json.dumps([dict(name=name)]), folder_paths.get_filename_list('loras'),+ lambda n: folder_paths.get_full_path_or_raise('loras', n))[0]['path']+ result = await asyncio.to_thread(check_installed_file, path)except (OSError, ValueError, TypeError) as error:+ if request.query.get('kind') == 'image':+ from .image_gen.loras import MESSAGE_ZH+ return web.json_response({'error': str(error),+ 'message_zh': MESSAGE_ZH.get(str(error), MESSAGE_ZH['invalid_loras'])}, status=400)return web.json_response({'error': str(error)}, status=400)return web.json_response(result, headers={'Cache-Control': 'no-store'})diff --git a/freevideo_engine/image_gen/compose.py b/freevideo_engine/image_gen/compose.pyindex 0dc0653..9065270 100644--- a/freevideo_engine/image_gen/compose.py+++ b/freevideo_engine/image_gen/compose.py@@ -137,4 +137,5 @@ def build(value, input_directory):load(name)if len(prompt) > 2000:raise ValueError('invalid_prompt')- return dict(prompt=prompt, width=width, height=height, seed=value.get('seed'), images=names)+ return dict(prompt=prompt, width=width, height=height, seed=value.get('seed'), images=names,+ **({'loras': value['loras']} if 'loras' in value else {}))diff --git a/freevideo_engine/image_gen/failure.py b/freevideo_engine/image_gen/failure.pyindex 57af20d..0882327 100644--- a/freevideo_engine/image_gen/failure.py+++ b/freevideo_engine/image_gen/failure.py@@ -13,16 +13,18 @@ from ..disk_space import ran_outfrom ..exit_diagnostics import clean_reading, clean_samplefrom ..diagnostic_resources import commit_exhaustedfrom ..failure_details import Redactor+from .loras import ERRORS as LORA_ERRORS, MESSAGE_ZH as LORA_MESSAGE_ZHERRORS = frozenset(('no_memory', 'small_memory', 'busy_video', 'busy', 'pack_missing', 'model_unreadable','device_unavailable', 'worker_exit', 'timeout', 'cancelled', 'disk_full',- 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'generation_failed'))+ 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'generation_failed')) | LORA_ERRORSNO_REPORT = frozenset(('small_memory', 'busy', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'cancelled',- 'disk_full'))+ 'disk_full')) | LORA_ERRORSSTAGES = ('start', 'load', 'text', 'encode', 'denoise', 'decode')KINDS = ('generate', 'edit', 'cast', 'compose', 'download', 'install')MESSAGE_ZH = {+ **LORA_MESSAGE_ZH,'no_memory': '显存不够,最小尺寸也没能生成。关掉其他占用显卡的程序后再试。','small_memory': '这台 Mac 的内存不足 16 GB,无法生成图片。','busy_video': '视频正在生成,完成后再试。',diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.pyindex e4e11ca..8957c7d 100644--- a/freevideo_engine/image_gen/image_runner.py+++ b/freevideo_engine/image_gen/image_runner.py@@ -293,7 +293,7 @@ class ImageRunner:reference_pixels=reference_pixels, shape=self.shape)def generate(self, prompt, *, width, height, seed=None, reference_images=None, edit_mask=None,- progress=None, cancel=None):+ loras=None, progress=None, cancel=None):plan = self.plan(width, height, reference_images)if self.full_card and not self._all_on_card(plan):self._to_streamed()@@ -305,9 +305,10 @@ class ImageRunner:self.te_stack.set_resident(plan.te_resident_layers)self.generator.vae_tile = plan.vae_tilingself.generator.vae_tile_size = plan.vae_tile_size+ extra = dict(loras=loras) if loras else {}return self.generator.generate(prompt, width=width, height=height, seed=seed,reference_images=reference_images, edit_mask=edit_mask,- progress=progress, cancel=cancel)+ progress=progress, cancel=cancel, **extra)def _all_on_card(self, plan):return (plan.feasible and _kept(plan) == len(self.dit_stack.layers)diff --git a/freevideo_engine/image_gen/loras.py b/freevideo_engine/image_gen/loras.pynew file mode 100644index 0000000..63023ba--- /dev/null+++ b/freevideo_engine/image_gen/loras.py@@ -0,0 +1,111 @@+"""Shared image LoRA admission; names resolve only through ComfyUI's libraries."""+import json+import math+from pathlib import Path++MAX_LORAS = 8+MAX_NAME = 512+MIN_STRENGTH, MAX_STRENGTH = -2., 2.+ERRORS = frozenset(('invalid_loras', 'invalid_lora_strength', 'duplicate_lora',+ 'too_many_loras', 'lora_not_found', 'invalid_lora_model'))+MESSAGE_ZH = {+ 'invalid_loras': 'LoRA 设置有误,请重新选择文件。',+ 'invalid_lora_strength': 'LoRA 强度必须在 -2 到 2 之间。',+ 'duplicate_lora': '同一个 LoRA 只能选择一次,请移除重复项。',+ 'too_many_loras': '一张图片最多使用 8 个 LoRA,请减少数量。',+ 'lora_not_found': 'LoRA 文件找不到,请重新选择。',+ 'invalid_lora_model': '此 LoRA 不适用于图片生成,请换一个。',+}+MESSAGE_EN = {+ 'invalid_loras': 'The LoRA selection is invalid. Choose the files again.',+ 'invalid_lora_strength': 'LoRA strength must be between -2 and 2.',+ 'duplicate_lora': 'Select each LoRA only once. Remove the duplicate.',+ 'too_many_loras': 'Use at most eight LoRAs per picture.',+ 'lora_not_found': 'The LoRA file could not be found. Choose it again.',+ 'invalid_lora_model': 'This LoRA cannot be used for pictures. Choose another one.',+}+++def validate_selection(value):+ """Canonical active items, with the same checks for HTTP and in-process jobs."""+ if not isinstance(value, list):+ raise ValueError('invalid_loras')+ if len(value) > MAX_LORAS:+ raise ValueError('too_many_loras')+ result, seen = [], set()+ for row in value:+ if not isinstance(row, dict) or set(row) - {'name', 'strength', 'enabled'}:+ raise ValueError('invalid_loras')+ name, strength = row.get('name'), row.get('strength', 1.)+ if type(row.get('enabled', True)) is not bool:+ raise ValueError('invalid_loras')+ if (not isinstance(name, str) or not 1 <= len(name) <= MAX_NAME+ or any(ord(c) < 32 for c in name) or ':' in name+ or any(part in ('', '.', '..') for part in name.replace('\\', '/').split('/'))+ or not name.lower().endswith('.safetensors')):+ raise ValueError('invalid_loras')+ if (type(strength) not in (int, float) or not MIN_STRENGTH <= strength <= MAX_STRENGTH+ or not math.isfinite(strength)):+ raise ValueError('invalid_lora_strength')+ if not row.get('enabled', True):+ continue+ if name in seen:+ raise ValueError('duplicate_lora')+ seen.add(name)+ # Match resolve_loras: zero strength does not load weights or add triggers.+ if strength != 0:+ result.append(dict(name=name, strength=float(strength)))+ return result+++def check_file(path):+ """Image counterpart of lora_check.check_installed_file, without exposing paths."""+ from .qwen_lora import inspect_file+ try:+ result = inspect_file(path)+ except (OSError, ValueError):+ result = dict(ok=False, reason='unreadable')+ usable = bool(result['ok'])+ return dict(usable=usable, reason=None if usable else result['reason'], precision=None,+ **({} if usable else {'message_zh': '此 LoRA 不适用于图片生成。'}))+++def resolve_paths(value, *, folders=None):+ rows = validate_selection(value)+ if not rows:+ return []+ if folders is None:+ import folder_paths as folders+ names = set(folders.get_filename_list('loras'))+ roots = [Path(root).resolve() for root in folders.get_folder_paths('loras')]++ def resolve(name):+ try:+ path = Path(folders.get_full_path_or_raise('loras', name)).resolve(strict=True)+ except (OSError, ValueError, KeyError):+ raise ValueError('lora_not_found') from None+ if not any(path.is_relative_to(root) for root in roots):+ raise ValueError('invalid_loras')+ if not path.is_file():+ raise ValueError('lora_not_found')+ if path.suffix.lower() != '.safetensors':+ raise ValueError('invalid_loras')+ return path++ if any(row['name'] not in names for row in rows):+ raise ValueError('lora_not_found')+ from ..comfy_assets import resolve_loras+ resolved = resolve_loras(json.dumps(rows), names, resolve)+ return [dict(name=row['name'], **item) for row, item in zip(rows, resolved)]+++def resolve_selection(value, *, folders=None):+ result = resolve_paths(value, folders=folders)+ for item in result:+ if not check_file(item['path'])['usable']:+ raise ValueError('invalid_lora_model')+ return result+++def check_named_file(name, *, folders=None):+ return check_file(resolve_paths([dict(name=name)], folders=folders)[0]['path'])diff --git a/freevideo_engine/image_gen/qwen_lora.py b/freevideo_engine/image_gen/qwen_lora.pynew file mode 100644index 0000000..38f544c--- /dev/null+++ b/freevideo_engine/image_gen/qwen_lora.py@@ -0,0 +1,208 @@+"""Community LoRAs for the Qwen-Image 2.1 DiT, applied as unmerged low-rank branches.++The DiT's Linears are int8 ConvRot matrices, which cannot absorb a weight delta,+so every LoRA stays a pair of small matrices and adds ``scale * B(A x)`` to the+projection's output (the same shape as the video model's online LoRA). Nothing+is written into the model's weights: attaching adds a forward wrapper per+target, detaching restores the original forward, and a request without LoRAs+runs the untouched model.++Accepted files are the layouts the community trains for this model: kohya+(``lora_down``/``lora_up`` and ``alpha``) and PEFT / ai-toolkit (``lora_A`` /+``lora_B``), under a ``diffusion_model.``, ``transformer.`` or+``base_model.model.`` prefix, or kohya's flat ``lora_unet_...`` names. The fused+``img_mlp.gate_up`` target is split into the model's ``gate_layer`` (first half+of the rows) and ``proj`` (second half), the order the checkpoint loader uses.+"""+import types+from functools import lru_cache+from pathlib import Path++import torch+import torch.nn.functional as F++UNUSABLE = 'This LoRA was not trained for the Qwen image model and cannot be used.'++_PREFIXES = ('model.diffusion_model.', 'diffusion_model.', 'base_model.model.', 'transformer.')+_DOWN = ('.lora_down.weight', '.lora_A.weight', '.lora_A.default.weight', '.lora_down.default.weight')+_UP = ('.lora_up.weight', '.lora_B.weight', '.lora_B.default.weight', '.lora_up.default.weight')+_ALPHA = ('.alpha', '.lora_alpha')+_CHUNK = 512+++@lru_cache(maxsize=1)+def layout():+ """name -> (in_features, out_features) of every LoRA target; ``gate_up`` is the fused pair, added beside its two halves."""+ from .qwen_dit import QwenImageDiT+ from .qwen_weights import _config+ with torch.device('meta'):+ model = QwenImageDiT(**_config({}, QwenImageDiT, None))+ shapes = {}+ for name, module in model.named_modules():+ if isinstance(module, torch.nn.Linear):+ shapes[name] = (module.in_features, module.out_features)+ for name in [n for n in shapes if n.endswith('.img_mlp.gate_layer')]:+ prefix = name[:-len('gate_layer')]+ inputs, rows = shapes[name]+ shapes[prefix + 'gate_up'] = (inputs, rows * 2)+ return shapes+++@lru_cache(maxsize=1)+def _flat_names():+ return {'lora_unet_' + name.replace('.', '_'): name for name in layout()}+++def _split(key):+ """-> (target name, 'down' | 'up' | 'alpha') or None for a key that is not a LoRA tensor."""+ for kind, suffixes in (('down', _DOWN), ('up', _UP), ('alpha', _ALPHA)):+ for suffix in suffixes:+ if key.endswith(suffix):+ base = key[:-len(suffix)]+ for prefix in _PREFIXES:+ if base.startswith(prefix):+ base = base[len(prefix):]+ break+ return _flat_names().get(base, base), kind+ return None+++def _entries(keys):+ """Group tensor keys by target: name -> {'down': key, 'up': key, 'alpha': key}."""+ groups = {}+ for key in keys:+ parsed = _split(key)+ if parsed is not None:+ groups.setdefault(parsed[0], {})[parsed[1]] = key+ return groups+++def inspect_file(path):+ """Header-only check: ``dict(ok, reason)``; reason is 'other-model', 'shape', 'empty' or 'unreadable'."""+ from safetensors import safe_open+ try:+ with safe_open(str(path), framework='pt', device='cpu') as handle:+ keys = list(handle.keys())+ groups = _entries(keys)+ if not keys:+ return dict(ok=False, reason='empty')+ shapes = layout()+ mapped = {name: group for name, group in groups.items() if name in shapes}+ if not mapped or not all('down' in g and 'up' in g for g in mapped.values()):+ return dict(ok=False, reason='other-model')+ for name, group in mapped.items():+ inputs, outputs = shapes[name]+ if 'alpha' in group:+ alpha = handle.get_tensor(group['alpha']).float()+ if alpha.numel() != 1 or not torch.isfinite(alpha).all():+ return dict(ok=False, reason='shape')+ down = tuple(handle.get_slice(group['down']).get_shape())+ up = tuple(handle.get_slice(group['up']).get_shape())+ if len(down) != 2 or len(up) != 2 or down[1] != inputs or up[0] != outputs or down[0] != up[1] or down[0] < 1:+ return dict(ok=False, reason='shape')+ return dict(ok=True, reason=None)+ except Exception:+ return dict(ok=False, reason='unreadable')+++def read(path, strength=1.0):+ """-> list of (target, a [r, in], b [out, r], scale) in bf16 on the CPU. ValueError when unusable."""+ from safetensors import safe_open+ verdict = inspect_file(path)+ if not verdict['ok']:+ raise ValueError(UNUSABLE)+ shapes = layout()+ result = []+ with safe_open(str(path), framework='pt', device='cpu') as handle:+ for name, group in sorted(_entries(handle.keys()).items()):+ if name not in shapes:+ continue # norm deltas and the like: not part of the supported targets+ a = handle.get_tensor(group['down']).to(torch.bfloat16).contiguous()+ b = handle.get_tensor(group['up']).to(torch.bfloat16).contiguous()+ rank = a.shape[0]+ alpha = float(handle.get_tensor(group['alpha']).float()) if 'alpha' in group else float(rank)+ result.append((name, a, b, float(strength) * alpha / rank))+ return result+++def file_bytes(loras):+ """Bytes the branches of these ``{'path', 'strength'}`` entries occupy once read (bf16)."""+ total = 0+ for item in loras:+ total += Path(item['path']).stat().st_size+ return total+++class Branch:+ """One LoRA's factors for one target; ``a``/``b`` live on ``home`` and move to the activation on use."""++ __slots__ = ('a', 'b', 'scale')++ def __init__(self, a, b, scale):+ self.a, self.b, self.scale = a, b, scale+++def _apply(module, raw, output):+ rows = raw.reshape(-1, raw.shape[-1])+ target = output.reshape(-1, output.shape[-1])+ if not target.is_contiguous():+ raise ValueError('LoRA projection output must be contiguous')+ for branch in module._qwen_lora:+ a, b = branch.a, branch.b+ if a.device != rows.device:+ a, b = a.to(rows.device, non_blocking=True), b.to(rows.device, non_blocking=True)+ for start in range(0, len(rows), _CHUNK):+ z = F.linear(rows[start:start + _CHUNK].to(a.dtype), a)+ target[start:start + _CHUNK].add_(F.linear(z, b).to(target.dtype), alpha=branch.scale)+ return output+++class LoraSet:+ """The branches of one request's LoRAs, attached to a DiT for the length of a with-block."""++ def __init__(self, loras, *, device='cpu', pin=True):+ self.entries = {} # module name -> [Branch]+ self.bytes = 0+ self._wrapped = []+ for item in loras:+ for name, a, b, scale in read(item['path'], item['strength']):+ if device != 'cpu':+ a, b = a.to(device), b.to(device)+ elif pin and torch.cuda.is_available():+ try:+ a, b = a.pin_memory(), b.pin_memory()+ except RuntimeError:+ pass+ self.bytes += a.numel() * a.element_size() + b.numel() * b.element_size()+ if name.endswith('.img_mlp.gate_up'):+ prefix = name[:-len('gate_up')]+ half = b.shape[0] // 2+ self.entries.setdefault(prefix + 'gate_layer', []).append(Branch(a, b[:half], scale))+ self.entries.setdefault(prefix + 'proj', []).append(Branch(a, b[half:], scale))+ else:+ self.entries.setdefault(name, []).append(Branch(a, b, scale))++ def attach(self, model):+ for name, branches in self.entries.items():+ module = model.get_submodule(name)+ module._qwen_lora = branches+ original = module.forward++ def forward(self, value, old=original):+ return _apply(self, value, old(value))+ module.forward = types.MethodType(forward, module)+ self._wrapped.append(module)+ return self++ def detach(self):+ for module in self._wrapped:+ module.__dict__.pop('forward', None)+ module.__dict__.pop('_qwen_lora', None)+ self._wrapped = []++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ self.detach()+ return Falsediff --git a/freevideo_engine/image_gen/qwen_pipeline.py b/freevideo_engine/image_gen/qwen_pipeline.pyindex 475cfa6..e724fe0 100644--- a/freevideo_engine/image_gen/qwen_pipeline.py+++ b/freevideo_engine/image_gen/qwen_pipeline.py@@ -44,6 +44,9 @@ from PIL import Imagefrom .qwen_text import QwenTextEncoder+# Free card memory that must remain after a request's LoRA branches are placed on the card; below it they+# stay in pinned host memory and move to the card per use.+LORA_GPU_MARGIN = 2 << 30TURBO_SIGMAS = (1.0, 0.978453, 0.95418, 0.926626, 0.89508, 0.845148, 0.704534, 0.414568, 0.0)@@ -257,7 +260,7 @@ class QwenImageGenerator:@torch.no_grad()def generate(self, prompt: str, *, width: int, height: int, seed: int | None = None, steps: int = 8,- reference_images=None, edit_mask=None, progress=None, cancel=None) -> Image.Image:+ reference_images=None, edit_mask=None, loras=None, progress=None, cancel=None) -> Image.Image:if self._closed:raise RuntimeError("Generator is closed")if not isinstance(prompt, str):@@ -269,6 +272,10 @@ class QwenImageGenerator:width, height = normalize_dimensions(width, height)if edit_mask is not None and not reference_images:raise ValueError("edit_mask needs a reference image: the first one is the picture being edited")+ if loras:+ from .qwen_lora import UNUSABLE, inspect_file+ if not all(inspect_file(item["path"])["ok"] for item in loras):+ raise ValueError(UNUSABLE)def check_cancel():if cancel is not None and cancel():@@ -347,7 +354,7 @@ class QwenImageGenerator:image_mask = torch.cat([image_mask, image_mask.new_ones(1, latents.shape[1] // 4)], dim=1)sigmas = torch.tensor(self.sample_sigmas, device=self.device, dtype=torch.float32)timesteps = sigmas[:-1] * 1000- with self._stage("denoise"):+ with self._stage("denoise"), self._lora_scope(loras):report("denoise", 0.0)for i, timestep in enumerate(timesteps):check_cancel()@@ -395,6 +402,23 @@ class QwenImageGenerator:report("decode", 1.0)return image+ @contextmanager+ def _lora_scope(self, loras):+ """Attach this request's LoRAs to the DiT for the denoising loop only; no LoRAs, no change."""+ if not loras:+ yield+ return+ from .qwen_lora import LoraSet+ on_card = False+ if torch.device(self.device).type == "cuda":+ floor = LORA_GPU_MARGIN + sum(Path(item["path"]).stat().st_size for item in loras)+ on_card = torch.cuda.mem_get_info(self.device)[0] > floor+ branches = LoraSet(loras, device=self.device if on_card else "cpu")+ try:+ yield branches.attach(self.transformer)+ finally:+ branches.detach()+def close(self) -> None:"""Release owned model references; all active stages have already left."""self.text_encoder = self.transformer = self.vae = Nonediff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.pyindex 6a71abb..8889c7e 100644--- a/freevideo_engine/image_gen/service.py+++ b/freevideo_engine/image_gen/service.py@@ -17,6 +17,7 @@ import timeimport uuidfrom . import image_pack, keep+from .loras import ERRORS as LORA_ERRORS, resolve_selectionfrom ..download_settings import observed_speedfrom .failure import (Context, HOST_MESSAGE_ZH, host_memory_shortage, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, UNIFIED_MESSAGE_ZH,classify_exception)@@ -34,7 +35,7 @@ INFERENCE_TIMEOUT = 600.QUEUE_TIMEOUT = 600.QUEUE_POLL = .1INVALID_ERRORS = frozenset(('invalid_request', 'invalid_prompt', 'invalid_dimensions', 'invalid_seed',- 'invalid_media', 'invalid_points', 'too_many_images'))+ 'invalid_media', 'invalid_points', 'too_many_images')) | LORA_ERRORSdef comfy_server():@@ -112,9 +113,10 @@ class Worker:class Service:- def __init__(self, installation, input_directory, *, video_busy=None, idle_seconds=45.):+ def __init__(self, installation, input_directory, *, video_busy=None, idle_seconds=45., folders=None):self.installation = installationself.input_directory = input_directory+ self.folders = foldersself.video_busy = video_busy or self._video_busyself.idle_seconds = idle_secondsself.mutex = threading.RLock()@@ -502,9 +504,18 @@ class Service:cast_view = value.get('cast_view')if cast_view is not None and (cast_view != 'back' or len(paths) != 1 or mask_path is not None):raise ValueError('invalid_request')- return dict(prompt=prompt, width=dimensions[0] // 32 * 32, height=dimensions[1] // 32 * 32,- steps=8, seed=seed, images=paths, **(dict(mask=mask_path) if mask_path else {}),- **(dict(cast_view='back') if cast_view == 'back' else {}), path=self.output_path(named=True))+ loras = resolve_selection(value.get('loras', []), folders=self.folders)+ if loras:+ from ..lora_triggers import apply_triggers+ prompt = apply_triggers(prompt, loras, folders=self.folders)+ result = dict(prompt=prompt, width=dimensions[0] // 32 * 32, height=dimensions[1] // 32 * 32,+ steps=8, seed=seed, images=paths, **(dict(mask=mask_path) if mask_path else {}),+ **(dict(cast_view='back') if cast_view == 'back' else {}), path=self.output_path(named=True),+ **(dict(loras=loras) if loras else {}))+ # Leave room for action, kind and the small size-fit record added at dispatch.+ if loras and len(json.dumps(result, ensure_ascii=True).encode('utf-8')) > BODY_LIMIT - 1024:+ raise ValueError('invalid_loras')+ return resultdef start(self, action, value):if action not in ('generate', 'compose'):@@ -1201,7 +1212,7 @@ def register():running, pending = server.prompt_queue.get_current_queue()return bool(running or pending) or Service._video_busy()- service = Service(installation, folder_paths.get_input_directory, video_busy=video_busy)+ service = Service(installation, folder_paths.get_input_directory, video_busy=video_busy, folders=folder_paths)server._freevideo_image_gen = servicetoken = secrets.token_urlsafe(24)diff --git a/freevideo_engine/image_gen/worker.py b/freevideo_engine/image_gen/worker.pyindex 0b549e3..33bfc28 100644--- a/freevideo_engine/image_gen/worker.py+++ b/freevideo_engine/image_gen/worker.py@@ -154,11 +154,13 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=Noneif value.get('mask'):region['edit_mask'] = stack.enter_context(Image.open(value['mask']))region['edit_mask'].load()+ loras = value.get('loras') or Noneback_check = Nonedef generate(seed):return runner.generate(value['prompt'], width=value['width'], height=value['height'],seed=seed, reference_images=references, **region,+ **(dict(loras=loras) if loras else {}),progress=progress, cancel=inbox.cancel.is_set)if value.get('cast_view') == 'back' and len(references) == 1 and not region:diff --git a/web/cast_generate.css b/web/cast_generate.cssindex bfd453b..828c679 100644--- a/web/cast_generate.css+++ b/web/cast_generate.css@@ -2,6 +2,11 @@.fv-cg-body{display:grid;grid-template-columns:minmax(0,2fr) minmax(0,3fr);gap:24px;align-items:start}.fv-cg [hidden]{display:none!important}.fv-cg-side{display:flex;flex-direction:column;gap:12px;min-width:0;align-self:stretch}+/* The LoRA list opens in the column, under the description, so it never leaves the dialog; its controls take the panel's look. */+.fv-cg-side .fv-ie-lora summary{display:inline-grid;box-sizing:border-box;height:32px;padding:0 9px;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);white-space:nowrap;font-variant-numeric:tabular-nums}+.fv-cg-side .fv-ie-lora-trigger::placeholder{color:var(--fv-disabled)}+.fv-cg-side .fv-ie-lora .fv-ie-lora-panel{position:static;width:auto;max-height:none;margin-top:8px;padding:0;border:0;background:none;box-shadow:none}+.fv-cg-side .fv-ie-lora select,.fv-cg-side .fv-ie-lora input:not([type=checkbox]),.fv-cg-side .fv-ie-lora button{box-sizing:border-box;min-height:30px;padding:3px 8px;border:1px solid var(--fv-border,#ffffff1f);border-radius:6px;background:var(--fv-raised,#202a38);color:inherit;font:inherit}.fv-cg-foot{display:grid;grid-template-columns:auto minmax(0,1fr) auto;align-items:center;gap:8px}.fv-cg-text{flex:1;box-sizing:border-box;width:100%;min-height:160px;resize:none;padding:12px;border:1px solid var(--fv-border,#ffffff15);border-radius:8px;background:var(--fv-card,#202a38);color:inherit;font:inherit;line-height:1.6;outline:none;transition:border-color .2s}.fv-cg-text:focus{border-color:color-mix(in srgb,var(--fv-accent,#6db8fa) 60%,transparent)}diff --git a/web/cast_generate.js b/web/cast_generate.jsindex cc2543f..99347dc 100644--- a/web/cast_generate.js+++ b/web/cast_generate.js@@ -4,6 +4,7 @@// and stay in the input folder; the dialog puts them on the card.import {createImageGenClient, imageGenProgress, imagePackUnavailableMessage, renderImageFailure,IMAGE_GEN_POLL_MS, IMAGE_GEN_INFO_POLL_MS} from './image_generation.js';+import {createImageLoras} from './image_lora.js';export {imageGenerationSupported as castGenerateSupported} from './image_generation.js';@@ -22,7 +23,7 @@ const randomSeed = () => crypto.getRandomValues(new Uint32Array(1))[0] >>> 1;export function createCastGenerator({client, onChange = () => {}, schedule = setTimeout, unschedule = clearTimeout}) {const fresh = () => CAST_VIEWS.map(view => ({view, file: '', fraction: 0}));let state = {pictures: fresh(), running: false, current: -1, error: '', report_id: null, waiting: '', description: ''};- let epoch = 0, timer = null, job = null, plan = null, seed = 0, disposed = false;+ let epoch = 0, timer = null, job = null, plan = null, seed = 0, disposed = false, loras;const emit = patch => { state = {...state, ...patch}; if (!disposed) onChange(state); };const setPicture = (index, patch) => emit({pictures: state.pictures.map((row, i) => i === index ? {...row, ...patch} : row)});const cancelJob = id => Promise.resolve().then(() => client.cancel(id)).catch(() => {});@@ -31,7 +32,8 @@ export function createCastGenerator({client, onChange = () => {}, schedule = setasync function runOne(index, alive) {const view = plan[index], front = state.pictures[0].file;const row = await client.generate({prompt: view.prompt, width: view.width, height: view.height, seed,- images: view.from_front ? [front] : [], ...(view.view === 'back' ? {cast_view: 'back'} : {})});+ images: view.from_front ? [front] : [], ...(view.view === 'back' ? {cast_view: 'back'} : {}),+ ...(loras === undefined ? {} : {loras})});if (!row.job) throw Object.assign(new Error(row.error || 'generation_failed'), {code: row.error, report_id: row.report_id, memory: row.memory});if (!alive()) { await cancelJob(row.job); return null; }job = row.job;@@ -54,14 +56,15 @@ export function createCastGenerator({client, onChange = () => {}, schedule = set}}- async function start(description) {+ async function start(description, {loras: selectedLoras} = {}) {description = String(description || '').trim();if (state.running || disposed || !description) return;// After a failure the same description goes on from the picture that failed.- const resume = !!state.error && description === state.description && plan && !done();+ const resume = !!state.error && description === state.description && plan && !done()+ && JSON.stringify(selectedLoras) === JSON.stringify(loras);const current = ++epoch;const alive = () => current === epoch && !disposed;- if (!resume) { seed = randomSeed(); plan = null; }+ if (!resume) { seed = randomSeed(); plan = null; loras = structuredClone(selectedLoras); }emit({running: true, error: '', report_id: null, waiting: '', description,...(resume ? {} : {pictures: fresh(), current: 0})});try {@@ -105,7 +108,7 @@ export function createCastGenerator({client, onChange = () => {}, schedule = setdispose() { disposed = true; return stop(); }};}-export function mountCastGenerate({container, api, text, t = text, viewURL, onState = () => {}, fetchApi, client}) {+export function mountCastGenerate({container, api, text, t = text, viewURL, onState = () => {}, fetchApi, client, loras}) {const tr = t || ((en) => en);const fetcher = fetchApi || (api ? api.fetchApi.bind(api) : globalThis.fetch.bind(globalThis));client = client || createImageGenClient(fetcher);@@ -129,8 +132,11 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onStfield.maxLength = 1000;const note = el('div', 'fv-cg-note'); note.setAttribute('role', 'status');const action = el('button', 'fv-cg-action'); action.type = 'button';+ // A caller that passes a fixed list owns the choice; otherwise the panel offers the same picker as image generation.+ const picker = loras === undefined ? createImageLoras({t: tr, fetchApi: fetcher, getPrompt: () => field.value,+ setPrompt: text => { field.value = text; }, onChange: () => render()}) : null;foot.append(note, action);- side.append(field, foot);+ side.append(field, ...(picker ? [picker.element] : []), foot);const grid = el('div', 'fv-cg-views');const cells = CAST_VIEWS.map(view => {const cell = el('div', 'fv-cg-view');@@ -157,6 +163,7 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onStconst isMissing = missing();root.dataset.state = isMissing ? 'missing' : running ? 'running' : generator.done() ? 'done' : state.error ? 'failed' : 'idle';field.readOnly = running;+ if (picker) picker.element.inert = running;action.hidden = !!isMissing;// One accent button at a time: Generate until the five pictures exist, then Regenerate is neutral.const finishedAll = generator.done() && !running;@@ -212,7 +219,8 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onStreturn;}attention = null;- void generator.start(field.value);+ const chosen = picker ? picker.value() : loras;+ void generator.start(field.value, picker && !chosen.length ? {} : {loras: chosen});}action.onclick = () => generator.state().running ? void generator.stop() : begin();field.addEventListener('input', () => {@@ -230,7 +238,7 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onStgetPictures: () => generator.done() && !generator.state().running ? generator.state().pictures.map(row => ({file: row.file, label: row.view})) : [],isBusy: () => generator.state().running,cancel: () => generator.stop(),- destroy() { disposed = true; clearTimeout(infoTimer); void generator.dispose(); root.remove(); },+ destroy() { disposed = true; picker?.dispose(); clearTimeout(infoTimer); void generator.dispose(); root.remove(); },element: root,};}diff --git a/web/health.js b/web/health.jsindex 0e2347d..3b1a292 100644--- a/web/health.js+++ b/web/health.js@@ -15,7 +15,7 @@ const launchVisit = query.get('freevideo') === 'launch';// Every script this folder ships; a test keeps the list in step with web/*.js.export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_confirm_settings.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js','editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js',- 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js',+ 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'image_lora.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js','progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_sections.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js','result_actions.js', 'result_reuse.js', 'sampling_effort.js', 'setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'replace_person.js', 'replace_state.js', 'optional_inputs.js', 'studio_queue.js', 'studio_size.js','toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js', 'whitebox_encode.js', 'whitebox_mp4.js', 'whitebox_recording.js', 'whitebox_render.js', 'whitebox_scenes.js', 'whitebox_worker.js',diff --git a/web/image_editor.css b/web/image_editor.cssindex ecb2b5b..5207ca6 100644--- a/web/image_editor.css+++ b/web/image_editor.css@@ -245,6 +245,25 @@ button.fv-ie-size:hover{background:var(--fv-hover);color:var(--fv-ink)}.fv-ie-menu summary{display:grid;place-items:center;border-radius:var(--fv-r-sm);cursor:pointer;list-style:none}.fv-ie-menu summary::-webkit-details-marker{display:none}.fv-ie-menu-list{position:absolute;right:0;bottom:calc(100% + 6px);z-index:3;width:236px;max-height:300px;overflow:auto;padding:6px;border:1px solid var(--fv-border);border-radius:var(--fv-r-md);background:var(--fv-card);box-shadow:0 14px 40px rgba(0,0,0,.5);display:flex;flex-direction:column;gap:2px}+.fv-ie-lora summary{width:auto;padding:0 9px;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);height:32px;white-space:nowrap;font-variant-numeric:tabular-nums}+.fv-ie-lora[data-active=true] summary{color:var(--fv-ink);border-color:var(--fv-accent-dim)}+.fv-ie-lora .fv-ie-lora-panel{top:calc(100% + 6px);right:0;bottom:auto;width:min(340px,calc(100vw - 24px));max-height:360px;gap:8px;padding:10px}+@media (max-width:700px){.fv-ie-lora .fv-ie-lora-panel{right:auto;left:0}}+.fv-ie-lora-list{display:flex;flex-direction:column;gap:8px}+.fv-ie-lora-row{display:grid;grid-template-columns:18px minmax(0,1fr) 64px 28px;align-items:center;gap:6px}+.fv-ie-lora-row select{min-width:0;width:100%;padding-right:22px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}+.fv-ie-lora-row input[type=number]{width:64px;padding:4px 6px;font-variant-numeric:tabular-nums}+.fv-ie-lora-row input[type=checkbox]{accent-color:var(--fv-accent);width:15px;height:15px}+.fv-ie-lora-row button{height:28px;padding:0 6px}+.fv-ie-lora-row[data-enabled=false]>:not(input[type=checkbox]){opacity:.45}+.fv-ie-lora-row .fv-ie-lora-trigger{grid-column:2/span 3;min-width:0;height:28px;box-sizing:border-box;padding:4px 8px;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);background:var(--fv-card);color:var(--fv-ink);font:inherit;font-size:var(--fv-micro)}+.fv-ie-lora-trigger::placeholder{color:var(--fv-disabled)}+.fv-ie-lora-problem{grid-column:2/span 3;margin:0;font-size:var(--fv-micro);color:var(--fv-danger)}+.fv-ie-lora-problem[hidden]{display:none}+.fv-ie-lora-row[data-unusable=true] .fv-ie-lora-trigger{display:none}.fv-ie-lora-row[data-unusable=true] input[type=number]{visibility:hidden}.fv-ie-lora-row button{justify-self:end;width:28px}+.fv-ie-lora-actions{display:flex;justify-content:center}+.fv-ie-lora-note{margin:0;font-size:var(--fv-micro);color:var(--fv-muted);text-align:center}+.fv-ie-lora-note:empty{display:none}.fv-ie-menu-label{padding:6px 8px 2px;color:var(--fv-muted);font-size:11px}.fv-ie-menu-item{display:flex;align-items:center;gap:8px;width:100%;padding:6px 8px;border:0;border-radius:6px;background:transparent;text-align:left}.fv-ie-menu-item:hover{background:var(--fv-hover)}diff --git a/web/image_generation.js b/web/image_generation.jsindex 74f174c..fbb81bd 100644--- a/web/image_generation.js+++ b/web/image_generation.js@@ -5,11 +5,12 @@ import {orientMatrix, rasterKinds, scaleSelection} from './image_editor_engine.jimport {saveReport, reportIssue, showReportTip, focusReportTip} from './report_issue.js';import {issueButton} from './issue_button.js';import {about} from './request_assets.js';+import {createImageLoras} from './image_lora.js';export const IMAGE_GEN_PROTOCOL = Object.freeze({root: '/freevideo/image-gen', header: 'X-FreeVideo-Image', upload: '/upload/image',installed: 'ready', phase: 'phase', bytes: 'bytes', downloading: 'downloading', checking: 'checking',- token: 'token', job: 'job', prompt: 'prompt', width: 'width', height: 'height', images: 'images', mask: 'mask', seed: 'seed',+ token: 'token', job: 'job', prompt: 'prompt', width: 'width', height: 'height', images: 'images', mask: 'mask', seed: 'seed', loras: 'loras',stage: 'stage', fraction: 'fraction', file: 'file', error: 'error', name: 'name', subfolder: 'subfolder',states: {queued: 'waiting', starting: 'queued', progress: 'running', complete: 'done', failed: 'error',cancelling: 'running', cancelled: 'error'},@@ -80,10 +81,10 @@ export function createImageGenClient(fetchImpl = globalThis.fetch.bind(globalThi}return {info,- async generate({prompt, width, height, images = [], mask, seed, cast_view}) {+ async generate({prompt, width, height, images = [], mask, seed, loras, cast_view}) {const value = await post('generate', {[p.prompt]: prompt, [p.width]: width, [p.height]: height,[p.images]: images, ...(mask ? {[p.mask]: mask} : {}), ...(seed === undefined ? {} : {[p.seed]: seed}),- ...(cast_view === 'back' ? {cast_view} : {})});+ ...(loras === undefined ? {} : {[p.loras]: loras}), ...(cast_view === 'back' ? {cast_view} : {})});return {...value, job: value[p.job]};},async status(job) {@@ -156,6 +157,12 @@ export function packView(info, t) {}export function imageGenError(code, info, t) {const reasons = {+ invalid_loras: ['The LoRA selection is invalid. Choose the files again.', 'LoRA 设置有误,请重新选择文件。'],+ invalid_lora_strength: ['LoRA strength must be between -2 and 2.', 'LoRA 强度必须在 -2 到 2 之间。'],+ duplicate_lora: ['Select each LoRA only once. Remove the duplicate.', '同一个 LoRA 只能选择一次,请移除重复项。'],+ too_many_loras: ['Use at most eight LoRAs per picture.', '一张图片最多使用 8 个 LoRA,请减少数量。'],+ lora_not_found: ['The LoRA file could not be found. Choose it again.', 'LoRA 文件找不到,请重新选择。'],+ invalid_lora_model: ['This LoRA cannot be used for pictures. Choose another one.', '此 LoRA 不适用于图片生成,请换一个。'],no_memory: ['Not enough GPU memory, even at the smallest size. Close other apps using the GPU, then try again.','显存不够,最小尺寸也没能生成。关掉其他占用显卡的程序后再试。'],busy_video: ['A video is generating. Try again when it finishes.', '视频正在生成,完成后再试。'],@@ -476,7 +483,9 @@ export function createImageGenSession({client, prepare, apply, onChange = () =>if (!alive()) return;if (state.stage === 'upload') emit({stage: ''});const withImages = images.length > 0;- const row = await client.generate({prompt: request.prompt, width: context.width, height: context.height, images, mask});+ const row = await client.generate({prompt: request.prompt, width: context.width, height: context.height, images, mask,+ ...(request.seed === undefined ? {} : {seed: request.seed}),+ ...(request.loras === undefined ? {} : {loras: request.loras})});const {job} = row;if (!job) throw failure(row.error, row);if (row.fitted) emit({fitted: row.fitted});@@ -548,7 +557,7 @@ export function imageJobStore(key, storage = globalThis.sessionStorage, now = Daexport function createImageGeneration({client, renderer, t, getDoc, getSelection, commit, el, button, segmented,library = [], resolve = file => file, icon = () => '', onRatioChange, onPlace, productVersion, fetchApi, resumeKey = null}) {- let mode = 'edit', prompt = '', mounted = null, references = [], tier = 'standard', past = [];+ let mode = 'edit', prompt = '', mounted = null, references = [], loras = null, tier = 'standard', past = [];let promptRequired = false, attentionTimer = null;const copies = new Map();// A reference is sent no larger than the largest picture the service makes.@@ -600,7 +609,8 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection}return;}- return session.start({mode, prompt, references: chosen()});+ const chosenLoras = loras?.value() || [];+ return session.start({mode, prompt, references: chosen(), ...(chosenLoras.length ? {loras: chosenLoras} : {})});}function drawEdited(canvas) {try {@@ -779,6 +789,7 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelectionb.disabled = disabled || (empty && b.dataset.value === 'edit');b.setAttribute('aria-pressed', String(b.dataset.value === mode));}); input.disabled = disabled;+ if (loras) { if (state.running) loras.element.setAttribute('inert', ''); else loras.element.removeAttribute('inert'); }renderReferences(refs, disabled);input.placeholder = mode === 'edit' ? getSelection() ? t('What to change in the selection', '要改选区里的什么') : t('What to change', '要改什么') : t('Describe the picture', '描述画面');input.setAttribute('aria-label', input.placeholder);@@ -845,7 +856,10 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelectionrefs.setAttribute('aria-label', t('Reference images', '参考图'));const box = el('div', null, 'fv-ie-gen-box');const input = el('textarea', null, 'fv-ie-gen-prompt'); input.rows = 1; input.maxLength = 2000; input.value = prompt;- box.append(input); field.append(refs, box, historyMenu());+ box.append(input);+ loras ||= createImageLoras({t, fetchApi, el, onChange: () => mounted && update(), getPrompt: () => prompt,+ setPrompt: value => { prompt = value; if (mounted) { mounted.input.value = value; mounted.grow(); } }});+ field.append(refs, box, historyMenu(), loras.element);const note = el('div', '', 'fv-ie-opt-note fv-ie-gen-note'); note.setAttribute('role', 'status');const action = button(t('Generate', '生成'), null, () => {const now = session.state(), pack = !now.running && now.info && !now.info.installed ? packView(now.info, t) : null;@@ -858,10 +872,10 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelectionbar.append(group);if (shape) bar.append(shape);bar.append(field, note, action, meter);- mounted = {bar, input, action, note, meter, fill, group, refs, ratio, size};const grow = () => {input.style.height = '0px';input.style.height = (prompt.trim() ? Math.min(80, Math.max(32, input.scrollHeight + 2)) : 32) + 'px';+ box.style.height = input.style.height;};input.oninput = () => {prompt = input.value;@@ -873,6 +887,7 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelectionevent.preventDefault(); void start();}};+ mounted = {bar, input, action, note, meter, fill, group, refs, ratio, size, grow};grow(); update();}return {mount, drawsSelection: () => mode === 'edit' && getDoc().layers.length > 0, unmount() { clearAttention(); promptRequired = false; mounted = null; },@@ -880,5 +895,5 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection// The blank canvas a reloaded generation was started on, to be set again by the editor.resumedCanvas: resumed?.context?.mode === 'new'? {width: resumed.context.doc.width, height: resumed.context.doc.height} : null,- dispose() { clearAttention(); return session.dispose(); }, state: session.state};+ dispose() { clearAttention(); loras?.dispose(); return session.dispose(); }, state: session.state};}diff --git a/web/image_lora.js b/web/image_lora.jsnew file mode 100644index 0000000..b036be5--- /dev/null+++ b/web/image_lora.js@@ -0,0 +1,183 @@+// A compact LoRA picker for image generation, with the video panel's rows: switch, file,+// strength, trigger words. It keeps its own list in this browser (the image editor has no+// graph node to hold it) and puts trigger words into the prompt the way the video panel does.+import {reconcileLoraTriggers} from './lora_triggers.js';+import {createVerdicts} from './lora_check.js';++export const IMAGE_LORA_LIMIT = 8, IMAGE_LORA_STRENGTH = 2;+const STORE = 'freevideo_image_loras';+const domEl = (tag, label, cls) => { const e = document.createElement(tag); if (label != null) e.textContent = label; if (cls) e.className = cls; return e; };++/** The rows worth sending: switched on, a real strength, a file that is there and fits. */+export function activeLoras(rows, {names = null, verdicts = null} = {}) {+ return rows.filter(row => row.enabled !== false && Number(row.strength) !== 0+ && (!names || names.includes(row.name)) && verdicts?.peek(row.name)?.usable !== false)+ .map(row => ({name: row.name, strength: Number(row.strength), enabled: true}));+}++export function createImageLoras({t, fetchApi, el = domEl, getPrompt = () => '', setPrompt = null, onChange = () => {}, storage = globalThis.localStorage}) {+ let rows = [], records = [], names = null, disposed = false, skipped = 0;+ const words = new Map(), loading = new Map(), drafts = new Map();+ const verdicts = createVerdicts(fetchApi, {kind: 'image'});+ try {+ const saved = JSON.parse(storage?.getItem(STORE) || 'null');+ if (Array.isArray(saved?.rows)) rows = saved.rows.filter(r => r && typeof r.name === 'string').slice(0, IMAGE_LORA_LIMIT)+ .map(r => ({name: r.name, strength: Number.isFinite(Number(r.strength)) ? Number(r.strength) : 1, enabled: r.enabled !== false}));+ // Which trigger words this panel put into a prompt is not carried over: a new description box starts unowned.+ } catch { /* an unreadable saved list starts empty */ }++ const menu = el('details', null, 'fv-ie-menu fv-ie-lora');+ const summary = el('summary', null, 'fv-ie-lora-summary');+ const panel = el('div', null, 'fv-ie-menu-list fv-ie-lora-panel');+ const list = el('div', null, 'fv-ie-lora-list'), actions = el('div', null, 'fv-ie-lora-actions'), note = el('p', '', 'fv-ie-lora-note');+ const add = el('button', t('Add', '添加')); add.type = 'button';+ actions.append(add); panel.append(list, actions, note); menu.append(summary, panel);+ summary.setAttribute('aria-label', 'LoRA');++ const usable = () => activeLoras(rows, {names, verdicts});+ function save() {+ try { storage?.setItem(STORE, JSON.stringify({rows})); } catch { /* private mode */ }+ }+ function drawSummary() {+ const count = usable().length;+ summary.textContent = count ? `LoRA · ${count}` : 'LoRA';+ menu.dataset.active = String(count > 0);+ }+ function syncPrompt() {+ if (!setPrompt) return;+ const next = reconcileLoraTriggers(getPrompt(), records, usable(),+ Object.fromEntries([...words].filter(([, w]) => typeof w === 'string')));+ records = next.records;+ if (next.prompt !== getPrompt()) setPrompt(next.prompt);+ }+ function changed() { syncPrompt(); save(); drawSummary(); onChange(); }++ async function triggerWords(name) {+ if (words.has(name)) return words.get(name);+ if (!loading.has(name)) loading.set(name, (async () => {+ // The first read of a big file hashes it once on the server; keep asking while that finishes.+ for (let attempt = 0; attempt < 10; attempt++) {+ try {+ const response = await fetchApi('/freevideo/lora/triggers?' + new URLSearchParams({name}),+ {cache: 'no-store', signal: AbortSignal.timeout(6500)});+ const result = await response.json();+ if (response.ok && typeof result?.words === 'string') { words.set(name, result.words); return result.words; }+ break;+ } catch (error) { if (error?.name !== 'TimeoutError') break; }+ }+ throw new Error(t('Could not read trigger words.', '无法读取触发词。'));+ })().finally(() => loading.delete(name)));+ return loading.get(name);+ }+ async function saveWords(name, value) {+ let result = null;+ try {+ const response = await fetchApi('/freevideo/lora/triggers', {method: 'POST', headers: {'Content-Type': 'application/json'},+ body: JSON.stringify({name, words: value.trim()})});+ result = await response.json();+ if (!response.ok || typeof result?.words !== 'string') result = null;+ } catch { /* same sentence below */ }+ if (!result) throw new Error(t('Could not save. Try again.', '没有保存,请再试一次。'));+ words.set(name, result.words);+ }++ function problemText(name, verdict) {+ if (names && !names.includes(name)) return t('The file is no longer in the LoRA folder.', '此文件已不在 LoRA 文件夹中。');+ if (verdict?.usable !== false) return '';+ return verdict.reason === 'unreadable'+ ? t('The file could not be read. It may be incomplete; download it again.', '文件无法读取,可能未下载完整,请重新下载。')+ : t('This LoRA cannot be used for pictures.', '此 LoRA 不适用于图片生成。');+ }++ function render() {+ if (disposed) return;+ list.replaceChildren();+ rows.forEach((row, index) => {+ const wrap = el('div', null, 'fv-ie-lora-row'); wrap.dataset.enabled = String(row.enabled !== false);+ const on = el('input'); on.type = 'checkbox'; on.checked = row.enabled !== false;+ on.setAttribute('aria-label', t('Enable LoRA', '启用 LoRA'));+ const select = el('select'); select.setAttribute('aria-label', t('LoRA model', 'LoRA 模型'));+ for (const name of [...new Set([...(names || []), row.name])]) { const option = el('option', name); option.value = name; select.append(option); }+ select.value = row.name; select.title = row.name;+ const strength = el('input'); strength.type = 'number'; strength.step = '.05';+ strength.min = String(-IMAGE_LORA_STRENGTH); strength.max = String(IMAGE_LORA_STRENGTH); strength.value = row.strength;+ strength.setAttribute('aria-label', t('LoRA strength', 'LoRA 强度'));+ const remove = el('button', '×'); remove.type = 'button'; remove.setAttribute('aria-label', t('Remove LoRA', '移除 LoRA'));+ const trigger = el('input', null, 'fv-ie-lora-trigger'); trigger.type = 'text'; trigger.maxLength = 300;+ trigger.placeholder = t('Trigger words', '触发词'); trigger.setAttribute('aria-label', trigger.placeholder);+ trigger.value = drafts.get(row.name) ?? words.get(row.name) ?? '';+ const problem = el('p', '', 'fv-ie-lora-problem'); problem.hidden = true;+ const show = () => {+ const text = problemText(row.name, verdicts.peek(row.name));+ problem.textContent = text; problem.hidden = !text;+ wrap.dataset.unusable = String(!!text);+ on.disabled = !!text; strength.disabled = trigger.disabled = !!text || row.enabled === false;+ };+ on.onchange = () => { row.enabled = on.checked; wrap.dataset.enabled = String(on.checked); show(); changed(); };+ select.onchange = () => {+ row.name = select.value; select.title = row.name; trigger.value = words.get(row.name) ?? '';+ if (verdicts.peek(row.name)?.usable === false) row.enabled = true;+ on.checked = row.enabled !== false; check(); changed();+ };+ strength.onchange = () => {+ const value = Number(strength.value);+ if (strength.value === '' || !(Math.abs(value) <= IMAGE_LORA_STRENGTH)) {+ strength.setCustomValidity(t(`Enter a value from -${IMAGE_LORA_STRENGTH} to ${IMAGE_LORA_STRENGTH}.`, `请输入 -${IMAGE_LORA_STRENGTH} 到 ${IMAGE_LORA_STRENGTH} 之间的数。`));+ strength.reportValidity(); return;+ }+ strength.setCustomValidity(''); row.strength = value; changed();+ };+ remove.onclick = () => { rows.splice(index, 1); render(); changed(); };+ trigger.onchange = async () => {+ const name = row.name, draft = trigger.value;+ drafts.set(name, draft);+ try { await saveWords(name, draft); drafts.delete(name); note.textContent = ''; syncPrompt(); save(); onChange(); }+ catch (error) { note.textContent = error.message; }+ };+ const check = () => verdicts.get(row.name).then(() => { if (!disposed && wrap.isConnected && select.value === row.name) { show(); drawSummary(); syncPrompt(); onChange(); } });+ wrap.append(on, select, strength, remove, trigger, problem);+ list.append(wrap); show();+ if (!verdicts.peek(row.name)) check();+ triggerWords(row.name).then(() => {+ if (!disposed && trigger.isConnected && !drafts.has(row.name)) trigger.value = words.get(row.name) ?? '';+ syncPrompt(); save(); drawSummary(); onChange();+ }, error => { note.textContent = error.message; });+ });+ add.disabled = rows.length >= IMAGE_LORA_LIMIT;+ if (!note.textContent || !rows.some(r => drafts.has(r.name))) note.textContent = skipped+ ? t(`${skipped} other LoRA file${skipped === 1 ? '' : 's'} in the folder ${skipped === 1 ? 'is' : 'are'} not in .safetensors format and not listed.`, `文件夹中另有 ${skipped} 个 LoRA 文件不是 .safetensors 格式,未列出。`) : '';+ drawSummary();+ }++ async function refresh() {+ try {+ const response = await fetchApi('/models/loras', {cache: 'no-store'});+ if (!response.ok) throw new Error(response.statusText);+ const listed = (await response.json()).filter(n => typeof n === 'string');+ skipped = listed.filter(n => /\.(ckpt|pt|pth|bin|sft)$/i.test(n)).length;+ names = listed.filter(n => n.toLowerCase().endsWith('.safetensors'));+ verdicts.clear(); render(); changed();+ } catch (error) {+ if (!disposed) note.textContent = t('Could not read the LoRA folder.', '无法读取 LoRA 文件夹。');+ }+ }+ add.onclick = async () => {+ await refresh();+ if (rows.length >= IMAGE_LORA_LIMIT) { note.textContent = t(`Use at most ${IMAGE_LORA_LIMIT} LoRAs`, `最多使用 ${IMAGE_LORA_LIMIT} 个 LoRA`); return; }+ if (!names?.length) { note.textContent = t('Put LoRA files (.safetensors) in the models/loras folder, then choose "Add".', '把 LoRA 文件(.safetensors)放进 models/loras 文件夹,再点“添加”。'); return; }+ rows.push({name: names.find(n => !rows.some(r => r.name === n)) ?? names[0], strength: 1, enabled: true});+ render(); changed();+ };+ menu.addEventListener('toggle', () => { if (menu.open) refresh(); });+ render();+ return {+ element: menu,+ /** The LoRAs to send with the request; empty when none is switched on. */+ value: usable,+ count: () => usable().length,+ refresh,+ /** After the prompt box is cleared or replaced, trigger words are put back for the rows still on. */+ reconcile: syncPrompt,+ dispose() { disposed = true; },+ };+}diff --git a/web/lora_check.js b/web/lora_check.jsindex 1889d7a..344c1f0 100644--- a/web/lora_check.js+++ b/web/lora_check.js@@ -43,7 +43,7 @@ export function feedbackDraft(name, verdict, described, t = (en) => en) {}/** Verdicts by name. Refresh also checks the currently installed model; failures say nothing. */-export function createVerdicts(fetchApi) {+export function createVerdicts(fetchApi, {kind = null} = {}) {let known = new Map(), pending = new Map();return {peek: name => known.get(name),@@ -53,7 +53,7 @@ export function createVerdicts(fetchApi) {if (!pending.has(name)) {const mine = pending, request = (async () => {try {- const response = await fetchApi('/freevideo/lora/check?' + new URLSearchParams({name}), {cache: 'no-store'});+ const response = await fetchApi('/freevideo/lora/check?' + new URLSearchParams(kind ? {name, kind} : {name}), {cache: 'no-store'});const verdict = response.ok ? await response.json() : null;if (mine !== pending) return null;if (verdict && typeof verdict.usable === 'boolean') known.set(name, verdict);