Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Align cast back-view body direction and bound orientation retries

Bowen Xueca67320347dc
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py
index 6d25a5a..8328b1c 100644
--- a/freevideo_engine/agent/jobs.py
+++ b/freevideo_engine/agent/jobs.py
@@ -1841,6 +1841,9 @@ class Jobs:
else:
action = 'generate'
value.update(prompt=spec['prompt'], images=[image_reference(self.port, name) for name in spec['images']])
+ metadata = spec.get('metadata') or {}
+ if metadata.get('cast_card') and metadata.get('cast_part') == 'back':
+ value['cast_view'] = 'back'
if row.get('_cancel'):
raise JobError('cancelled', IMAGE_ERRORS['cancelled'])
row['_image'] = self.port.image_start(action, value)
diff --git a/freevideo_engine/image_gen/back_view.py b/freevideo_engine/image_gen/back_view.py
new file mode 100644
index 0000000..4dd3e1d
--- /dev/null
+++ b/freevideo_engine/image_gen/back_view.py
@@ -0,0 +1,146 @@
+"""Optional local body-orientation check for cast backs; never downloads weights.
+
+The installation includes torchvision, but not pose weights. Without a local
+checkpoint this check is skipped. Face visibility is deliberately not a proxy
+for torso direction: a backwards head on a frontal body is still rejected.
+"""
+from dataclasses import dataclass
+import math
+from pathlib import Path
+import secrets
+import time
+
+MAX_RETRIES = 2
+_POSE_FILE = 'keypointrcnn_resnet50_fpn_coco-fc266e95.pth'
+
+
+@dataclass(frozen=True)
+class Assessment:
+ body: str
+ back_score: float
+
+ def __post_init__(self):
+ if (self.body not in ('front', 'back', 'side', 'unknown')
+ or not math.isfinite(self.back_score) or not 0 <= self.back_score <= 1):
+ raise ValueError('invalid body assessment')
+
+
+def assess_keypoints(points, scores, box):
+ """COCO anatomical left/right shoulders and hips, independent of the head.
+
+ Low-confidence, occluded, overlapping or conflicting joints abstain. This
+ cannot judge garment fronts or guarantee that a hallucinated pose is sound.
+ """
+ width = box[2] - box[0]
+ joints = (5, 6, 11, 12) # left/right shoulder, left/right hip
+ if (not math.isfinite(width) or width <= 0 or len(points) < 17 or len(scores) < 17
+ or any(points[i][2] <= 0 or scores[i] < 2. for i in joints)):
+ return Assessment('unknown', .5)
+ shoulders = (points[6][0] - points[5][0]) / width
+ hips = (points[12][0] - points[11][0]) / width
+ if not all(math.isfinite(v) for v in (shoulders, hips)):
+ return Assessment('unknown', .5)
+ score = .5 + .5 * max(-1., min(1., (shoulders + hips) / 2))
+ if shoulders > .1 and hips > .1:
+ return Assessment('back', score)
+ if shoulders < -.1 and hips < -.1:
+ return Assessment('front', score)
+ return Assessment('side', .5)
+
+
+def load_detector(pack_dir):
+ """Use an already present torchvision checkpoint, on CPU; absent means skip.
+
+ A pack may supply detectors/<filename>, or an existing Torch Hub cache may
+ supply it. No package installation, network access, GPU residency or eager
+ torch import on installations without that checkpoint.
+ """
+ import os
+ cache = Path(os.environ.get('TORCH_HOME') or
+ Path(os.environ.get('XDG_CACHE_HOME') or Path.home() / '.cache') / 'torch')
+ try:
+ checkpoint = next((p for p in (Path(pack_dir) / 'detectors' / _POSE_FILE,
+ cache / 'hub' / 'checkpoints' / _POSE_FILE) if p.is_file()), None)
+ except OSError:
+ return None
+ if checkpoint is None:
+ return None
+ try:
+ import torch
+ from torchvision.models.detection import keypointrcnn_resnet50_fpn
+ from torchvision.transforms.functional import pil_to_tensor
+ model = keypointrcnn_resnet50_fpn(weights=None, weights_backbone=None,
+ min_size=512, max_size=768)
+ model.load_state_dict(torch.load(checkpoint, map_location='cpu', weights_only=True))
+ model.eval()
+ except Exception:
+ # An optional cache/dependency failure must not fail image generation.
+ return None
+
+ def detect(image):
+ with torch.inference_mode():
+ prediction = model([pil_to_tensor(image.convert('RGB')).float() / 255])[0]
+ people = [i for i, score in enumerate(prediction['scores'].tolist()) if score >= .9]
+ if len(people) != 1:
+ return Assessment('unknown', .5)
+ i = people[0]
+ return assess_keypoints(prediction['keypoints'][i].tolist(),
+ prediction['keypoints_scores'][i].tolist(), prediction['boxes'][i].tolist())
+ return detect
+
+
+def generate_back(generate, seed, detector, *, cancel=None):
+ """At most three candidates, retaining the highest BODY back score.
+
+ ``generate(seed)`` uses the unchanged front reference, dimensions and steps.
+ A detector failure/abstention returns the best existing image. Diagnostics
+ contain fixed labels, seeds and timings, never prompts or user paths.
+ """
+ cancelled = cancel or (lambda: False)
+
+ def check_cancel():
+ if cancelled():
+ raise InterruptedError('cancelled')
+
+ started = time.monotonic()
+ check_cancel()
+ initial = seed if seed is not None or detector is None else secrets.randbits(64)
+ best = generate(initial)
+ diagnostics = dict(retries=0, selected_seed=initial, checks=[], extra_generation_seconds=0.,
+ check_seconds=0., skipped='detector_unavailable' if detector is None else '')
+ if detector is None:
+ return best, diagnostics
+ best_score = -1.
+ image = best
+ current_seed = initial
+ for attempt in range(MAX_RETRIES + 1):
+ check_cancel()
+ check_started = time.monotonic()
+ try:
+ assessment = detector(image)
+ if not isinstance(assessment, Assessment):
+ raise ValueError('invalid body assessment')
+ except InterruptedError:
+ raise
+ except Exception:
+ diagnostics['skipped'] = 'detector_failed'
+ # Retain the earlier assessed best; the first image if no check worked.
+ diagnostics['check_seconds'] += time.monotonic() - check_started
+ break
+ diagnostics['check_seconds'] += time.monotonic() - check_started
+ check_cancel()
+ diagnostics['checks'].append(dict(seed=current_seed, body=assessment.body,
+ back_score=assessment.back_score))
+ if assessment.back_score > best_score:
+ best, best_score = image, assessment.back_score
+ diagnostics['selected_seed'] = current_seed
+ if assessment.body != 'front' or attempt == MAX_RETRIES:
+ break
+ current_seed = (initial + (attempt + 1) * 104729) % 2**64
+ check_cancel()
+ retry_started = time.monotonic()
+ image = generate(current_seed)
+ diagnostics['extra_generation_seconds'] += time.monotonic() - retry_started
+ diagnostics['retries'] += 1
+ diagnostics['seconds'] = time.monotonic() - started
+ return best, diagnostics
diff --git a/freevideo_engine/image_gen/cast_views.py b/freevideo_engine/image_gen/cast_views.py
index a316d03..012b99c 100644
--- a/freevideo_engine/image_gen/cast_views.py
+++ b/freevideo_engine/image_gen/cast_views.py
@@ -29,8 +29,9 @@ _VIEW_TEXT = {
'with both eyes visible.'),
'side': ('将图1中的同一个人完全转向画面左边,呈现90度正侧面。鼻尖、胸口、脚尖朝向画面左边,'
'只能看见一只眼睛,近侧肩膀遮住远侧肩膀。'),
- 'back': ('Turn the person in image 1 all the way around. Show the back of the head and body. '
- 'The face is hidden. Show the back of the clothes, without front buttons or a shirt opening.'),
+ 'back': ('将图1中的同一个人整个身体转过180度,完全背对镜头。'
+ '头、肩膀、躯干、臀部和双脚一起转过去,后脑勺、发旋、颈后、背部、肩胛骨、臀部和脚后跟朝向镜头。'
+ '头和身体方向一致。看不到脸、胸口、衣服前襟、前面的扣子或衣领开口,耳朵只露边缘。只展示衣服背面。'),
'face': 'front-facing face close-up, framed from shoulders upward, centered head, eyes looking at the camera, neutral expression',
}
diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py
index a0b2808..3e3e60c 100644
--- a/freevideo_engine/image_gen/service.py
+++ b/freevideo_engine/image_gen/service.py
@@ -432,9 +432,12 @@ class Service:
raise ValueError('invalid_media')
except OSError:
raise ValueError('invalid_media') from None
+ cast_view = value.get('cast_view')
+ if cast_view is not None and (cast_view != 'back' or len(paths) != 1 or mask_path is not None):
+ raise ValueError('invalid_request')
return dict(prompt=prompt, width=dimensions[0] // 32 * 32, height=dimensions[1] // 32 * 32,
steps=8, seed=seed, images=paths, **(dict(mask=mask_path) if mask_path else {}),
- path=self.output_path(named=True))
+ **(dict(cast_view='back') if cast_view == 'back' else {}), path=self.output_path(named=True))
def start(self, action, value):
if action not in ('generate', 'compose'):
diff --git a/freevideo_engine/image_gen/worker.py b/freevideo_engine/image_gen/worker.py
index d97d807..f45022d 100644
--- a/freevideo_engine/image_gen/worker.py
+++ b/freevideo_engine/image_gen/worker.py
@@ -90,6 +90,8 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None
inbox = Inbox(stream, send)
runner = None
+ back_detector = None
+ back_detector_loaded = False
try:
while not inbox.closed.is_set():
try:
@@ -124,11 +126,22 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None
if value.get('mask'):
region['edit_mask'] = stack.enter_context(Image.open(value['mask']))
region['edit_mask'].load()
- image = runner.generate(
- value['prompt'], width=value['width'], height=value['height'],
- seed=value.get('seed'), reference_images=references, **region,
- progress=progress,
- cancel=inbox.cancel.is_set)
+ back_check = None
+ def generate(seed):
+ return runner.generate(
+ value['prompt'], width=value['width'], height=value['height'],
+ seed=seed, reference_images=references, **region,
+ progress=progress, cancel=inbox.cancel.is_set)
+
+ if value.get('cast_view') == 'back' and len(references) == 1 and not region:
+ from .back_view import generate_back, load_detector
+ if not back_detector_loaded:
+ back_detector = load_detector(pack_dir)
+ back_detector_loaded = True
+ image, back_check = generate_back(generate, value.get('seed'), back_detector,
+ cancel=inbox.cancel.is_set)
+ else:
+ image = generate(value.get('seed'))
if inbox.cancel.is_set():
raise InterruptedError('cancelled')
# The parent chooses a fresh path under ComfyUI input.
@@ -136,6 +149,8 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None
with Path(value['path']).open('xb') as target:
image.save(target, format='PNG')
plan = runner.stats().get('plan')
+ if back_check is not None:
+ plan = dict(plan or {}, back_check=back_check)
result = dict(phase='result', path=value['path'], seconds=time.monotonic() - started, plan=plan)
except Exception as error:
code = classify_exception(error, stage=context.timing.stage, cancelled=inbox.cancel.is_set())
diff --git a/web/cast_generate.js b/web/cast_generate.js
index c959787..0d861dc 100644
--- a/web/cast_generate.js
+++ b/web/cast_generate.js
@@ -31,7 +31,7 @@ export function createCastGenerator({client, onChange = () => {}, schedule = set
async function runOne(index, alive) {
const view = plan[index], front = state.pictures[0].file;
const row = await client.generate({prompt: view.prompt, width: view.width, height: view.height, seed,
- images: view.from_front ? [front] : []});
+ images: view.from_front ? [front] : [], ...(view.view === 'back' ? {cast_view: 'back'} : {})});
if (!row.job) throw Object.assign(new Error(row.error || 'generation_failed'), {code: row.error, report_id: row.report_id});
if (!alive()) { await cancelJob(row.job); return null; }
job = row.job;
diff --git a/web/image_generation.js b/web/image_generation.js
index b6da368..d1ea405 100644
--- a/web/image_generation.js
+++ b/web/image_generation.js
@@ -72,9 +72,10 @@ export function createImageGenClient(fetchImpl = globalThis.fetch.bind(globalThi
}
return {
info,
- async generate({prompt, width, height, images = [], mask, seed}) {
+ async generate({prompt, width, height, images = [], mask, seed, cast_view}) {
const value = await post('generate', {[p.prompt]: prompt, [p.width]: width, [p.height]: height,
- [p.images]: images, ...(mask ? {[p.mask]: mask} : {}), ...(seed === undefined ? {} : {[p.seed]: seed})});
+ [p.images]: images, ...(mask ? {[p.mask]: mask} : {}), ...(seed === undefined ? {} : {[p.seed]: seed}),
+ ...(cast_view === 'back' ? {cast_view} : {})});
return {...value, job: value[p.job]};
},
async status(job) {