Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Agent/cast: every character carries a sheet of all views plus the face close-up

Bowen Xued372c2d0f495
diff --git a/freevideo_engine/agent/cast.py b/freevideo_engine/agent/cast.py
index 1b32163..5f61162 100644
--- a/freevideo_engine/agent/cast.py
+++ b/freevideo_engine/agent/cast.py
@@ -40,6 +40,18 @@ TURN = dict(aspect='9:16', seconds=6)
CLOSE_UP = dict(aspect='1:1', seconds=4)
SCENE = dict(aspect='16:9', seconds=6)
+# The sheet of every view a shot carries for each person, put together here from the card's own pictures: up to six
+# equal cells (three to a row) on a light grey ground, each picture scaled whole and centred, no text.
+SHEET_VERSION = 1
+SHEET_ORDER = ('front', 'three_quarter', 'side', 'back', 'face') # then any other label, in card order
+SHEET_CELL = (736, 828)
+SHEET_GAP = 48
+SHEET_GROUND = (238, 238, 238)
+SHEET_COLUMNS = 3
+SHEET_MAX = 6
+SHEET_SIZE = (SHEET_COLUMNS * SHEET_CELL[0] + (SHEET_COLUMNS + 1) * SHEET_GAP, 2 * SHEET_CELL[1] + 3 * SHEET_GAP) # 2400 x 1800
+_SHEETS_SEEN = set()
+
def reference_prompt(card, *, face=False):
"""The one place the reference clip prompts are written."""
@@ -99,6 +111,8 @@ def view(card):
value['voice'] = dict(deepcopy(card['voice']), media=media_view(card['voice']['media']),
clip=assets.view(card['voice']['clip']['file']))
value['generate_error'] = errors.with_reason(value.get('generate_error'))
+ if card.get('_sheet'):
+ value['sheet'] = media_view(card['_sheet']) # the sheet made from the pictures, when the card has none of its own
return value
@@ -364,7 +378,12 @@ class Cast:
def get(self, identity):
with self.changed:
- return view(self._record(identity)['card'])
+ card = deepcopy(self._record(identity)['card'])
+ if card.get('kind') == 'character':
+ sheet = self.sheet(card)
+ if sheet is not None:
+ card['_sheet'] = sheet
+ return view(card)
def list(self, *, q=None, kind=None, origin=None, production=None, limit=50):
with self.changed:
@@ -544,10 +563,49 @@ class Cast:
# Filming ------------------------------------------------------------------------------------------
def snapshot(self, identity):
- """The current card for a submission, or None once it is deleted or unknown."""
+ """The current card for a submission, or None once it is deleted or unknown. A card with several pictures
+ and no sheet of its own carries the one made from them as `_sheet` (never part of the public card)."""
with self.changed:
record = self.records.get(identity)
- return deepcopy(record['card']) if record else None
+ card = deepcopy(record['card']) if record else None
+ if card is not None and card.get('kind') == 'character':
+ sheet = self.sheet(card)
+ if sheet is not None:
+ card['_sheet'] = sheet
+ return card
+
+ def sheet(self, card):
+ """The sheet of every view made from this card's pictures, as a media item, or None when it has no sheet to
+ make (fewer than two pictures, or one cannot be read). The file is named by what it is made from, so a
+ picture changed gives a new one and the same pictures reuse the old."""
+ if any(row['label'] == 'sheet' for row in card.get('images') or []):
+ return None
+ try:
+ port = self.jobs.port
+ rank = {label: i for i, label in enumerate(SHEET_ORDER)}
+ rows = sorted(enumerate(card.get('images') or []), key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0]))
+ files = [port.input_path(row['media']['file']) for _, row in rows[:SHEET_MAX]
+ if isinstance(row['media'].get('file'), str)]
+ if len(files) < 2:
+ return None
+ pictures = [path.read_bytes() for path in files] # the same bytes are named and drawn, so a picture
+ digest = hashlib.sha256(str(SHEET_VERSION).encode()) # changed meanwhile cannot sit under the old name
+ for data in pictures:
+ digest.update(hashlib.sha256(data).digest())
+ relative = 'freevideo/agent/%s/sheet-%s.png' % (card['id'], digest.hexdigest()[:16])
+ target = port.input_path(relative, create=True)
+ if not _sheet_intact(target):
+ temporary = target.with_name(target.name + '.' + uuid.uuid4().hex[:8] + '.tmp')
+ try:
+ compose_sheet(pictures, temporary)
+ os.replace(temporary, target)
+ finally:
+ temporary.unlink(missing_ok=True)
+ # A sheet is never deleted here: a queued shot or an earlier answer may still name it.
+ return dict(file=relative)
+ except Exception: # A sheet is a help: without one the person carries the face alone.
+ logging.getLogger(__name__).warning('no sheet for card %s', card.get('id'), exc_info=True)
+ return None
def used(self, identities, film):
"""Record that a film uses these cards; a film never leaves used_by once it is there."""
@@ -881,6 +939,61 @@ class Cast:
self.changed.notify_all()
+def _sheet_intact(path):
+ """Whether this is a whole sheet: a file left half written or cut short is made again. A file already checked
+ and unchanged is not decoded twice."""
+ try:
+ status = path.stat()
+ key = (str(path), status.st_size, status.st_mtime_ns)
+ if key in _SHEETS_SEEN:
+ return True
+ from PIL import Image
+ with Image.open(path) as opened:
+ if opened.size != SHEET_SIZE:
+ return False
+ opened.load()
+ except Exception:
+ return False
+ if len(_SHEETS_SEEN) > 256:
+ _SHEETS_SEEN.clear()
+ _SHEETS_SEEN.add(key)
+ return True
+
+
+def compose_sheet(pictures, out):
+ """Write one PNG of these pictures (two to six, as file bytes or paths): always three columns by two rows of
+ equal cells, a light grey ground (cells with no picture stay empty), each picture scaled whole to fit its cell
+ and centred, nothing cropped, no text."""
+ import io
+ from PIL import Image, ImageOps
+ (width, height), gap = SHEET_CELL, SHEET_GAP
+ sheet = Image.new('RGB', SHEET_SIZE, SHEET_GROUND)
+ for index, source in enumerate(pictures[:SHEET_MAX]):
+ with Image.open(io.BytesIO(source) if isinstance(source, bytes) else source) as opened:
+ picture = ImageOps.exif_transpose(opened)
+ if picture.mode in ('RGBA', 'LA', 'P'):
+ flat = Image.new('RGB', picture.size, SHEET_GROUND)
+ picture = picture.convert('RGBA')
+ flat.paste(picture, mask=picture.getchannel('A'))
+ picture = flat
+ picture = ImageOps.contain(picture.convert('RGB'), (width, height), Image.LANCZOS)
+ row, column = divmod(index, SHEET_COLUMNS)
+ sheet.paste(picture, (gap + column * (width + gap) + (width - picture.width) // 2,
+ gap + row * (height + gap) + (height - picture.height) // 2))
+ sheet.save(out, format='PNG')
+
+
+def continued_pictures(cards):
+ """The pictures a shot that continues the previous one carries for these cards (the current cards, from
+ `Cast.snapshot`), at most three: a single person carries the sheet then the face close-up; two or three people
+ carry one each, the face (else the front view, the three-quarter view, the first other picture), no sheet.
+ Each is a media item."""
+ from . import drama
+ cards = list(cards)[:spec.CONTINUED_PICTURES]
+ return [media for card in cards
+ for media in drama.person_pictures(card, alone=len(cards) == 1, continues=True)][:spec.CONTINUED_PICTURES]
+
+
def _unreadable(error):
"""A clip that is gone or cannot be decoded, rather than a fault of ours."""
if isinstance(error, (JobError, ValueError, OSError)):
diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.py
index 4d7f320..96deda1 100644
--- a/freevideo_engine/agent/comfy.py
+++ b/freevideo_engine/agent/comfy.py
@@ -101,13 +101,6 @@ class ComfyPort:
cache = getattr(self.server, '_freevideo_capabilities', None)
return cache.get(_query(query)) if hasattr(cache, 'get') else dict(UNAVAILABLE)
- def prepared_precision(self):
- """'int8' or 'fp8', the family of the model setup prepared for this machine; None while unknown."""
- cache = getattr(self.server, '_freevideo_capabilities', None)
- snapshot = getattr(cache, 'snapshot', None)
- saved = snapshot[0].get('prepared_format', 'int8') if snapshot else None
- return next((p for p in ('int8', 'fp8') if isinstance(saved, str) and saved.startswith(p)), None)
-
def output_summary(self, relative):
from .jobs import library_id
from ..comfy_library import _report
diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py
index a9ce60e..8908a9d 100644
--- a/freevideo_engine/agent/drama.py
+++ b/freevideo_engine/agent/drama.py
@@ -61,18 +61,9 @@ class Screenplay(ValueError):
CAPTIONED = 3 # the score of a take with unwanted captions
CARD = re.compile(r'c[0-9a-f]{12}') # characters.<key>.card names a cast library card
# How a shot carries its cast cards. Change these after the reference tests; nothing else needs to.
-PICTURE_ORDER = ('sheet', 'front', 'three_quarter', 'face') # then any other label, in card order
-PICTURES_EACH = 6
-PICTURES_LEAST = 3 # What a person carried before six: a device that cannot hold more still shoots with these.
-# Six pictures cost extra time on every step. A class of machine starts from six only when it was measured at 25%
-# extra or less on every step, cold and warm: (card memory tier in GiB, people in the shot, weight precision).
-# Everything else, and everything unmeasured, starts from three. Only Linux is measured. Two people went 45-79% over
-# at 8 to 24 GiB, so only one-person classes can ever be listed. No class has passed yet.
-SIX_PICTURES_CLASSES = frozenset()
-SIX_PICTURES_TIERS_GIB = (8, 10, 12, 16, 20, 24, 32) # measured card sizes; a card takes the largest one it reaches
-SIX_PICTURES_SYSTEMS = ('Linux',)
-TIER_SLACK_GIB = .5 # A "24 GB" card reports a little under 24 GiB.
-CONTINUED_ORDER = ('face', 'front', 'three_quarter') # a continuing shot's picture for each person
+# Each person is two pictures: one sheet of every view (made by the cast library when the card has none), then
+# the face close-up. A continuing shot with several people has no room for two each: it carries one picture a person.
+FACE_ORDER = ('face', 'front', 'three_quarter') # the one picture standing for a person, then any other label, in card order
MAX_VOICES = 3 # voice clips per shot, the model limit
MAX_REFERENCES = 32
@@ -299,46 +290,51 @@ def subject_definitions(subjects):
return 'subject_definitions: ' + ' '.join(parts)
-def _pictures(card, each=PICTURES_EACH):
- rank = {label: i for i, label in enumerate(PICTURE_ORDER)}
- rows = sorted(enumerate(card.get('images') or []), key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0]))
- return [row['media'] for _, row in rows[:each]]
+def _views(card):
+ """The card's pictures other than a sheet, best stand-in first: the face, the front, the three-quarter view, the rest."""
+ rank = {label: i for i, label in enumerate(FACE_ORDER)}
+ rows = [(i, row) for i, row in enumerate(card.get('images') or []) if row['label'] != 'sheet']
+ rows.sort(key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0]))
+ return [row['media'] for _, row in rows]
-def _continued_pictures(card, alone):
- """The pictures a continuing shot carries for one person.
+def sheet_of(card):
+ """The card's sheet of every view: one it carries itself, else the one the cast library made from its pictures."""
+ own = next((row['media'] for row in card.get('images') or [] if row['label'] == 'sheet'), None)
+ return own or card.get('_sheet')
- Alone in the shot: the sheet, then a close face (else the front view), because a sheet
- alone makes the face too small. With others: one picture, face > front > three_quarter."""
- by = {}
- for item in card.get('images') or []:
- by.setdefault(item['label'], item['media'])
- rest = [item['media'] for item in card.get('images') or [] if item['label'] not in CONTINUED_ORDER]
- ordered = [by[label] for label in CONTINUED_ORDER if label in by] + rest
- if not alone:
- return ordered[:1]
- sheet = by.get('sheet')
- close = by.get('face') or by.get('front') or next((m for m in ordered if m is not sheet), None)
- return [m for m in (sheet, close) if m is not None] or ordered[:1]
+def person_pictures(card, *, alone=True, continues=False):
+ """The pictures one person carries into a shot, in order: the sheet, then the face close-up (the front view
+ when there is no face). Two of them on every device. A card that has no sheet (one picture, or none could be
+ made) carries its face alone (else the front view). A shot that continues the previous one with several
+ people carries only the face (else the front view) of each, with no sheet."""
+ views = _views(card)
+ if continues and not alone:
+ return views[:1] or ([sheet_of(card)] if sheet_of(card) else [])
+ sheet = sheet_of(card)
+ if sheet is None:
+ return views[:1]
+ return [sheet] + [view for view in views[:1] if view.get('file') != sheet.get('file')]
-def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH):
+
+def cast_references(play, shot, cards, *, continues=False):
"""Which cast cards a shot carries, as references and a prompt prefix.
cards maps a card ID to the current card, or None once it is gone. The same
- function serves plan previews and the moment a take is submitted. each is
- the most pictures one person carries; pictures_each says what was used.
+ function serves plan previews and the moment a take is submitted.
+ pictures_each is the most pictures one person carries in this shot.
cast_reference: None (no character in the shot has a card), 'own' (the shot
names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted),
'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out
entirely; 'voice_omitted' lists people whose pictures are carried but whose
- voice did not fit. A shot that continues the previous one carries one picture
- per person (face, else front, else three-quarter; a lone person gets the sheet and
- then a face close-up) for at most CONTINUED_PICTURES people, and no voice clip: its voices carry on from the previous shot."""
+ voice did not fit. A shot that continues the previous one carries no voice
+ clip (its voices carry on from the previous shot) and at most CONTINUED_PICTURES
+ pictures: two for a single person, else one for each of up to that many people."""
characters = play.get('characters', {})
keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)]
result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[],
- prefix=None, subjects=[], cards={}, pictures_each=each, clip_seconds=[])
+ prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[])
if not keys:
return result
# A references list, even an empty one, is the shot's own choice: cards are added only when it is left out.
@@ -350,18 +346,18 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH):
spoken[line['who']] = spoken.get(line['who'], 0) + 1
# People with more lines come first when not everyone fits; ties keep the order of appearance.
chosen, pictures, voices = [], 0, 0
- alone = continues and sum(1 for k in keys if cards.get(characters[k]['card']) is not None
- and _pictures(cards[characters[k]['card']])) == 1
+ alone = sum(1 for k in keys if cards.get(characters[k]['card']) is not None
+ and (cards[characters[k]['card']].get('images') or [])) == 1
for key in sorted(keys, key=lambda k: -spoken.get(k, 0)):
identity = characters[key]['card']
card = cards.get(identity)
if card is None:
result['missing'].append(key)
continue
- images = _pictures(card, each) # a continuing shot ignores each: it carries at most 2, within the 3 it may hold
+ images = person_pictures(card, alone=alone, continues=continues)
if continues:
if images and len(chosen) < specs.CONTINUED_PICTURES:
- chosen.append((key, card, _continued_pictures(card, alone), False))
+ chosen.append((key, card, images, False))
else:
result['omitted'].append(key)
continue
@@ -391,6 +387,7 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH):
result['people'].append(dict(character=key, card=card['id'], revision=card['revision'],
pictures=len(images), voice=voice))
result['references'].extend(clips)
+ result['pictures_each'] = max((len(images) for _, _, images, _ in chosen), default=0) or None
if subjects:
result['prefix'] = subject_definitions(subjects)
result['subjects'] = subjects
@@ -399,27 +396,6 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH):
return result
-def pictures_start(machine, people=1):
- """Pictures a person carries before the length check: six for a measured class, else three.
- machine is a capability table's `machine` row; a missing or unreadable one is unmeasured."""
- try:
- if machine['system'] in SIX_PICTURES_SYSTEMS:
- vram = float(machine['vram_gib']) + TIER_SLACK_GIB
- tier = max((t for t in SIX_PICTURES_TIERS_GIB if t <= vram), default=None)
- if tier is not None and (tier, people, machine['precision']) in SIX_PICTURES_CLASSES:
- return PICTURES_EACH
- except (KeyError, TypeError, ValueError):
- pass
- return PICTURES_LEAST
-
-
-def pictures_note(each, cut=True):
- """One sentence when the length check carried fewer pictures than this device started from; otherwise None."""
- if each is None or each >= PICTURES_EACH or not cut:
- return None
- return 'Each person carries %d pictures in this shot; this device cannot hold more at this length.' % each
-
-
PROMPT_LIMIT = 20000
DESCRIPTION_CUTS = (500, 250, 120, 0)
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index ef9efb4..07b4cec 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -543,8 +543,8 @@ GUIDE['cast'] = [
'Changes need the revision you read (body revision or If-Match); stale_revision returns the current card in '
'error.card. Fields the user changed return changed_by_user: keep the user\'s version.',
'Film with a card by setting characters.<key>.card. Cards are read when each take is submitted. Shots that '
- 'continue the previous one carry one card picture per person (a person alone: the sheet and a face close-up) for '
- 'at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
+ 'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, '
+ 'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
'A shot\'s own references replace the card pictures; references=[] sends none. '
'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.',
'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). '
diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py
index d172622..37b8b20 100644
--- a/freevideo_engine/agent/production.py
+++ b/freevideo_engine/agent/production.py
@@ -722,8 +722,7 @@ def view(state):
row.update(cast_reference=latest.get('cast_reference'), cast_omitted=deepcopy(latest.get('cast_omitted', [])),
cast_voice_omitted=deepcopy(latest.get('cast_voice_omitted', [])),
cast_missing=deepcopy(latest.get('cast_missing', [])),
- cast_pictures_each=latest.get('cast_pictures_each'),
- cast_note=drama.pictures_note(latest.get('cast_pictures_each'), latest.get('cast_pictures_cut', True)))
+ cast_pictures_each=latest.get('cast_pictures_each'))
row['takes'] = []
for take in shot['takes']:
item = {key: deepcopy(take[key]) for key in ('take', 'job', 'generation', 'seed', 'state', 'stale',
diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py
index 64bcb6d..fcfb811 100644
--- a/freevideo_engine/agent/productions.py
+++ b/freevideo_engine/agent/productions.py
@@ -70,11 +70,6 @@ def pictures_each(resolved):
return resolved['pictures_each'] if resolved['people'] else None
-def pictures_cut(resolved):
- """True when the length check took pictures away from what this device starts with."""
- return bool(resolved['people'] and resolved.get('pictures_cut'))
-
-
class MaterialRejected(JobError):
"""External callers named material the content-only boundary does not admit: the job validation error."""
status = 422
@@ -171,7 +166,6 @@ def failure(view):
class Productions:
def __init__(self, jobs):
self.jobs = jobs
- self._machine = None # the capability table's machine row, once known
self.directory = jobs.directory / 'productions'
self.records = {}
self.changed = threading.Condition(jobs.lock)
@@ -375,7 +369,6 @@ class Productions:
row.update(cast_reference=resolved['cast_reference'], cast=resolved['people'],
cast_omitted=resolved['omitted'], cast_voice_omitted=resolved['voice_omitted'],
cast_missing=resolved['missing'], cast_pictures_each=pictures_each(resolved),
- cast_note=drama.pictures_note(pictures_each(resolved), pictures_cut(resolved)),
references=len(resolved['references']))
try:
spec._text(row['prompt'], '/screenplay/shots/%d/prompt' % i, 20000)
@@ -921,8 +914,7 @@ class Productions:
cast_omitted=list(resolved['omitted']),
cast_voice_omitted=list(resolved['voice_omitted']),
cast_missing=list(resolved['missing']),
- cast_pictures_each=pictures_each(resolved),
- cast_pictures_cut=pictures_cut(resolved))
+ cast_pictures_each=pictures_each(resolved))
self._save(marker) # Durable before Jobs may accept it.
self.jobs.submit(item, client='production:' + identity, parent=dict(production=identity, ref=ref))
except spec.SpecError as error:
@@ -957,80 +949,9 @@ class Productions:
return resolved
def _carry(self, play, shot, cards, item, *, continues):
- """The cast a shot carries: this device's starting number of pictures a person (six for a measured
- class of card, people and precision, else three), fewer where it cannot hold them, down to three. Plan
- previews and submissions both come here, so they agree."""
- if continues:
- # A continuing shot carries one picture a person whatever the device holds.
- resolved = drama.cast_references(play, shot, cards, continues=True, each=drama.PICTURES_LEAST)
- resolved.update(pictures_each=None, pictures_cut=False)
- return resolved
- people = len(drama.cast_references(play, shot, cards, continues=continues, each=drama.PICTURES_LEAST)['people'])
- start = each = self._pictures_start(item, people)
- while True:
- resolved = drama.cast_references(play, shot, cards, continues=continues, each=each)
- if each <= drama.PICTURES_LEAST or item is None or not self._too_long(item, resolved):
- resolved['pictures_cut'] = each < start
- return resolved
- each -= 1
-
- def _pictures_start(self, item, people):
- """Where the picture count starts for this many people on this machine, from the capability table's
- graphics card, system and precision. Any canvas names the same machine, so a shot without a take asks
- about the default canvas."""
- if self._machine is None:
- from .length import capability_table
- width, height = spec.canvas_size(item or {})
- table = capability_table(self.jobs.port, dict(width=width, height=height, task='ref2va', loras=False), wait=3.)
- if table.get('status') != 'ready':
- return drama.pictures_start(None) # Not worked out yet: three, and the next ask looks again.
- precision = getattr(self.jobs.port, 'prepared_precision', lambda: None)()
- # The machine does not change while this server runs.
- self._machine = dict(table.get('machine') or {}, precision=precision if isinstance(precision, str) else None)
- return drama.pictures_start(self._machine, people)
-
- def _too_long(self, item, resolved):
- """True when this device cannot generate the shot with these references. A table that is not ready
- (still computing, unavailable) or lacks this canvas says False and keeps the pictures; a ready table
- that admits no length at this canvas, or a limit this shot's length cannot be planned under, says True."""
- pictures = sum(p['pictures'] for p in resolved['people'])
- if not pictures:
- return False
- try:
- from ..chain_plan import PlanError, plan as chain_plan
- from .length import capability_table
- media = dict(references=deepcopy(resolved['references']))
- normalized = spec.normalize(dict(deepcopy(item), media=media))
- rows = ([dict(kind='image', width=normalized['width'], height=normalized['height'])] * pictures
- + [dict(kind='audio', seconds=seconds) for seconds in resolved['clip_seconds']])
- table = capability_table(self.jobs.port, normalized, wait=3., references=rows,
- task='ref2va_av' if resolved['clip_seconds'] else 'ref2va')
- if table.get('status') != 'ready':
- return False
- index = next((i for i, row in enumerate(table.get('sizes', []))
- if (row['width'], row['height']) == (normalized['width'], normalized['height'])), None)
- tiers = table.get('tiers', {})
- tier = tiers.get(normalized['quality'])
- if tier is None:
- # Same plan and step cache first, then the same plan (see length.single_frame_limit).
- same = [row for row in tiers.values()
- if row.get('steps') == normalized['base_steps'] and row.get('two_pass') == normalized['two_pass']
- and (not normalized['two_pass'] or normalized['refine_steps'] == 3)]
- tier = next((row for row in same if row.get('step_cache') == normalized.get('step_cache')),
- same[0] if same else None)
- maxima = tier.get('max_frames') if tier else None
- if index is None or not isinstance(maxima, list) or index >= len(maxima):
- return False
- limit = maxima[index]
- if limit is None:
- return True # no length at all fits at this canvas with these pictures
- try:
- chain_plan(normalized['seconds'], max_frames=limit, continued=bool(normalized.get('continue_from')))
- except PlanError:
- return True
- return False
- except Exception: # The limit is advisory: a failure here keeps the pictures and lets the job decide.
- return False
+ """The cast a shot carries: each person's sheet and face close-up on every device (one picture a person when
+ a continuing shot has several people). Plan previews and submissions both come here, so they agree."""
+ return drama.cast_references(play, shot, cards, continues=continues)
def _current(self, play):
"""Cards as they are now; a card deleted since the film started is None (the shot uses text)."""
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 12ab2bc..e19542c 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -910,8 +910,7 @@ def production_response():
cast_reference=CAST_REFERENCE, cast_omitted=dict(type='array', items=string()),
cast_voice_omitted=dict(type='array', items=string()),
cast_missing=dict(type='array', items=string()),
- cast_pictures_each=dict(type=['integer', 'null'], description='Pictures each person carries in this shot: six or three by graphics card class, fewer (down to three) when this device cannot hold the starting number at the shot length.'),
- cast_note=dict(type=['string', 'null'], description='One sentence saying so, only when it was reduced from the starting number.'),
+ cast_pictures_each=dict(type=['integer', 'null'], description='Most pictures one person carries in this shot: two (the sheet of every view and the face close-up), or one each when a continuing shot has several people.'),
takes=dict(type='array', items=dict(type='object', properties=dict(take=number(1, integer=True), job=REF,
task={'oneOf': [JOB, {'type': 'null'}]}, generation=number(0, integer=True), seed=number(0, integer=True),
stale={'type': 'boolean'}, error={'type': ['object', 'null']},
diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py
index f1d0e61..ff304d4 100644
--- a/freevideo_engine/comfy_assets.py
+++ b/freevideo_engine/comfy_assets.py
@@ -75,7 +75,11 @@ def resolve_assets(serialized, input_directory):
root = Path(input_directory).resolve()
result = {}
for row in rows:
- if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled'}:
+ if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast'}:
+ raise ValueError('Invalid media item')
+ # The character an @ mention added the row for; a continued part keeps one picture per character.
+ if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) != {'id'}
+ or not isinstance(row['cast']['id'], str) or not 0 < len(row['cast']['id']) <= 128):
raise ValueError('Invalid media item')
if type(row.get('enabled', True)) is not bool:
raise ValueError('Media enabled must be a boolean')
@@ -101,7 +105,10 @@ def resolve_assets(serialized, input_directory):
raise ValueError('Select only one ' + role + ' frame')
result[role] = str(path)
elif role == 'reference':
- result.setdefault('references', []).append({'kind': kind, 'path': str(path)})
+ reference = {'kind': kind, 'path': str(path)}
+ if cast_id(row) is not None:
+ reference['cast'] = cast_id(row) # only a continued part reads it (comfy_media.continued)
+ result.setdefault('references', []).append(reference)
else:
raise ValueError('Media role must be first, last or reference')
# Mixing keyframes and references is checked when the video is made (comfy_media.export):
@@ -110,15 +117,16 @@ def resolve_assets(serialized, input_directory):
def continued_assets(serialized):
- """The panel rows a continued part uses: enabled image references in panel order, at most three.
+ """The panel rows a continued part uses: enabled image references in panel order, at most three
+ (continuation.pick_pictures: with two or more characters, one picture each).
Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks."""
- from .continuation import PICTURES_MAX
+ from .continuation import pick_pictures
try:
rows = json.loads(serialized)
except (TypeError, ValueError):
return '[]'
- kept = []
+ pictures = []
for row in rows if isinstance(rows, list) else ():
if (isinstance(row, dict) and row.get('enabled', True) is True and row.get('role') == 'reference'
and isinstance(row.get('file'), str)):
@@ -126,9 +134,23 @@ def continued_assets(serialized):
image = media_kind(row['file']) == 'image'
except ValueError:
image = False
- if image and len(kept) < PICTURES_MAX:
- kept.append(dict(file=row['file'], role='reference'))
- return json.dumps(kept)
+ if image:
+ pictures.append(row)
+ return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)])
+
+
+def cast_id(row):
+ """The character a panel row belongs to (the media list marks rows added by an @ mention), else None."""
+ cast = row.get('cast') if isinstance(row, dict) else None
+ value = cast.get('id') if isinstance(cast, dict) else None
+ return value if isinstance(value, (str, int)) and not isinstance(value, bool) else None
+
+
+def _kept_row(row):
+ kept = dict(file=row['file'], role='reference')
+ if cast_id(row) is not None:
+ kept['cast'] = dict(id=cast_id(row))
+ return kept
def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, **images):
diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.py
index dbf0646..2f8b889 100644
--- a/freevideo_engine/comfy_media.py
+++ b/freevideo_engine/comfy_media.py
@@ -50,13 +50,15 @@ def continued(exported, notes=None):
It starts from the footage, so a first or last frame image becomes one more picture after the
reference pictures; pre-encoded conditioning, clips and sound are left out. When keyframes are
used or pictures are left out, notes gets one line saying so."""
- from .continuation import PICTURES_MAX
+ from .continuation import PICTURES_MAX, pick_pictures
media = (exported or {}).get('media') or {}
kept = {key: value for key, value in media.items()
if key not in ('first', 'last', 'references', 'conditioning_info')}
pictures = [row for row in media.get('references', []) if row.get('kind') == 'image']
keyframes = [name for name in ('first', 'last') if media.get(name)]
- candidates = pictures + [{'kind': 'image', 'path': media[name]} for name in keyframes]
+ # Characters added by an @ mention keep one picture each when there are several (pick_pictures).
+ chosen = pick_pictures(pictures, lambda row: row.get('cast'))
+ candidates = [_plain(row) for row in chosen] + [{'kind': 'image', 'path': media[name]} for name in keyframes]
if candidates:
kept['references'] = candidates[:PICTURES_MAX]
lines = [continued_note(len(pictures), keyframes, len(candidates[:PICTURES_MAX]))]
@@ -70,6 +72,11 @@ def continued(exported, notes=None):
return {'media': kept} if len(kept) > 1 else {}
+def _plain(row):
+ """A reference row without the panel's character mark, which the engine's request does not carry."""
+ return {key: value for key, value in row.items() if key != 'cast'}
+
+
def continued_note(pictures, keyframes, used):
"""One line for a continued part whose keyframes became pictures or whose pictures did not all fit."""
frames = 'the %s frame%s' % (' and '.join(keyframes), 's' if len(keyframes) > 1 else '')
@@ -146,6 +153,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c
torch.save(value, path)
media['conditioning_info'] = info
result['conditioning'] = str(path)
+ if not continuing and media.get('references'):
+ media['references'] = [_plain(row) for row in media['references']]
if len(media) > 1:
result['media'] = media
return result
diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py
index 6a50d9f..f432520 100644
--- a/freevideo_engine/continuation.py
+++ b/freevideo_engine/continuation.py
@@ -40,6 +40,24 @@ UNSUPPORTED_REFINE = UNSUPPORTED + ' Clear either "Second pass of preview" or "C
UNSUPPORTED_MEDIA = UNSUPPORTED + ' Connect up to 3 pictures as references, and remove video, audio and pre-encoded inputs.'
# A continued part keeps the cast's pictures, one per person and at most three people in a frame.
PICTURES_MAX = 3
+
+
+def pick_pictures(rows, person):
+ """The pictures a continued part keeps from image rows in panel order: at most PICTURES_MAX.
+
+ person(row) is the id of the character a row belongs to, or None. With two or more characters each keeps
+ only the last of its rows (the face close-up, or the one picture its card has) so that everyone is seen; the
+ other pictures follow. With none or one character the first rows are kept."""
+ rows = list(rows)
+ last = {}
+ for index, row in enumerate(rows):
+ if person(row) is not None:
+ last[person(row)] = index
+ if len(last) < 2:
+ return rows[:PICTURES_MAX]
+ faces = [rows[index] for index in sorted(last.values())]
+ others = [row for row in rows if person(row) is None]
+ return (faces + others)[:PICTURES_MAX]
INVALID_FRAMES = 'Frame count must be 5 + 17n and at least 39 (39, 56, 73, 90, ...).'
INVALID_FILE = 'Choose an existing MP4 video using its absolute path.'
diff --git a/web/cast_mention.js b/web/cast_mention.js
index b332747..c243c25 100644
--- a/web/cast_mention.js
+++ b/web/cast_mention.js
@@ -3,10 +3,10 @@
// of the media list that carry `cast: {id, name, description}`. When a request is made the names become <Subject N>
// and the subject definitions go in front, the same wording the Agent's drama planner writes (agent/drama.py).
export const MAX_PEOPLE = 3;
-export const MAX_PICTURES_EACH = 6;
+export const MAX_PICTURES_EACH = 2;
export const MAX_REFERENCES = 32;
-// Which of a card's pictures come first; the others follow in the card's own order.
-export const PICTURE_ORDER = ['sheet', 'front', 'three_quarter', 'face'];
+// Which of a card's pictures stands for the person after the sheet; the others follow in the card's own order.
+export const PICTURE_ORDER = ['face', 'front', 'three_quarter'];
// subjects: [{name, description, pictures: [first, last] | null, audio: number | null}]. This is the one JS place for
// the wording; its Python twin is subject_definitions() and tests/fixtures/subject_definitions.json holds the cases
@@ -34,15 +34,22 @@ export function subjectDefinitions(subjects) {
const IMAGE = /\.(png|jpe?g|webp|bmp|tiff?)$/i;
const enabled = row => row && row.enabled !== false;
// The number H3 gives each reference picture: images in list order (connected inputs come after the list).
-// A part that continues a video keeps the first CONTINUE_PICTURES of them (continuation.PICTURES_MAX).
+// A part that continues a video keeps CONTINUE_PICTURES of them (continuation.PICTURES_MAX): the first ones, or, with two or
+// more characters, the last picture of each (the face) and then the pictures of no character (continuation.pick_pictures).
export const CONTINUE_PICTURES = 3;
-function pictureNumbers(rows, limit = Infinity) {
- const numbers = new Map();
- let count = 0;
+function pictureNumbers(rows, continued = false) {
+ const picked = [];
rows.forEach((row, index) => {
- if (enabled(row) && row.role === 'reference' && IMAGE.test(row.file || '') && count < limit) numbers.set(index, ++count);
+ if (enabled(row) && row.role === 'reference' && IMAGE.test(row.file || '')) picked.push(index);
});
- return numbers;
+ let kept = picked;
+ if (continued) {
+ const last = new Map();
+ picked.forEach(index => { if (rows[index].cast?.id) last.set(rows[index].cast.id, index); });
+ kept = last.size < 2 ? picked : [...last.values()].sort((a, b) => a - b).concat(picked.filter(index => !rows[index].cast?.id));
+ kept = kept.slice(0, CONTINUE_PICTURES);
+ }
+ return new Map(kept.map((index, at) => [index, at + 1]));
}
// The characters present in the media list, in list order, with the rows that carry each one's pictures.
@@ -86,8 +93,8 @@ export function mentionRanges(text, people) {
// The prompt a request carries. Characters mentioned in the text are numbered in order of first mention; one whose
// pictures are gone is written as a plain name. `active: false` (first/last frames) leaves every name plain too, since
// pictures of people cannot go with those. `continued` (the part continues a video) lets the characters stay: the part
-// keeps the first three reference pictures and its frames come after them, so a character whose pictures are not among
-// those three is written as a plain name.
+// keeps three reference pictures (see pictureNumbers) and its frames come after them, so a character whose pictures are
+// not among those three is written as a plain name.
export function expandCast(text, rows, {active = true, continued = false} = {}) {
text = String(text ?? '');
const list = Array.isArray(rows) ? rows : [];
@@ -95,7 +102,7 @@ export function expandCast(text, rows, {active = true, continued = false} = {})
const ranges = mentionRanges(text, people);
if (!ranges.length) return {text, subjects: []};
if (!continued && list.some(row => enabled(row) && (row.role === 'first' || row.role === 'last'))) active = false;
- const numbers = pictureNumbers(list, continued ? CONTINUE_PICTURES : Infinity);
+ const numbers = pictureNumbers(list, continued);
const order = [];
let result = '', at = 0;
for (const range of ranges) {
@@ -123,13 +130,20 @@ export function expandCast(text, rows, {active = true, continued = false} = {})
return {text: subjectDefinitions(subjects) + '\n\n' + result, subjects};
}
-// The pictures of a card to put in the media list: the card's own order after the preferred labels, the first six.
+// The pictures of a card to put in the media list, as the Agent carries them (agent/drama.py person_pictures): the sheet of
+// every view (the card's own, else the one the library made), then the face close-up, else the front view. A card with no
+// sheet (one picture, or none could be made) carries its face alone.
export function cardPictures(card) {
const rank = new Map(PICTURE_ORDER.map((label, index) => [label, index]));
- return (card?.images || []).map((image, index) => ({image, index}))
- .filter(({image}) => image?.media?.file && IMAGE.test(image.media.file))
+ const usable = (card?.images || []).map((image, index) => ({image, index}))
+ .filter(({image}) => image?.media?.file && IMAGE.test(image.media.file));
+ const views = usable.filter(({image}) => image.label !== 'sheet')
.sort((a, b) => (rank.get(a.image.label) ?? PICTURE_ORDER.length) - (rank.get(b.image.label) ?? PICTURE_ORDER.length) || a.index - b.index)
- .slice(0, MAX_PICTURES_EACH).map(({image}) => image.media.file);
+ .map(({image}) => image.media.file);
+ const sheet = usable.find(({image}) => image.label === 'sheet')?.image.media.file
+ || (IMAGE.test(card?.sheet?.file || '') ? card.sheet.file : null);
+ if (!sheet) return views.slice(0, 1);
+ return [sheet, ...views.slice(0, 1).filter(file => file !== sheet)];
}
// Whether a character can be mentioned now; `reason` is the one thing in the way.
diff --git a/web/editor_references.js b/web/editor_references.js
index 4bd6f80..9476cbf 100644
--- a/web/editor_references.js
+++ b/web/editor_references.js
@@ -4,8 +4,17 @@
const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
const IMAGE = /\.(png|jpe?g|webp|bmp|tiff?)$/i;
const MAX = 32;
-// The engine keeps the first three references of a part that continues.
+// The engine keeps three references of a part that continues: the first three, or, with two or more characters,
+// the last picture of each (the face) and then the others (continuation.pick_pictures).
const CONTINUE_MAX = 3;
+const personOf = ({row}) => typeof row.cast?.id === 'string' && row.cast.id ? row.cast.id : null;
+function pickContinued(list) {
+ const last = new Map();
+ list.forEach((entry, at) => { const person = personOf(entry); if (person) last.set(person, at); });
+ if (last.size < 2) return list.slice(0, CONTINUE_MAX);
+ const faces = [...last.values()].sort((a, b) => a - b).map(at => list[at]);
+ return [...faces, ...list.filter(entry => !personOf(entry))].slice(0, CONTINUE_MAX);
+}
export function createEditorReferences({t, source, size = () => null, error = () => {}, notify = () => {}}) {
const root = el('div', 'fv-ed-refs'), chips = el('div', 'fv-ed-ref-chips');
const add = el('button', 'fv-ed-ref-add'); add.type = 'button';
@@ -20,7 +29,7 @@ export function createEditorReferences({t, source, size = () => null, error = ()
const isImage = file => IMAGE.test(file || '');
// The pictures this part uses: all of them, or the first reference images of a part that continues
// (sound or video references do not count there, as the engine skips them).
- const used = () => continuing ? active().filter(({row}) => (row.role || 'reference') === 'reference' && isImage(row.file)).slice(0, CONTINUE_MAX) : active();
+ const used = () => continuing ? pickContinued(active().filter(({row}) => (row.role || 'reference') === 'reference' && isImage(row.file))) : active();
const full = () => continuing ? used().length >= CONTINUE_MAX : active().length >= MAX;
// On hover without a full stop; in the notice line with one.
const waitText = () => t('Available after generation finishes', '生成完成后可改');
diff --git a/web/freevideo.js b/web/freevideo.js
index f027031..7365589 100644
--- a/web/freevideo.js
+++ b/web/freevideo.js
@@ -351,8 +351,8 @@ function mediaPanel(node, mount = null) {
let uploading = false;
serialized.serializeValue = () => {
if (node.isUploading) throw new Error(text("Wait for media uploads to finish before running", "请等待素材上传完成后再生成"));
- // The run takes only what it uses; the editing data of a picture stays in the workflow.
- try { return JSON.stringify(read().map(({file, role, enabled}) => ({file, role, enabled}))); } catch { return serialized.value; }
+ // The run takes only what it uses (and which character a picture belongs to); the editing data of a picture stays in the workflow.
+ try { return JSON.stringify(read().map(({file, role, enabled, cast}) => ({file, role, enabled, ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id}} : {})}))); } catch { return serialized.value; }
};
async function upload(files) {
if (node.isUploading || !files.length) return;