Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Drama: numbered cuts in one shot, six reference sections for picture shots, length from the...

Bowen Xue08e6456d80a2
diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py
index ac0b32b..e174368 100644
--- a/freevideo_engine/agent/drama.py
+++ b/freevideo_engine/agent/drama.py
@@ -20,7 +20,7 @@ import re
import time
from . import spec as specs
-from ..chain_plan import CONTINUE_SECONDS
+from ..chain_plan import CONTINUE_SECONDS, BLOCK_SECONDS
EXAMPLE = {
'title': '末班车', 'name': 'the-last-train', 'aspect': '9:16', 'quality': 'light',
@@ -67,8 +67,10 @@ MAX_VOICES = 3 # voice clips per shot, the model limit
MAX_REFERENCES = 32
-def check(play):
- """Validate every independent field, mutating only missing shot ids."""
+def check(play, single_seconds=BLOCK_SECONDS):
+ """Validate every independent field, mutating only missing shot ids.
+
+ single_seconds: the longest shot this machine generates at once, which bounds a shot that carries pictures."""
problems = []
def problem(path, message):
@@ -109,6 +111,12 @@ def check(play):
validate(specs._text, play[field], '/' + field, 20000, empty=True)
if 'name' in play and (not isinstance(play['name'], str) or not specs.SLUG.fullmatch(play['name'])):
problem('/name', 'must be a short project name')
+ if 'continuity' in play:
+ if not isinstance(play['continuity'], dict):
+ problem('/continuity', 'must be an object of text by character name, and optionally "set"')
+ else:
+ for key, text in play['continuity'].items():
+ validate(specs._text, text, '/continuity/' + str(key).replace('~', '~0').replace('/', '~1'), 20000, empty=True)
if 'captions' in play:
validate(specs._bool, play['captions'], '/captions')
if 'quality' in play and (not isinstance(play['quality'], str) or play['quality'] not in specs.QUALITY):
@@ -136,8 +144,8 @@ def check(play):
problem(path + '/id', '/shots/%d/id must be a unique short name' % i)
if isinstance(shot['id'], str):
names.add(shot['id'])
- if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')):
- problem(path, '/shots/%d needs action, idea or prompt' % i)
+ if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')) and not shot.get('cuts'):
+ problem(path, '/shots/%d needs action, idea, prompt or cuts' % i)
if shot.get('continue') and i == 0:
problem(path + '/continue', '/shots/0 cannot continue a previous shot')
if 'continue' in shot:
@@ -159,12 +167,13 @@ def check(play):
if ((shot.get('continue') or long) and play.get('quality') in specs.QUALITY
and not specs.QUALITY[play['quality']][0] and all(p['path'] != '/quality' for p in problems)):
problem('/quality', '"light" or "medium" (two-pass) when a shot continues another or is longer than 15 s')
- if long and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')):
- problem(path + '/seconds', 'at most 15 with a first frame or references')
- elif long and not shot.get('continue') and _carded(play, shot):
- # Cast cards travel as references, which one generation of at most 15 s carries.
- problem(path + '/seconds', 'at most 15 when a character in the shot has a cast card '
- '(a following shot with continue=true carries the same people further)')
+ over = (isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and seconds > single_seconds)
+ if over and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')):
+ problem(path + '/seconds', 'at most %d with a first frame or references on this machine' % single_seconds)
+ elif over and not shot.get('continue') and _carded(play, shot):
+ # Cast cards travel as references, which one generation (at most single_seconds) carries.
+ problem(path + '/seconds', 'at most %d on this machine when a character in the shot has a cast card '
+ '(a following shot with continue=true carries the same people further)' % single_seconds)
if shot.get('first') is not None:
validate(specs.media_item, shot['first'], path + '/first', image=True)
if 'references' in shot:
@@ -185,6 +194,8 @@ def check(play):
for j, key in enumerate(shot['cast']):
if not isinstance(key, str) or key not in characters:
problem(path + '/cast/%d' % j, 'must name a character')
+ if 'cuts' in shot:
+ _check_cuts(shot, path, characters, seconds, problem, validate)
lines = shot.get('lines', [])
if not isinstance(lines, list):
problem(path + '/lines', 'must be a list')
@@ -205,12 +216,62 @@ def check(play):
return play
+def _check_cuts(shot, path, characters, seconds, problem, validate):
+ """Numbered cuts inside one generation: each starts at "at" seconds (the first at 0), strictly later than the one before."""
+ cuts = shot['cuts']
+ if not isinstance(cuts, list) or not 1 <= len(cuts) <= 8:
+ problem(path + '/cuts', 'must be a list of 1 to 8 cuts')
+ return
+ last = None
+ for j, cut in enumerate(cuts):
+ here = '%s/cuts/%d' % (path, j)
+ if not isinstance(cut, dict):
+ problem(here, 'must be an object')
+ continue
+ for key in cut:
+ if key not in CUT_FIELDS:
+ problem(here + '/' + key, 'is not a cut field (%s)' % ', '.join(CUT_FIELDS))
+ at = cut.get('at', 0 if j == 0 else None)
+ if isinstance(at, bool) or not isinstance(at, (int, float)) or at < 0 or (j == 0 and at != 0):
+ problem(here + '/at', 'must be the second the cut starts (0 for the first cut)')
+ elif last is not None and at <= last:
+ problem(here + '/at', 'must be later than the cut before')
+ elif isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and at >= seconds:
+ problem(here + '/at', 'must be before the end of the shot (%s s)' % seconds)
+ else:
+ last = at
+ for field in ('framing', 'action', 'camera'):
+ if field in cut:
+ validate(specs._text, cut[field], here + '/' + field, 20000, empty=True)
+ if not any(isinstance(cut.get(k), str) and cut[k].strip() for k in ('action', 'framing')):
+ problem(here, 'needs an action or framing')
+ for k, line in enumerate(cut.get('lines', []) if isinstance(cut.get('lines', []), list) else [None]):
+ if not isinstance(line, dict) or line.get('who') not in characters:
+ problem('%s/lines/%d' % (here, k), 'must be a line from a named character')
+ else:
+ validate(specs._text, line.get('line'), '%s/lines/%d/line' % (here, k), 20000)
+
+
+CUT_FIELDS = ('at', 'framing', 'action', 'camera', 'lines')
+
+
+def _stamp(seconds):
+ whole = int(seconds)
+ return '%02d:%02d.%03d' % (whole // 60, whole % 60, round((seconds - whole) * 1000))
+
+
+def _mentions(text, word):
+ """Whole-word, case-insensitive match; a blank word never matches."""
+ word = (word or '').strip()
+ return bool(word) and re.search(r'(?<!\w)' + re.escape(word) + r'(?!\w)', text, re.I) is not None
+
+
def cast(play, shot):
"""Characters in this shot, speakers first, in order of appearance."""
order = [line['who'] for line in shot.get('lines', [])]
- order += [key for key in play.get('characters', {}) if key in (shot.get('cast') or [])
- or play['characters'][key].get('name', key).lower() in shot.get('action', '').lower()
- or key.lower() in shot.get('action', '').lower()]
+ action = shot.get('action', '')
+ order += [key for key, person in play.get('characters', {}).items() if key in (shot.get('cast') or [])
+ or _mentions(action, person.get('name', key)) or _mentions(action, key)]
return list(dict.fromkeys(order))
@@ -225,6 +286,14 @@ def _carded(play, shot):
SHOWN_TEXT = re.compile(r'["“「][^"”」]+["”」]')
+def _cap(name):
+ """A name as written, with its first letter upper-case: 'Old Ren' stays 'Old Ren'."""
+ return name[:1].upper() + name[1:]
+
+
+PAPER = re.compile(r'\b(note|letter|paper|card|sign|screen|book|poster|message|menu|map)\b', re.I)
+
+
def prompt(play, shot):
"""The H3 prompt of one shot, from its structured fields."""
if shot.get('prompt'):
@@ -232,34 +301,84 @@ def prompt(play, shot):
characters = play.get('characters', {})
people = cast(play, shot)
lead = ', '.join(x for x in (play.get('style'), shot.get('framing')) if x)
- parts = ['[Shot 1] ' + (lead + '.' if lead else '')]
- if shot.get('continue'):
- parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
- for key in people:
- person = characters[key]
- if person.get('look'):
- parts.append('%s is %s.' % ((person.get('name') or key).capitalize(), person['look'].rstrip('.')))
- parts.append(named(play, people, shot.get('action', '')))
- if shot.get('camera'):
- parts.append(shot['camera'])
- speakers = {}
- for line in shot.get('lines', []):
+ # A speaker's number is its place in the order of vocal events of this video (the official rule): the first
+ # voice heard is (S1), the next new one (S2); the number is reused at every later line of the same shot.
+ heard = [line['who'] for cut in (shot.get('cuts') or [shot]) for line in cut.get('lines', [])]
+ numbers = {key: number for number, key in enumerate(dict.fromkeys(heard), 1)}
+ spoken = set()
+
+ def say(line):
who = line['who']
person = characters[who]
- first = who not in speakers
- speakers.setdefault(who, len(speakers) + 1)
+ first = who not in spoken
+ spoken.add(who)
voice = (' with a %s voice' % person['voice']) if first and person.get('voice') else ''
language = line.get('language') or person.get('language') or play.get('language') or guess(line['line'])
- parts.append('%s%s (S%d) %s: <d>[%s] %s</d>' % ((person.get('name') or who).capitalize(), voice,
- speakers[who], line.get('how') or 'says', language, line['line']))
+ return '%s%s (S%d) %s: <d>[%s] %s</d>' % (_cap(person.get('name') or who), voice,
+ numbers[who], line.get('how') or 'says', language, line['line'])
+
+ def looks():
+ return ['%s is %s.' % (_cap(characters[key].get('name') or key), characters[key]['look'].rstrip('.'))
+ for key in people if characters[key].get('look')]
+
+ def continuity():
+ """The film's table of clothes, props and places, the rows of the people in this shot (and 'set'), in every shot."""
+ table = play.get('continuity')
+ if not isinstance(table, dict):
+ return []
+ rows = ['%s: %s.' % (_cap(characters[key].get('name') or key), table[key].strip().rstrip('.'))
+ for key in people if isinstance(table.get(key), str) and table[key].strip()]
+ if isinstance(table.get('set'), str) and table['set'].strip():
+ rows.append(table['set'].strip().rstrip('.') + '.')
+ return ['Continuity - ' + ' '.join(rows)] if rows else []
+
+ cuts = shot.get('cuts')
+ if cuts:
+ # Numbered cuts inside one generation: [Shot 1] has no time, the others say when they start.
+ seconds = shot.get('seconds')
+ parts = []
+ for number, cut in enumerate(cuts, 1):
+ row = [('[Shot 1] ' if number == 1 else '[Shot %d] At %s, ' % (number, _stamp(cut['at']))).rstrip()]
+ if number == 1:
+ row += [lead + '.' if lead else '']
+ if shot.get('continue'):
+ row.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
+ row += looks() + continuity()
+ elif cut.get('framing'):
+ row.append(cut['framing'].strip().rstrip('.') + '.')
+ if number == 1 and cut.get('framing'):
+ row.append(cut['framing'].strip().rstrip('.') + '.')
+ if cut.get('action'):
+ row.append(named(play, people, cut['action']))
+ if cut.get('camera'):
+ row.append(cut['camera'])
+ row += [say(line) for line in cut.get('lines', [])]
+ if number == len(cuts) and isinstance(seconds, (int, float)) and not isinstance(seconds, bool):
+ row.append('The shot ends at %s.' % _stamp(seconds))
+ parts.append(' '.join(x for x in row if x))
+ body = '\n'.join(parts)
+ else:
+ parts = ['[Shot 1] ' + (lead + '.' if lead else '')]
+ if shot.get('continue'):
+ parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
+ parts += looks() + continuity()
+ parts.append(named(play, people, shot.get('action', '')))
+ if shot.get('camera'):
+ parts.append(shot['camera'])
+ parts += [say(line) for line in shot.get('lines', [])]
+ body = ' '.join(p for p in parts if p)
if not play.get('captions'):
# Quoted words are text the shot shows (a sign, a menu); then only captions are ruled out.
scene = ' '.join(shot.get(k) or '' for k in ('framing', 'action', 'camera')) + ' ' + (play.get('style') or '')
- parts.append('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.')
+ scene += ' ' + ' '.join(c.get(k) or '' for c in (cuts or []) for k in ('framing', 'action', 'camera'))
+ body += ' ' + ('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.')
+ if PAPER.search(scene) and not SHOWN_TEXT.search(scene):
+ # A model writes the spoken line on any note it is asked to show close; a blank or unreadable face avoids it.
+ body += ' Any note, paper or sign in frame has a blank face with only faint folds, seen from the side or out of focus.'
sound = shot.get('sound') or 'Natural ambience of the scene.'
# Only a shot's own music is written; a score for the whole film is imported and laid on the sound track.
music = shot.get('music') or 'N/A'
- return ('integrated_multimodal_description: ' + ' '.join(p for p in parts if p) +
+ return ('integrated_multimodal_description: ' + body +
'\n\noverall_soundscape: ' + sound + '\n\nnon_diegetic_music: ' + music)
@@ -325,7 +444,7 @@ def cast_references(play, shot, cards, *, continues=False):
function serves plan previews and the moment a take is submitted.
pictures_each is the most pictures one person carries in this shot.
cast_reference: None (no character in the shot has a card), 'own' (the shot
- names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted),
+ names its own first frame, which carries only the written descriptions, or an empty list of references), 'missing' (a card was deleted),
'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out
entirely; 'voice_omitted' lists people whose pictures are carried but whose
voice did not fit. A shot that continues the previous one carries no voice
@@ -334,13 +453,31 @@ def cast_references(play, shot, cards, *, continues=False):
characters = play.get('characters', {})
keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)]
result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[],
- prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[])
+ prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[], own_pictures=0)
if not keys:
return result
- # A references list, even an empty one, is the shot's own choice: cards are added only when it is left out.
- if shot.get('first') is not None or isinstance(shot.get('references'), list):
+ own = shot.get('references') if isinstance(shot.get('references'), list) else None
+ if own is not None and not own:
+ # An empty list is the shot's own choice of no pictures: no cards are added.
result['cast_reference'] = 'own'
return result
+ if shot.get('first') is not None and own is None:
+ # A first frame cannot travel with reference pictures; the people still get their written descriptions.
+ subjects = []
+ for key in keys:
+ card = cards.get(characters[key]['card'])
+ if card is None:
+ result['missing'].append(key)
+ continue
+ subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'),
+ pictures=None, audio=None))
+ subjects = [s for s in subjects if (s['description'] or '').strip()]
+ if subjects:
+ result['prefix'] = subject_definitions(subjects)
+ result['subjects'] = subjects
+ result['cast_reference'] = 'missing' if result['missing'] else 'own'
+ return result
+ offset = len(own or []) # the shot's own pictures come first; the cards follow them
spoken = {}
for line in shot.get('lines', []):
spoken[line['who']] = spoken.get(line['who'], 0) + 1
@@ -356,13 +493,13 @@ def cast_references(play, shot, cards, *, continues=False):
continue
images = person_pictures(card, alone=alone, continues=continues)
if continues:
- if images and len(chosen) < specs.CONTINUED_PICTURES:
+ if images and offset + len(chosen) < specs.CONTINUED_PICTURES:
chosen.append((key, card, images, False))
else:
result['omitted'].append(key)
continue
voice = card.get('voice') is not None and voices < MAX_VOICES
- if pictures + len(images) + voices + voice > MAX_REFERENCES:
+ if offset + pictures + len(images) + voices + voice > MAX_REFERENCES:
result['omitted'].append(key)
continue
if card.get('voice') is not None and not voice:
@@ -370,7 +507,7 @@ def cast_references(play, shot, cards, *, continues=False):
pictures, voices = pictures + len(images), voices + voice
chosen.append((key, card, images, voice))
chosen.sort(key=lambda row: keys.index(row[0]))
- subjects, clips, picture = [], [], 0
+ subjects, clips, picture = [], [], offset
for key, card, images, voice in chosen:
result['cards'][card['id']] = card['revision']
result['references'].extend(dict(item) for item in images)
@@ -386,7 +523,15 @@ def cast_references(play, shot, cards, *, continues=False):
subjects.append(subject)
result['people'].append(dict(character=key, card=card['id'], revision=card['revision'],
pictures=len(images), voice=voice))
+ # Someone left out of the pictures still gets a written definition, numbered after the pictured people,
+ # so every carded person in the shot is a Subject and keeps one number in the story and in the voice lines.
+ for key in result['omitted']:
+ card = cards.get(characters[key]['card'])
+ if card is not None and (card.get('description') or '').strip():
+ subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'),
+ pictures=None, audio=None))
result['references'].extend(clips)
+ result['own_pictures'] = offset
result['pictures_each'] = max((len(images) for _, _, images, _ in chosen), default=0) or None
if subjects:
result['prefix'] = subject_definitions(subjects)
@@ -400,6 +545,56 @@ PROMPT_LIMIT = 20000
DESCRIPTION_CUTS = (500, 250, 120, 0)
+def ref2va(text, resolved):
+ """A prompt written for pictures: tags in the story, then summary, retention_analysis and detailed_description.
+
+ Reference generation takes six sections in a fixed order (subject_definitions comes from the cards, see with_cast);
+ the first-frame form keeps integrated_multimodal_description. Names become <Subject N> everywhere in the story."""
+ head = 'integrated_multimodal_description: '
+ subjects = resolved.get('subjects') or []
+ if not resolved.get('references') or not subjects or not text.startswith(head) or '\n\noverall_soundscape:' not in text:
+ return text
+ body, rest = text[len(head):].split('\n\noverall_soundscape:', 1)
+ tags = {}
+ for number, subject in enumerate(subjects, 1):
+ name = (subject.get('name') or '').strip()
+ if name:
+ tags[name] = number
+ for name, number in sorted(tags.items(), key=lambda kv: -len(kv[0])):
+ body = re.sub(r'(?<![\w<])%s(?!\w)' % re.escape(name), '<Subject %d>' % number, body, flags=re.I)
+ marks = [(m.start(), int(m.group(1))) for m in re.finditer(r'\[Shot (\d+)\]', body)]
+ total = len(marks)
+
+ def appears(number):
+ tag = '<Subject %d>' % number
+ shots = [n for i, (at, n) in enumerate(marks)
+ if tag in body[at:marks[i + 1][0] if i + 1 < total else len(body)]]
+ if total and len(shots) == total:
+ return 'all shots'
+ return ', '.join('[Shot %d]' % n for n in shots) if shots else 'the story'
+
+ voiced = any(s.get('audio') for s in subjects)
+ first = re.sub(r'\[Shot \d+\](?: At [\d:.]+,)?\s*', '', body.split('\n')[0]).strip()
+ if len(first) > 240:
+ cut = first[:240]
+ first = (cut[:cut.rfind('. ') + 1] if '. ' in cut else cut.rstrip()).strip()
+ summary = ('[reference generation%s] ' % (' + audio reference' if voiced else '')) + first
+ lines = []
+ for number, subject in enumerate(subjects, 1):
+ if not subject.get('pictures'):
+ lines.append('<Subject %d> (appears in %s): described - identity, hair, clothing and accessories follow '
+ 'the written description.' % (number, appears(number)))
+ continue
+ lines.append('<Subject %d> (appears in %s): fully_preserved - identity, hair, clothing and accessories are kept '
+ 'exactly as in the reference pictures.' % (number, appears(number)))
+ for number, subject in enumerate(subjects, 1):
+ if subject.get('audio'):
+ lines.append("<Audio %d>: reference - the target references the voice timbre of <Audio %d> for <Subject %d>'s lines."
+ % (subject['audio'], subject['audio'], number))
+ return ('summary: ' + summary + '\n\nretention_analysis:\n' + '\n'.join(lines) +
+ '\n\ndetailed_description:\n' + body + '\n\noverall_soundscape:' + rest)
+
+
def with_cast(text, resolved, *, limit=PROMPT_LIMIT):
"""The submitted prompt: the subject definitions, then the shot's own prompt.
@@ -407,6 +602,7 @@ def with_cast(text, resolved, *, limit=PROMPT_LIMIT):
shortened (the pictures still show each person). Plan previews and submissions both call this."""
if not resolved.get('prefix'):
return text
+ text = ref2va(text, resolved)
full = resolved['prefix'] + '\n\n' + text
if len(full) <= limit or not resolved.get('subjects'):
return full
@@ -424,8 +620,9 @@ def named(play, people, text):
for key in people:
name = play.get('characters', {})[key].get('name')
if name and key.lower() != name.lower():
- text = text.replace(key.capitalize(), name.capitalize() if text.startswith(key.capitalize())
- else name).replace(key, name)
+ text = re.sub(r'(?<!\w)(?:%s|%s)(?!\w)' % (re.escape(key.capitalize()), re.escape(key)),
+ lambda hit: name[:1].upper() + name[1:] if hit.start() == 0 and hit.group() == key.capitalize()
+ else name, text)
return text
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index cd6ce81..7f4bfeb 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -626,7 +626,7 @@ GUIDE['cast'] = [
'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, '
'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
'A shot\'s own references replace the card pictures; references=[] sends none. '
- 'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.',
+ 'A shot with a carded character is at most one generation long on this machine (freevideo_capabilities reports it; 15 s when unknown); continue it in the next shot to go longer.',
'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). '
'It defaults to the installed image model, otherwise video frames (labels guessed, label_guess). '
'When generation ends the card gets the pictures and a new revision; a failure '
diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py
index ce05deb..2a46e12 100644
--- a/freevideo_engine/agent/production.py
+++ b/freevideo_engine/agent/production.py
@@ -73,7 +73,7 @@ def _compiled(play, shot):
# What a screenplay shot and a line may say; anything else would be ignored without a word.
SHOT_FIELDS = ('id', 'seconds', 'action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music', 'takes', 'seed',
- 'continue', 'first', 'references', 'cast', 'lines')
+ 'continue', 'first', 'references', 'cast', 'lines', 'cuts')
LINE_FIELDS = ('who', 'line', 'how', 'language')
@@ -101,7 +101,7 @@ def _fields(play, problems):
problems.append(dict(path=prefix + name, message='is not a field here; use ' + ', '.join(known)))
-def plan(request, *, fields=True):
+def plan(request, *, fields=True, single_seconds=drama.BLOCK_SECONDS):
"""Return {'request': normalized copy, 'preview': {'shots': [...], 'total': N}}.
fields=False rechecks a saved screenplay, which may predate the unknown-field rule."""
@@ -112,7 +112,7 @@ def plan(request, *, fields=True):
play = deepcopy(request.get('screenplay'))
play_ok = True
try:
- drama.check(play)
+ drama.check(play, single_seconds)
except drama.Screenplay as error:
play_ok = False
problems.extend(dict(path='/screenplay' + p['path'], message=_rule(p)) for p in error.problems)
diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py
index 3c9f7d2..b050a06 100644
--- a/freevideo_engine/agent/productions.py
+++ b/freevideo_engine/agent/productions.py
@@ -24,6 +24,8 @@ import uuid
from . import drama, production, schema, spec, errors, operation_receipts, take_qa
from .jobs import DONE, JobError, iso, parts_problems
from .forecast import generation_estimate
+from .length import WAIT_SECONDS, capability_table, single_frame_limit
+from .. import chain_plan
SHEET_LEAST, SHEET_MOST = 6, 24 # frames per take when the caller gives no count: one per second
@@ -206,7 +208,7 @@ class Productions:
raise ValueError('Invalid submission marker')
production.view(state)
try:
- checked = production.plan(request, fields=False)
+ checked = production.plan(request, fields=False, single_seconds=math.inf) # saved: the limit held when it was accepted
if [s['id'] for s in state['shots']] != [s['id'] for s in checked['preview']['shots']]:
raise JobError('internal_error', 'Saved shots differ from the current plan')
production.advance(deepcopy(state), deepcopy(record['children']), now=state['updated_at'])
@@ -366,8 +368,24 @@ class Productions:
if problems:
_domain(production._invalid, problems)
+ def _single_seconds(self, request):
+ """The longest shot this machine generates at once for the film's size and quality; unknown keeps 15 s."""
+ play = request.get('screenplay') if isinstance(request, dict) else None
+ try:
+ probe = dict(prompt='x', aspect=play.get('aspect', '9:16'), quality=play.get('quality', 'light'), seconds=15)
+ if play.get('megapixels'):
+ probe['megapixels'] = play['megapixels']
+ probe = spec.normalize(probe, port=self.jobs.port)
+ # Pictures and cards make the shot a reference generation, which has its own limit.
+ table = capability_table(self.jobs.port, probe, wait=WAIT_SECONDS, task='ref2va')
+ limit = single_frame_limit(probe, table=table)
+ return chain_plan.single_seconds(limit)
+ except Exception: # A film that does not normalize is reported by its own checks.
+ return chain_plan.BLOCK_SECONDS
+
def plan(self, request, *, local=True):
- result = _domain(production.plan, {k: v for k, v in request.items() if k != 'ref'})
+ result = _domain(production.plan, {k: v for k, v in request.items() if k != 'ref'},
+ single_seconds=self._single_seconds(request))
state = _domain(production.new, 'dplanpreview', result['request'], now=iso(self.jobs.clock()))
play = result['request']['screenplay']
with self.changed:
@@ -382,7 +400,7 @@ class Productions:
row.update(cast_reference=resolved['cast_reference'], cast=resolved['people'],
cast_omitted=resolved['omitted'], cast_voice_omitted=resolved['voice_omitted'],
cast_missing=resolved['missing'], cast_pictures_each=pictures_each(resolved),
- references=len(resolved['references']))
+ references=len(resolved['references']) + resolved['own_pictures'])
try:
spec._text(row['prompt'], '/screenplay/shots/%d/prompt' % i, 20000)
except spec.SpecError as error:
@@ -411,7 +429,7 @@ class Productions:
resolved = self._carry(state['screenplay'], state['screenplay']['shots'][shot['index'] - 1],
self._current(state['screenplay']), item, continues=shot['continues'])
if resolved['references']:
- item['media'] = dict(references=deepcopy(resolved['references']))
+ item['media'] = dict(references=self._own(item) + deepcopy(resolved['references']))
estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item, port=self.jobs.port))
return dict(estimate, revision=revision)
except Exception: # Timing is optional; a forecast failure must not reject a film or job.
@@ -449,7 +467,8 @@ class Productions:
shapes.append([normalized[k] for k in ('width', 'height', 'frames')])
current = next((j for j in state['joins'] if j['ref'] == state['current_join']), None)
transition = current['transition'] if current else state['screenplay'].get('transition', 'cut')
- signature = dict(shapes=shapes, transition=transition)
+ seconds = state['screenplay'].get('transition_seconds', .5)
+ signature = dict(shapes=shapes, transition=transition, transition_seconds=seconds)
if previous and previous.get('signature') == signature and previous.get('ref') == state['current_join']:
return previous
samples = []
@@ -460,7 +479,7 @@ class Productions:
if saved.get('transition', 'cut') != transition or saved.get('width') or saved.get('height'):
continue
if saved.get('audio', True) is not True or (
- transition == 'crossfade' and saved.get('transition_seconds', .5) != .5):
+ transition == 'crossfade' and saved.get('transition_seconds', .5) != seconds):
continue
if any(set(clip) != {'from_job'} for clip in saved['clips']):
continue
@@ -970,7 +989,8 @@ class Productions:
raise JobError('production_limit', 'Too many child jobs')
if 'join' in action:
item = dict(kind='assemble', clips=[children[r]['id'] for r in item['takes']],
- transition=item['transition'])
+ transition=item['transition'],
+ transition_seconds=record['state']['screenplay'].get('transition_seconds', .5))
else:
take = next(t for s in record['state']['shots'] for t in s['takes'] if t['job'] == ref)
if take['generation'] > MAX_GENERATION:
@@ -1025,12 +1045,18 @@ class Productions:
if resolved['cast_reference'] is None:
return resolved
if resolved['references']:
- item['media'] = dict(references=deepcopy(resolved['references']))
+ item['media'] = dict(references=self._own(item) + deepcopy(resolved['references']))
+ if resolved['prefix']:
item['prompt'] = drama.with_cast(item['prompt'], resolved)
item['metadata'] = dict(item.get('metadata') or {}, cast=dict(resolved['cards']),
cast_reference=resolved['cast_reference'])
return resolved
+ @staticmethod
+ def _own(item):
+ """The pictures the shot itself names; the cast cards are added after them."""
+ return deepcopy(((item or {}).get('media') or {}).get('references') or [])
+
def _carry(self, play, shot, cards, item, *, continues):
"""The cast a shot carries: each person's sheet and face close-up on every device (one picture a person when
a continuing shot has several people). Plan previews and submissions both come here, so they agree."""
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 1adc468..f4dcf9a 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -234,7 +234,7 @@ def generate(*, local=False):
def assemble(*, local=False):
return obj(dict(kind={'const': 'assemble', 'type': 'string'},
clips=array({'anyOf': [JOB, SAVED, media(local=local, clip=True)]}, 1, 200),
- transition=enum(('cut', 'crossfade'), default='cut'), transition_seconds=number(.04, 5, default=.5),
+ transition=enum(('cut', 'crossfade'), default='cut'), transition_seconds=number(1 / 24, 3, default=.5),
width=nullable(number(64, 4096, integer=True, multipleOf=2)), height=nullable(number(64, 4096, integer=True, multipleOf=2)),
audio={'type': 'boolean', 'default': True}, name=SLUG, **common()), ('kind', 'clips'),
**{'x-job-kind': 'assemble'})
diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py
index fc3602d..ffd3174 100644
--- a/freevideo_engine/agent/voice.py
+++ b/freevideo_engine/agent/voice.py
@@ -471,14 +471,6 @@ def _measure(audio, first, last):
return (low + span[0]) / RATE, (low + span[1]) / RATE
-def _level(audio, first, last):
- """RMS of the sound in [first, last) of a mono 44.1 kHz file: the loudness the new voice is to come back with."""
- import numpy as np
- low, high = max(0, int(first * RATE)), min(len(audio), int(last * RATE))
- part = np.asarray(audio[low:high], dtype=np.float64)
- return round(float(np.sqrt(np.mean(np.square(part)))), 6) if len(part) else None
-
-
def _windows(every, todo, spans, base, limit):
"""Where each changed line's new voice goes: the original line plus a little quiet each side, and how far it may reach."""
windows, failures = {}, {}
@@ -579,9 +571,6 @@ def run_voice_replace(jobs, row, *, poll=.25):
window = windows[line['id']]
made = dict(key=line['id'], text=line['text'].strip(), seconds=round(length, 3),
onset=round(window['onset'] - window['first'], 3), offset=round(offset - window['first'], 3))
- level = _level(mono, windows[line['id']]['onset'], windows[line['id']]['offset'])
- if level:
- made['level'] = level
if own and line['end'] - line['start'] >= 1. and line['original_text'].strip():
made['reference'] = dict(file=str(original_wav), text=line['original_text'].strip(),
start=base + line['start'], end=base + line['end'])
diff --git a/freevideo_engine/chain_plan.py b/freevideo_engine/chain_plan.py
index 8e87173..03d5be8 100644
--- a/freevideo_engine/chain_plan.py
+++ b/freevideo_engine/chain_plan.py
@@ -101,6 +101,14 @@ def _lengths(total, first, following, least, continued):
return lengths
+def single_seconds(max_frames=None):
+ """Whole seconds of the longest first block a device with this frame limit generates at once (unknown: 15 s)."""
+ if max_frames is None:
+ return BLOCK_SECONDS
+ limit = _number(max_frames, 'The frame limit', whole=True)
+ return (limit - (limit - 5) % UNIT) // FPS if limit >= 39 else 0
+
+
def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, block_seconds=None, continued=False,
max_parts=None, first_frame=False):
"""`max_frames`: the most frames this device generates at once for the current settings (unknown: 15 s).
diff --git a/freevideo_engine/dubbing/align.py b/freevideo_engine/dubbing/align.py
index 355b04c..5c3ec0d 100644
--- a/freevideo_engine/dubbing/align.py
+++ b/freevideo_engine/dubbing/align.py
@@ -180,20 +180,14 @@ def fit(synth, target, *, onset=0., offset=None, tolerance=TOLERANCE, max_stretc
tried.append((speed, voice, rate, seconds(voice, rate), edges))
return tried[-1]
- # A line that is too long is first closed by the tempo change, which keeps the voice natural; the model
- # speed only takes what the tempo change cannot (a faster model speed costs naturalness quickly).
- # A line that is too short is slowed by the tempo change in the same way, inside the same range.
- def aim(length):
- return min(max(length, wanted / (max_stretch * .96)), wanted * max_stretch * .96) # inside this range no model speed is needed
-
def off(entry):
- return abs(entry[3] - aim(entry[3])) / wanted
+ return abs(entry[3] - wanted) / wanted
speed = 1.0
entry = run(speed)
while off(entry) > tolerance and len(tried) < 3:
# the length scales with 1/speed, so the next speed is the measured ratio
- speed = min(max(speed * entry[3] / aim(entry[3]), low), high)
+ speed = min(max(speed * entry[3] / wanted, low), high)
if speed == tried[-1][0]:
break
entry = run(speed)
diff --git a/freevideo_engine/dubbing/service.py b/freevideo_engine/dubbing/service.py
index 0c36c65..a0ae7b2 100644
--- a/freevideo_engine/dubbing/service.py
+++ b/freevideo_engine/dubbing/service.py
@@ -90,11 +90,6 @@ def request(value):
raise ValueError('invalid_request')
if line.get('reference') is not None:
row['reference'] = reference_of(line['reference'], required=True)
- level = line.get('level')
- if level is not None:
- if isinstance(level, bool) or not isinstance(level, (int, float)) or not 0 < level <= 1:
- raise ValueError('invalid_request')
- row['level'] = float(level)
instruction = line.get('instruction')
if instruction not in (None, ''):
if not isinstance(instruction, str) or len(instruction) > 200:
diff --git a/freevideo_engine/dubbing/worker.py b/freevideo_engine/dubbing/worker.py
index 7ce212e..8bca359 100644
--- a/freevideo_engine/dubbing/worker.py
+++ b/freevideo_engine/dubbing/worker.py
@@ -181,13 +181,11 @@ def run(request, model_dir, device='cuda'):
used = []
match = {} # per line with its own reference: the voiced RMS the line is to come back with
for index, line in enumerate(lines):
- if line.get('level'):
- match[index] = float(line['level'])
own = line.get('reference')
if own:
try:
used.append(('line',) + prepare(own, 'line%d' % index) + (cut['reference-line%d.wav' % index], None))
- match.setdefault(index, levels.get('line%d' % index))
+ match[index] = levels.get('line%d' % index)
continue
except ValueError as error:
if str(error) != 'reference_short' or job_ref is None: