Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

44.0 KB
Raw Blame
"""Editing projects for agents: the page's timeline, read and changed through the API.

A project is the editor page's own record (web/editor_model.js). Operations run
the page's rule code in QuickJS (editor_runtime), so the Agent and the page cut,
move and make room the same way; the result is saved with the page's save, whose
revision check makes the later of two writers receive project_changed.
"""
from __future__ import annotations

from copy import deepcopy
import hashlib
import json
import logging
from pathlib import Path
import re
import secrets
import threading
import time

from .. import comfy_editor
from . import editor_runtime, errors, schema, spec
from .jobs import JobError, library_id, library_view

MAX_OPERATIONS = 64
MAX_RECEIPTS = 256
# Rule codes that describe the timeline's state; the rest are request errors.
STATE_CODES = {'busy': 'clip_busy', 'locked': 'clip_locked', 'audio_full': 'audio_full', 'overlap': 'overlap'}
# What a rejected operation needed, by the page's rule code.
RULE_EXPECTED = {
    'range': 'clip_ids that exist and a frame inside them (frames are whole numbers at 24 fps)',
    'missing': 'clip_ids from this project revision',
    'text': 'a short description of at most one paragraph',
    'ref': 'a ref of 1-128 letters, digits, - or _, or an empty string to remove it',
    'source': 'a source that has a picture or sound',
    'invalid': 'an op named in the operations schema with its fields',
}
# The note operation shares the page's 'text' code with describe; what it needed is its own.
NOTE_EXPECTED = 'picture clips and a text of at most 120 characters on one line (an empty text removes the note)'
# An agent acts on the last answer it read, so the answer that brings new picture clips asks for their notes.
# It comes first in that answer: the project after it carries every clip's description, and a real agent
# that printed only the start of a long answer never saw a request placed after it.
NOTE_NEXT = ('Picture clips %s have no note yet. Write each one now with a note operation: in the user\'s language, '
             'what is in the picture, subject first; not its position and not its progress.')


def note_next(result, project, ids):
    missing = [clip['id'] for track in project['tracks'] if track['kind'] == 'video' for clip in track['clips']
               if clip['id'] in ids and not clip.get('note')]
    return dict(next=NOTE_NEXT % ', '.join(missing), **result) if missing else result


def new_project(given):
    """The page's empty timeline with what the caller sent on top; a name alone is enough to start one."""
    return dict(dict(version=1, id=None, revision=0, name='', fps=comfy_editor.FPS, width=1344, height=768, settings=None, serial=0,
                     tracks=[dict(id='v1', kind='video', clips=[]), dict(id='a1', kind='audio', clips=[])]), **given)


class ProjectError(JobError):
    """A project error; a conflict carries the current project so the caller can retry from it."""

    def __init__(self, code, message=None, *, project=None, operation=None):
        super().__init__(code, message or errors.ERRORS[code][2])
        self.status = errors.ERRORS[code][0]
        self.project, self.operation = project, operation

    def view(self):
        value = errors.build(self.code, validation=True, details=[], message=self.message,
                             next_action=errors.ERRORS[self.code][3])
        if self.operation is not None:
            value['details'].append(dict(operation=self.operation))
        if self.project is not None:
            value['project'] = deepcopy(self.project)
            value['current_revision'] = self.project['revision']
        return value


def _as_expected(project, clip_id, expect):
    from .voice import signature
    return signature(_clips(project).get(clip_id)) == expect


def _invalid(path, expected, received=None):
    return spec.SpecError([schema.diagnostic(path, expected, received)])


def _refused(project, error):
    """The editor's own check names the part it refused; an Agent cannot take its next step without it."""
    # The editor's reasons are fixed phrases ("Invalid clip", "Invalid source"); they repeat nothing supplied.
    reason = str(error) if type(error) is ValueError and re.fullmatch(r'[A-Z][A-Za-z ]{2,60}', str(error)) else ''
    row = schema.diagnostic('/project', 'a valid editor project' + (' (%s)' % (reason[:1].lower() + reason[1:]) if reason else ''),
                            project)
    row['next'] = ('Correct that part of /project. For a new timeline send only {"project": {"name": "..."}}; '
                   'then add each clip with an insert operation.')
    return spec.SpecError([row])


def _digest(scope, payload):
    return hashlib.sha256(json.dumps(dict(scope=scope, body=payload), sort_keys=True, separators=(',', ':'),
                                     ensure_ascii=True, allow_nan=False).encode()).hexdigest()


def _operation_key(ref, client=None):
    """The page save's 32-hex operation id for an agent ref, so a lost reply can be replayed."""
    key = 'agent:' + ref if client is None else 'agent-client:' + json.dumps(
        [client, ref], ensure_ascii=True, separators=(',', ':'))
    return hashlib.sha256(key.encode()).hexdigest()[:32]


def _clips(project):
    return {clip['id']: clip for track in project['tracks'] for clip in track['clips']}


def changed_clip_ids(before, after):
    """Ids of parts added, removed or changed by an edit, in timeline order of the result, then removed ones."""
    old, new = _clips(before), _clips(after)
    changed = [identity for identity, clip in new.items() if old.get(identity) != clip]
    return changed + [identity for identity in old if identity not in new]


def _claims(project):
    """(path, source key, length) wherever a project says how long a source is: every clip and every version of it."""
    tracks = project.get('tracks') if isinstance(project, dict) else None
    for t, track in enumerate(tracks if isinstance(tracks, list) else ()):
        clips = track.get('clips') if isinstance(track, dict) else None
        for c, clip in enumerate(clips if isinstance(clips, list) else ()):
            if not isinstance(clip, dict):
                continue
            path, versions = '/project/tracks/%d/clips/%d' % (t, c), clip.get('versions')
            rows = [(path, clip.get('source'), clip.get('length'))]
            rows += [('%s/versions/%d' % (path, v), row, row.get('length'))
                     for v, row in enumerate(versions if isinstance(versions, list) else ()) if isinstance(row, dict)]
            for where, source, length in rows:
                if (isinstance(source, dict) and isinstance(source.get('kind'), str) and isinstance(source.get('id'), str)
                        and comfy_editor._whole(length, 1, 10**7)):
                    yield where + '/length', source['kind'] + ':' + source['id'], int(length)


class Projects:
    def __init__(self, root, *, directory=None):
        self.root = root  # The ComfyUI output folder, where the page keeps FreeVideo/.editor
        self.lock = threading.Lock()
        self.listen_lock = threading.Lock()
        self.receipts = {}  # (client, ref) -> {digest, result}; in memory, bounded; the page save keeps the durable one.
        self.prepared = {}  # job id -> the reserved project, until its chain id is written into it.

    def _load(self, identity):
        try:
            return comfy_editor.load(self.root, identity)
        except (FileNotFoundError, ValueError) as error:
            raise ProjectError('not_found', 'The project does not exist.') from error

    def _receipt(self, scope, payload, ref, client=None):
        if ref is None:
            return None, None
        digest = _digest(scope, payload)
        with self.lock:
            old = self.receipts.get((client, ref))
        if old is not None:
            if old['digest'] != digest:
                raise JobError('idempotency_conflict', 'Key already used')
            return digest, deepcopy(old['result'])
        return digest, None

    def _remember(self, ref, digest, result, client=None):
        if ref is None:
            return
        with self.lock:
            self.receipts[client, ref] = dict(digest=digest, result=deepcopy(result))
            while len(self.receipts) > MAX_RECEIPTS:
                self.receipts.pop(next(iter(self.receipts)))

    def _save(self, project, *, ref=None, create=None, client=None):
        operation = _operation_key(ref, client) if ref is not None else None
        try:
            return comfy_editor.save(self.root, project, create=create, operation=operation)
        except comfy_editor.Conflict as error:
            raise ProjectError('project_changed', project=self._load(error.identity or project['id'])['project']) from error
        except comfy_editor.MissingProject as error:
            raise ProjectError('not_found', 'The project does not exist.') from error
        except comfy_editor.TooLarge as error:
            raise ProjectError('too_large') from error
        except OSError as error:
            raise ProjectError('storage_unavailable') from error
        except (ValueError, TypeError, KeyError, RecursionError) as error:
            raise _refused(project, error) from error

    def _rejected(self, error, stored, path, received, *, operation=None):
        if error['code'] in STATE_CODES:
            raise ProjectError(STATE_CODES[error['code']], project=stored, operation=operation)
        if error['code'] == 'text' and isinstance(received, dict) and received.get('op') == 'note':
            raise _invalid(path, NOTE_EXPECTED, received)
        raise _invalid(path, RULE_EXPECTED.get(error['code'], 'an operation this timeline allows'), received)

    # Reads ---------------------------------------------------------------------------------------------
    def list(self):
        return dict(items=comfy_editor.list_projects(self.root))

    def get(self, identity):
        return self._load(identity)

    # Whole-project writes ------------------------------------------------------------------------------
    def _lengths(self, project, stored=None):
        """A written project cannot say a source is longer than it is.

        The page takes every length from the file; a whole-project write states it. A clip that asks for
        frames its source does not have would only be found by the export, which stops there.
        """
        known = {row[1:] for row in _claims(stored)} if stored else set()
        for path, key, length in _claims(project):
            if (key, length) in known:
                continue
            try:
                info = comfy_editor.sources(self.root, [key])[key]
            except ValueError:
                continue  # not a source at all: the editor's own check refuses it and names the part
            known.add((key, length))
            if not info['missing'] and info['frames'] and length > info['frames']:
                raise _invalid(path, 'at most %d, the number of frames this source has' % info['frames'], length)

    def create(self, value, *, ref=None, client=None):
        schema.validate(schema.project_schema('create'), value)
        digest, replay = self._receipt('projects:create', value, ref, client)
        if replay is not None:
            return replay
        project = value.get('project') if isinstance(value, dict) else None
        project = new_project(project) if isinstance(project, dict) else None
        if project is None or project.get('id') is not None or project.get('revision') != 0:
            raise _invalid('/project', 'a new project: its name is enough, with no id and no revision')
        # The page's receipt makes a repeated create return the first project.
        create = _operation_key(ref, client) if ref is not None else None
        self._lengths(project)
        result = dict(project=self._save(deepcopy(project), create=create))
        self._remember(ref, digest, result, client)
        return result

    def write(self, identity, value, *, ref=None, client=None):
        schema.validate(schema.project_schema('write'), value)
        digest, replay = self._receipt('projects:write', dict(id=identity, body=value), ref, client)
        if replay is not None:
            return replay
        project = value.get('project') if isinstance(value, dict) else None
        if not isinstance(project, dict) or project.get('id') != identity:
            raise _invalid('/project/id', 'the project id in the path', project.get('id') if isinstance(project, dict) else None)
        before = self._load(identity)['project']
        self._lengths(project, before)
        saved = self._save(deepcopy(project), ref=ref, client=client)
        result = dict(project=saved, changed_clip_ids=changed_clip_ids(before, saved))
        self._remember(ref, digest, result, client)
        return result

    # Operations ----------------------------------------------------------------------------------------
    def _media(self, operations):
        """Insert operations name a source key; the server fills in its length and streams, and for a
        saved video the description it was generated from."""
        keys = [op['source'] for op in operations if op.get('op') == 'insert']
        found = comfy_editor.sources(self.root, keys) if keys else {}
        resolved = []
        for index, op in enumerate(operations):
            if op.get('op') != 'insert':
                resolved.append(op)
                continue
            info = found[op['source']]
            if info['missing'] or not info['frames']:
                raise ProjectError('source_missing', 'The source to insert is not available.', operation=index)
            kind, identity = op['source'].split(':', 1)
            media = dict(source=dict(kind=kind, id=identity), frames=info['frames'], hasVideo=info['has_video'],
                         hasAudio=info['has_audio'], width=info['width'], height=info['height'])
            if info.get('text'):
                media['text'] = info['text']
            resolved.append(dict({k: v for k, v in op.items() if k != 'source'}, media=media))
        return resolved

    def operations(self, identity, value, *, client=None, ref=None):
        """Apply 1-64 operations to the revision the caller read; all succeed and save once, or nothing changes."""
        schema.validate(schema.project_schema('operations'), value)
        revision, operations = value.get('revision'), value.get('operations')
        digest, replay = self._receipt('projects:operations', dict(id=identity, body=value), ref, client)
        if replay is not None:
            return replay
        stored = self._load(identity)['project']
        if stored['revision'] != revision:
            raise ProjectError('project_changed', project=stored)
        try:
            for index, op in enumerate(operations):
                if op.get('op') == 'insert':
                    comfy_editor.source_key(dict(zip(('kind', 'id'), op['source'].split(':', 1))))
        except (ValueError, TypeError, AttributeError) as error:
            raise _invalid('/operations/%d/source' % index,
                           'a source key: work:<library id>, generated:<file> or import:<id>', op.get('source')) from error
        reply = editor_runtime.runtime().batch(stored, self._media(operations))
        if not reply['ok']:
            index = reply['index']
            self._rejected(reply['error'], stored, '/operations/%d' % index if index is not None else '/project',
                           operations[index] if index is not None else None, operation=index)
        saved = self._save(reply['project'], ref=ref, client=client)
        result = dict(project=saved, changed_clip_ids=changed_clip_ids(stored, saved), results=reply['results'])
        result = note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored)))
        self._remember(ref, digest, result, client)
        return result


    # Generation ----------------------------------------------------------------------------------------
    def generate(self, identity, value, *, jobs, capability=None, ref=None, client=None):
        """Make new video and sound for a range or parts, as the page's Generate does.

        capability(size) returns the device's capability reply for that size, or None.

        The parts are planned and saved first (the reservation), so a page that
        saved in between makes this fail with project_changed before any GPU
        work; only then is the chain submitted. Its blocks fill the parts as
        they finish (deliver), whether or not the page is open.
        """
        schema.validate(schema.project_schema('generate'), value)
        digest, replay = self._receipt('projects:generate', dict(id=identity, body=value), ref, client)
        if replay is not None:
            return replay
        text, target, settings = value.get('text', ''), value['target'], value.get('settings') or {}
        stored = self._load(identity)['project']
        if stored['revision'] != value['revision']:
            raise ProjectError('project_changed', project=stored)
        rules = editor_runtime.runtime()
        plan = rules.generation(mode='plan', project=stored, target=target, text=text, capability=None)
        # Part lengths follow what this device can make at the project's size, as on the page;
        # until that is measured the page's 15 s parts apply.
        reply = capability(plan['size']) if capability is not None and plan['ok'] else None
        if isinstance(reply, dict) and reply.get('status', 'ready') == 'ready':
            plan = rules.generation(mode='plan', project=stored, target=target, text=text, capability=reply)
        if not plan['ok']:
            self._rejected(plan['error'], stored, '/target', target)
        found = comfy_editor.sources(self.root, plan['keys']) if plan['keys'] else {}
        if any(info['missing'] or not info.get('path') for info in found.values()):
            raise ProjectError('source_missing', 'The part this generation continues from is not available.')
        request = rules.generation(mode='request', project=plan['project'], ids=plan['ids'], text=text,
                                   paths={key: info['path'] for key, info in found.items()},
                                   sizes={key: dict(width=info['width'], height=info['height']) for key, info in found.items()})
        if not request['ok']:
            self._rejected(request['error'], stored, '/target', target)
        size, limit = plan['size'], plan['max_request_frames']
        if request.get('size') and request['size'] != size:
            # A part continuing footage of another size has that size's limit, as on the page.
            size = request['size']
            reply = capability(size) if capability is not None else None
            if isinstance(reply, dict) and reply.get('status', 'ready') == 'ready':
                limit = rules.generation(mode='plan', project=stored, target=target, text=text, capability=reply,
                                         size=size).get('max_request_frames')
        longest = max(block['frames'] for block in request['blocks'])
        if limit is not None and longest > limit:
            raise _invalid('/target', 'parts of at most %d frames at this size and quality; choose a shorter range'
                           % limit, longest)
        seed = settings['seed'] if settings.get('seed') is not None else secrets.randbelow(2**32)
        blocks = [dict(block, seed=seed + index) for index, block in enumerate(request['blocks'])]
        prompt = text.strip() or next((b['text'] for b in blocks if b['text'].strip()), '') or 'Continue the video.'
        frames = sum(block['frames'] for block in blocks)
        spec = dict({k: v for k, v in settings.items() if k in ('quality', 'sampling', 'loras', 'force_regenerate')},
                    kind='generate', prompt=prompt, seed=seed, width=size['width'],
                    height=size['height'], seconds=min(60., max(1.625, frames / 24)),
                    label='Editing project ' + identity)
        saved = self._save(plan['project'])  # The reservation: the page's revision check orders it.
        editor = dict(project=identity, ids=plan['ids'], blocks=blocks, revision=saved['revision'])
        try:
            job = jobs.submit(spec, client=client, editor=editor)[0]
        except Exception:
            self._release(identity, plan['ids'])
            raise
        with self.lock:
            self.prepared[job['id']] = saved
        result = note_next(dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']),
                           saved, set(plan['ids']))
        self._remember(ref, digest, result, client)
        return result

    def export(self, identity, value, *, jobs, ref=None, client=None):
        """Queue an export job for the revision the caller read; a stale revision starts nothing."""
        schema.validate(schema.project_schema('export'), value)
        digest, replay = self._receipt('projects:export', dict(id=identity, body=value), ref, client)
        if replay is not None:
            return replay
        stored = self._load(identity)['project']
        if stored['revision'] != value['revision']:
            raise ProjectError('project_changed', project=stored)
        job = jobs.submit(dict(kind='export', editor_project=identity, revision=value['revision']), client=client)[0]
        result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
        self._remember(ref, digest, result, client)
        return result

    def listen(self, identity, value, *, jobs, ref=None, client=None):
        """Queue a listening job (align or transcribe) for the revision the caller read; a stale revision starts nothing."""
        schema.validate(schema.project_schema('listen'), value)
        if (value['mode'] == 'align') != ('lines' in value):
            raise _invalid('/lines', 'lines for mode align, and none for transcribe', value.get('lines'))
        with self.listen_lock:  # the same key sent twice at once queues one job
            digest, replay = self._receipt('projects:listen', dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            job = jobs.submit(dict(kind='listen', editor_project=identity,
                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
            self._remember(ref, digest, result, client)
            return result

    def music(self, identity, value, *, jobs, ref=None, client=None):
        """Queue a music job for the revision the caller read; a stale revision starts nothing."""
        schema.validate(schema.project_schema('music'), value)
        with self.listen_lock:  # the same key sent twice at once queues one job
            digest, replay = self._receipt('projects:music', dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            if not comfy_editor.duration(stored):
                raise _invalid('/revision', 'a revision whose timeline has clips', value['revision'])
            job = jobs.submit(dict(kind='music', editor_project=identity,
                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
            self._remember(ref, digest, result, client)
            return result

    def finish_sound(self, identity, value, *, jobs, ref=None, client=None):
        """Queue one job that finishes the sound of the cut the caller read; a stale revision starts nothing."""
        schema.validate(schema.project_schema('finish_sound'), value)
        with self.listen_lock:
            digest, replay = self._receipt('projects:finish_sound', dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            if not comfy_editor.duration(stored):
                raise _invalid('/revision', 'a revision whose timeline has clips', value['revision'])
            job = jobs.submit(dict(kind='finish_sound', editor_project=identity,
                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
            self._remember(ref, digest, result, client)
            return result

    def stems(self, identity, value, *, jobs, ref=None, client=None):
        """Queue a sound-splitting job for one sound clip of the revision the caller read."""
        schema.validate(schema.project_schema('stems'), value)
        with self.listen_lock:
            digest, replay = self._receipt('projects:stems', dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            clip = _clips(stored).get(value['clip_id'])
            if clip is None or not clip.get('source') or clip.get('stem_of'):
                raise _invalid('/clip_id', 'a sound clip with media that is not itself a separated part', value['clip_id'])
            job = jobs.submit(dict(kind='stems', editor_project=identity,
                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
            self._remember(ref, digest, result, client)
            return result

    def stems_export(self, identity, value, *, ref=None, client=None):
        """Copy the four wav files of a split clip into the saved sounds; the clip must be one that was split."""
        from ..stems.separate import SeparationError, export_tracks
        schema.validate(schema.project_schema('stems_export'), value)
        with self.listen_lock:
            digest, replay = self._receipt('projects:stems_export', dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            parts = [clip for clip in _clips(stored).values() if clip.get('stem_of') == value['clip_id']]
            if not parts:
                raise _invalid('/clip_id', 'a clip that was split into dialogue, music and ambience', value['clip_id'])
            folder = parts[0]['source']['id'].rsplit('/', 1)[0]
            if not folder.startswith('freevideo-stems/') or folder.count('/') != 1:
                raise _invalid('/clip_id', 'a clip split by this version', value['clip_id'])
            name = folder.split('/', 1)[1]
            try:
                files = export_tracks(Path(self.root) / folder, Path(self.root) / 'freevideo-stems-saved' / name)
            except SeparationError as error:
                raise JobError('stems_failed', 'The separated sound cannot be saved (%s)' % error) from error
            result = dict(editor_project=identity, revision=stored['revision'], clip_id=value['clip_id'],
                          folder='freevideo-stems-saved/' + name, files=files)
            self._remember(ref, digest, result, client)
            return result

    def _voice_job(self, name, kind, identity, value, *, jobs, ref, client):
        """Queue a voice job for the revision the caller read; the selected clip must be a sound clip with media."""
        schema.validate(schema.project_schema(name), value)
        with self.listen_lock:
            digest, replay = self._receipt('projects:' + name, dict(id=identity, body=value), ref, client)
            if replay is not None:
                return replay
            stored = self._load(identity)['project']
            if stored['revision'] != value['revision']:
                raise ProjectError('project_changed', project=stored)
            selection = value['selection']
            clip = _clips(stored).get(selection['clip_id'])
            if clip is None or not clip.get('source'):
                raise _invalid('/selection/clip_id', 'a sound clip with media', selection['clip_id'])
            if 'end' in selection and selection['end'] <= selection.get('start', 0):
                raise _invalid('/selection/end', 'a time after the start', selection['end'])
            if kind == 'voice':
                ids = [line['id'] for line in value['lines']]
                if len(set(ids)) != len(ids):
                    raise _invalid('/lines', 'lines with different ids', ids[:3])
                for i, line in enumerate(value['lines']):
                    if line['end'] <= line['start']:
                        raise _invalid('/lines/%d/end' % i, 'a time after the start', line['end'])
                from . import voice
                if not value.get('recognition'):
                    raise _invalid('/recognition', 'the recognition voice_lines gave for these lines; run voice_lines first')
                wrong = voice.check(value['recognition'], identity, selection['clip_id'], value['lines'])
                if wrong:
                    raise _invalid(*wrong)
                reference = (value.get('voice') or {}).get('reference')
                if reference and reference['clip_id'] not in _clips(stored):
                    raise _invalid('/voice/reference/clip_id', 'a clip of this project', reference['clip_id'])
            job = jobs.submit(dict(kind=kind, editor_project=identity,
                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
            self._remember(ref, digest, result, client)
            return result

    def voice_lines(self, identity, value, *, jobs, ref=None, client=None):
        """Queue the writing down of a selection's lines (what is said, and when)."""
        return self._voice_job('voice_lines', 'voice_lines', identity, value, jobs=jobs, ref=ref, client=client)

    def voice_replace(self, identity, value, *, jobs, ref=None, client=None):
        """Queue the new voice for the lines the caller changed."""
        return self._voice_job('voice_replace', 'voice', identity, value, jobs=jobs, ref=ref, client=client)

    def voice_estimate(self, identity, value, *, ref=None, client=None):
        """Seconds the changed lines take to make; asked at once, nothing is stored."""
        from . import voice
        schema.validate(schema.project_schema('voice_estimate'), value)
        self._load(identity)
        return voice.estimate(value['lines'], value.get('limit'))

    def voice_shorten(self, identity, value, *, ref=None, client=None):
        """Rewriting a line to a length needs a text model; none is wired in this version."""
        schema.validate(schema.project_schema('voice_shorten'), value)
        self._load(identity)
        raise JobError('shorten_unavailable', 'No text model is available to shorten the line')

    def lay_stems(self, identity, clip_id, sources, *, expect=None):
        """Put the three parts on sub-tracks under the clip and silence the clip; returns the save and the new clip IDs.

        `expect` (voice.signature of the clip) makes it a no-op that raises project_changed when the clip is not as it was."""
        before, refused = {}, {}
        def change(project):
            before['ids'] = set(_clips(project))
            if expect is not None and not _as_expected(project, clip_id, expect):
                refused['stale'] = True
                return dict(ok=True, project=project)
            reply = editor_runtime.runtime().call('placeStems', project, [clip_id, {
                name: dict(kind='generated', id=source.split(':', 1)[1]) for name, source in sources.items()}],
                module='model', serialize=True)[0]
            if not reply['ok']:
                refused.update(reply['error'])
                return dict(ok=True, project=project)
            return dict(ok=True, project=reply['value'])
        saved = self._update(identity, change)
        if refused.get('stale'):
            raise JobError('project_changed', 'The clip changed while the voice was made')
        if refused:
            code = STATE_CODES.get(refused.get('code'), 'stems_failed')
            raise JobError(code, 'The separated sound cannot be laid under the clip (%s)' % refused.get('code'))
        return saved, sorted(set(_clips(saved)) - before['ids'])

    def replace_voice_source(self, identity, clip_id, source, *, expect=None):
        """The dialogue clip takes the file with the replaced voice; its place and range stay. Returns the save.

        `expect` (voice.signature of the clip) makes it a no-op that raises project_changed when the clip is not as it was."""
        refused = {}
        def change(project):
            if expect is not None and not _as_expected(project, clip_id, expect):
                refused['stale'] = True
                return dict(ok=True, project=project)
            reply = editor_runtime.runtime().call('replaceSource', project, [clip_id, dict(
                kind='generated', id=source.split(':', 1)[1])], module='model', serialize=True)[0]
            if not reply['ok']:
                refused.update(reply['error'])
                return dict(ok=True, project=project)
            return dict(ok=True, project=reply['value'])
        saved = self._update(identity, change)
        if refused.get('stale'):
            raise JobError('project_changed', 'The clip changed while the voice was made')
        if refused:
            code = STATE_CODES.get(refused.get('code'), 'voice_failed')
            raise JobError(code, 'The replaced voice cannot be put on the sound track (%s)' % refused.get('code'))
        return saved

    def lay_music(self, identity, source):
        """Put a finished score on a2 from frame 0 of the project as it is now; returns the save and the new clip IDs."""
        found = comfy_editor.sources(self.root, [source])[source]
        if found['missing'] or not found['frames']:
            raise JobError('music_failed', 'The music file is not readable')
        kind, path = source.split(':', 1)
        operation = dict(op='insert', at=0, media=dict(source=dict(kind=kind, id=path), frames=found['frames'], hasVideo=False,
                                                        hasAudio=True, width=0, height=0, targetTrack='a2'))
        before, refused = {}, {}
        def change(project):
            before['ids'] = set(_clips(project))
            if any(track['id'] == 'a2' and track['clips'] for track in project['tracks']):
                # An insert there would push the whole cut later; an earlier score is removed first.
                refused.update(code='audio_full')
                return dict(ok=True, project=project)
            reply = editor_runtime.runtime().batch(project, [operation])
            if not reply['ok']:
                refused.update(reply['error'])
                return dict(ok=True, project=project)  # nothing changes; the reason is raised below
            return reply
        saved = self._update(identity, change)
        if refused:
            code = STATE_CODES.get(refused.get('code'), 'music_failed')
            raise JobError(code, 'Sound track a2 cannot take the music from frame 0 (%s)' % refused.get('code'))
        return saved, sorted(set(_clips(saved)) - before['ids'])

    def _update(self, identity, change, *, tries=4):
        """Read, change and save; a page save in between makes it read again. None when nothing changed."""
        for _ in range(tries):
            current = self._load(identity)['project']
            reply = change(current)
            if not reply['ok']:
                raise ProjectError('invalid_output', reply['error']['message'])
            if reply['project'] == current:
                return None
            try:
                return comfy_editor.save(self.root, reply['project'])
            except comfy_editor.Conflict:
                continue
        raise ProjectError('project_changed', project=self._load(identity)['project'])

    def _release(self, identity, ids):
        """A reservation whose job was refused returns to draft, as the page's failed start does."""
        try:
            self._update(identity, lambda project: editor_runtime.runtime().deliver(
                project=project, ids=ids, job=None, settle='draft'))
        except (JobError, OSError) as error:
            logging.getLogger(__name__).warning('Project %s parts stayed reserved (%s)', identity, type(error).__name__)

    def attach(self, row, *, resumed=False):
        """Write the chain id into the reserved parts (the page's assignJob), or mark them queued again on resume."""
        editor, chain = row['editor'], row['chain_id']
        with self.lock:
            prepared = self.prepared.pop(row['id'], None)
        def change(project):
            if resumed:
                return editor_runtime.runtime().deliver(project=project, ids=editor['ids'], job=chain, settle='queued')
            # After a restart the reservation is gone from memory; the parts' own plan stands in for it.
            return editor_runtime.runtime().deliver(project=project, ids=editor['ids'], job=chain,
                                                    assign=prepared or project)
        self._apply(row, change)

    def deliver(self, row, chain):
        """Fill, fail or settle the parts from the chain's blocks; idempotent, so every poll may call it."""
        editor, sources = row['editor'], {}
        state = [[b['id'], b['state'], b['output']] for b in chain['blocks']] + [chain['state']]
        if editor.get('delivered') == state:
            return
        outputs = [b['output'] for b in chain['blocks'] if b['state'] == 'done' and b['output']]
        if outputs:
            found = comfy_editor.sources(self.root, ['generated:' + output for output in outputs])
            sources = {output: found['generated:' + output] for output in outputs}
        if self._apply(row, lambda project: editor_runtime.runtime().deliver(
                project=project, ids=editor['ids'], job=chain['id'], chain=chain, sources=sources)):
            editor['delivered'] = state

    def _apply(self, row, change):
        editor = row['editor']
        try:
            saved = self._update(editor['project'], change)
        except (JobError, OSError) as error:
            # The job goes on; the next poll tries again, and an open page fills the parts the same way.
            editor['problem'] = getattr(error, 'code', 'storage_unavailable')
            return False
        editor.pop('problem', None)
        if saved is not None:
            editor['revision'] = saved['revision']
        return True


# Export ----------------------------------------------------------------------------------------------
# Both are about a clip in the project, so the answer says to look there, not to resume a generation.
EXPORT_REASONS = {'source_missing': 'clip_source_unusable', 'unreadable': 'clip_source_unusable', 'disk_full': 'storage_unavailable'}


def run_export(port, row, *, poll=.25):
    """An export job: one saved revision rendered by the page's export queue, in its turn with the page's exports."""
    from .. import editor_export
    from ..editor_text import MissingGlyphs
    spec = row['spec']
    queue = port.editor_exports()
    if queue is None:
        raise JobError('not_ready', 'The editor export service is not running')
    try:
        view = queue.submit(spec['editor_project'], spec['revision'])
    except editor_export.ExportConflict as error:
        raise JobError('project_changed', 'The project changed since you read it (now revision %d)' % error.revision) from error
    except MissingGlyphs as error:
        raise JobError('missing_glyphs', 'Text in the project uses characters the fonts cannot draw',
                       details=[dict(items=error.items)]) from error
    except FileNotFoundError as error:
        raise JobError('not_found', 'The project does not exist') from error
    except editor_export.ExportError as error:
        raise JobError('busy', 'The editor export queue is full; retry after the page\'s exports finish') from error
    except ValueError as error:
        raise JobError('invalid_request', 'This project cannot be exported: it has no picture or sound') from error
    identity, cancelled = view['id'], False
    row['_phase'] = 'exporting'
    while view['state'] == 'running':
        if row.get('_cancel') and not cancelled:
            view, cancelled = queue.cancel(identity), True
            continue
        row['_detail'] = json.dumps(dict(fraction=round(view['fraction'], 3)))
        time.sleep(poll)
        view = queue.get(identity)
    if view['state'] == 'cancelled':
        raise JobError('cancelled', 'Cancelled')
    if view['state'] != 'done':
        code = EXPORT_REASONS.get(view.get('reason'), 'export_failed')
        raise JobError(code, 'The export failed (%s)' % (view.get('reason') or 'failed'))
    relative = view['path']
    return dict(library_id=library_id(relative), video=library_view(port, relative),
                editor_project=spec['editor_project'], revision=spec['revision'])


# Listen ----------------------------------------------------------------------------------------------
LISTEN_REASONS = {'no_engine': 'no_speech_model', 'language': 'no_speech_model', 'no_sound': 'invalid_request', 'too_long': 'invalid_request',
                  'source_missing': 'clip_source_unusable', 'unreadable': 'clip_source_unusable',
                  'disk_full': 'storage_unavailable'}
LISTEN_MESSAGES = {
    'no_engine': 'No speech model is installed on this machine; write the subtitle lines yourself with the subtitles edit',
    'no_sound': 'The timeline has no sound to listen to',
    'too_long': 'The sound is too long for one listening job; give a shorter range'}


def run_listen(port, row, *, poll=.25):
    """A listening job: the page's subtitle ears (align or transcribe) over one saved revision."""
    spec = row['spec']
    queue = port.editor_ears()
    if queue is None:
        raise JobError('not_ready', 'The subtitle service is not running')
    request = dict(kind=spec['mode'], id=spec['editor_project'], revision=spec['revision'], language=spec['language'])
    request.update({k: spec[k] for k in ('lines', 'range') if k in spec})
    try:
        view = queue.submit(request)
    except FileNotFoundError as error:
        raise JobError('not_found', 'The project does not exist') from error
    except (ValueError, TypeError) as error:
        raise JobError('invalid_request', 'This project cannot be listened to') from error
    except Exception as error:
        if getattr(error, 'reason', None) is None:
            raise
        raise JobError('busy', 'The subtitle queue is full; retry shortly') from error
    identity, cancelled = view['id'], False
    row['_phase'] = 'listening'
    try:
        while view['state'] == 'running':
            if row.get('_cancel') and not cancelled:
                view, cancelled = queue.cancel(identity), True
                continue
            row['_detail'] = json.dumps(dict(fraction=round(view['fraction'], 3)))
            time.sleep(poll)
            view = queue.get(identity)
    except KeyError as error:
        # The page's queue forgot this job (its record cap recycles finished ones) before the result was read.
        raise JobError('listen_failed', 'Listening finished but the subtitle service no longer has the result; retry') from error
    if view['state'] == 'cancelled':
        raise JobError('cancelled', 'Cancelled')
    if view['state'] != 'done':
        reason = view.get('reason') or 'failed'
        raise JobError(LISTEN_REASONS.get(reason, 'listen_failed'),
                       LISTEN_MESSAGES.get(reason) or 'Listening failed (%s)' % reason)
    return dict(editor_project=spec['editor_project'], revision=spec['revision'], **view['result'])