Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
/**
* Scene contract — the flat description the app hands the native D3D compositor so it can
* compute the composed frame itself (preview AND export) with **no POC-fixture logic**.
*
* Division of labour:
* - App (TS, this file): *serialize* the document + editor settings into this flat shape.
* Pure data mapping, no per-frame math.
* - Native (Rust, separate): owns the per-frame math (layout geometry, zoom easing, effect
* application) — it reads this description + the current time and composes.
*
* This replaces the hardcoded fixture `timeline()` (A↔B layout animation + 6s zoom schedule)
* that used to drive the compositor.
*
* Contract changes must update the mirrored Rust scene and the shared preview/export tests.
*/
import type {
Rotation3DPreset,
SpeedRegion,
WallpaperMotion,
WebcamBackgroundMode,
} from "@/components/video-editor/types";
import { DEFAULT_CROP_REGION, getZoomScale } from "@/components/video-editor/types";
import { annotationFontSizeFraction } from "@/lib/ai-edition/annotationScale";
import {
captionCuesToTextRegions,
deriveCaptionCues,
getCaptionSettings,
getCaptionTranslations,
} from "@/lib/ai-edition/captions";
import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";
import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks";
import { createId } from "@/lib/ai-edition/document/ids";
import { legacyCameraRegions, materializeLayouts } from "@/lib/ai-edition/document/layout";
import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";
import {
type PlaybackSegment,
projectRawTimelineSecToPlayback,
readSpeedRegions,
resolvePlaybackSegments,
} from "@/lib/ai-edition/document/timeline";
import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema";
import { nativeShaderEffectValue } from "@/lib/ai-edition/shaderEffects";
import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";
import { assetCameraSource } from "@/lib/ai-edition/timeline/camera";
import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration";
import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time";
import { takeProgramme } from "@/lib/ai-edition/timeline/take-programme";
import {
coalesceRegionsForRuler,
projectRegionsToSource,
} from "@/lib/ai-edition/timeline/timelineMap";
import {
MAGNIFICATION_REFERENCE_PX,
MAX_ZOOM_SCALE,
maxZoomScaleFor,
} from "@/lib/ai-edition/timeline/zoom-scale";
import {
computeCompositeLayout,
isWebcamBlockLayout,
paddedContentSize,
type RenderRect,
resolveWebcamLayoutPreset,
resolveWebcamReactiveZoom,
webcamSizeToFraction,
} from "@/lib/compositeLayout";
import type { CursorKind } from "@/lib/cursor/cursorThemes";
import { parseCssGradient, resolveLinearGradientAngle } from "@/lib/exporter/gradientParser";
import { LAYOUT_TRANSITION_SECONDS, type LayoutPreset } from "@/lib/layoutTypes";
import type { FrameTheme, RecordingFrame, WebcamAnchor } from "@/lib/projectDefaults";
import { resolveTextFontFamily } from "@/lib/textFonts";
import { classifyWallpaper } from "@/lib/wallpaper";
import type { CompositorClipInput } from "./contracts";
import { ROUNDNESS_REFERENCE_PX } from "./paramUnits";
/** Background behind the screen. Parsed from `settings.wallpaper`. */
export type SceneBackground =
| { kind: "color"; color: string } // "#rrggbb"
// linear-gradient(deg, c1, c2, …). `motion` is omitted when still, so a project without
// animation sends the same payload as before the setting existed. `offsets` (0..1, one per
// stop) place the stops where CSS puts them: the thumbnail draws a middle stop at its
// offset, and so must the compositor.
| {
kind: "gradient";
angleDeg: number;
stops: string[];
offsets: number[];
motion?: Exclude<WallpaperMotion, "none">;
}
// "/wallpapers/…" or a data: URL. `motion` as for the gradient: omitted when still.
| { kind: "image"; path: string; motion?: Exclude<WallpaperMotion, "none"> };
/** A timeline zoom region (from `document.zoomRanges`). Times in seconds. */
export interface SceneZoomRegion {
/** Stable id — native uses it to pair adjacent regions for connected zoom-pan. */
id: string;
startSec: number;
endSec: number;
/** Target scale (>1 zooms in). Derived from `depth` (or `customScale` when present). */
scale: number;
/** Focus point, 0..1 of the frame. */
focusX: number;
focusY: number;
/** "auto" follows cursor telemetry instead of the fixed focus point. */
focusMode: "manual" | "auto" | null;
/** The zoom's 3D camera: a fixed angle or a moving camera (`regions.rs::camera_for`).
* `null` = flat; the native side also renders an unknown value flat. */
rotation: Rotation3DPreset | null;
/** Index of the clip (within `SceneDescription.clips`) whose source time this region's
* `startSec`/`endSec` are expressed in — disambiguates clips whose source windows
* numerically overlap (same or different asset). Unset only for a region that
* `projectRegionsToSourceTime` couldn't place on any clip. */
clipIndex?: number;
/** The whole region lies on a stretch a trim removed. Its `startSec`/`endSec` are outside
* `clips[clipIndex]`'s source window on purpose, and `clipIndex` is the segment the cut
* interrupts (`cutAddressingSegmentIndex`) — the ONLY thing addressing it.
*
* Native shows it when the playhead is parked on the cut and gates it HARD on its own
* span: no ease-in / ease-out window, and no chaining with a neighbouring zoom. That gate
* is what keeps the render cut — an export never composes a frame at those source times,
* and a transition envelope would otherwise reach the kept frames beside the cut. */
/** Omitted (not `false`) when there is no trim under the region. See issue #216. */
underTrim?: boolean;
/** When true, cursor is hidden during this zoom region. */
hideCursor?: boolean;
}
/** One fragment of a shader effect region (`(document.shaderEffectRanges ?? [])`), projected onto its
* clip's source time like the other regions, with its definition. `scene.rs`
* `SceneShaderEffect` reads it; docs/shader-effects.md, B.5. */
export interface SceneShaderEffect {
startSec: number;
endSec: number;
clipIndex?: number;
/** The WHOLE pill on the programme clock (trims skipped, speed applied): the compositor's
* `ctx.region` runs across it instead of restarting at each fragment. */
programmeStartSec: number;
programmeEndSec: number;
target: "image" | "text" | "both";
contract: number;
code: string;
params: Array<{ name: string; type: "number" | "toggle" | "color" | "point" }>;
/** Every declared parameter, defaults filled in: one float, or 2 (point) or 4 (color). */
values: Record<string, number | number[]>;
}
/** A "Full Camera" timeline region (from `legacyEditor.cameraFullscreenRegions`). Times in seconds. */
export interface SceneCameraFullscreenRegion {
startSec: number;
endSec: number;
/** See `SceneZoomRegion.clipIndex`. */
clipIndex?: number;
/** See `SceneZoomRegion.underTrim`. Full-Camera needs no extra gate — its envelope is
* already contained in `[startSec, endSec]` — so this only carries the intent. */
underTrim?: boolean;
/** A cut (trim or clip boundary) split the region and this piece's start is that cut, not
* the region's own start: native skips the lead-in so the camera stays full across the
* cut. Omitted when false. */
continuesBefore?: boolean;
/** Same at the end: no lead-out at the cut. Omitted when false. */
continuesAfter?: boolean;
}
/** A speed region projected onto each clip's source time. The native compositor matches
* these against each decoded frame's SOURCE time, the same way zoom regions do — that's
* why the spans live in seconds and the underlying projection is `projectRegionsToSourceTime`.
* A region straddling a clip boundary splits into one entry per covered clip; both fragments
* carry the SAME `speed` value (the projection function only rewrites `startMs`/`endMs`/`id`,
* every other field passes through verbatim). */
export interface SceneSpeedRegion {
startSec: number;
endSec: number;
/** Playback rate multiplier (1 = unchanged). */
speed: number;
/** See `SceneZoomRegion.clipIndex`. */
clipIndex?: number;
}
/** A timeline annotation (`document.annotations`), projected onto each clip's source time the
* same way zoom and speed regions are.
*
* Until now annotations never crossed this bridge at all: `AnnotationLayer` draws them as DOM
* siblings of the preview, so they showed up in the preview and were simply absent from every
* render. Sending them here is what lets the compositor draw them into the actual output.
*
* Coordinate space, which the renderer must match exactly: `x`/`y`/`w`/`h` are fractions of the
* SCREEN layer's rect, not of the output frame — the web overlay is handed
* `layout.screenRect` as its container. And they are deliberately NOT affected by the zoom
* crop: the overlay is a sibling of the element carrying the zoom transform, so annotations
* hold still while the content zooms underneath them.
*
* The exception is `kind: "blur"`: a privacy mask that held still would let the content it
* hides slide out from under it. Native renders it on the content instead, through the zoom and
* any 3D tilt (`FrameGeometry::privacy_mask`), so its rect names what it covers at rest. The
* selection outline in the editor overlay still sits on the unzoomed rect.
*
* `space: "frame"` opts an entry out of that and measures it against the output frame instead.
* Captions set it, since a subtitle belongs to the frame the viewer sees and has to hold still
* when padding resizes the footage under it (issue #396), and so do text, image and arrow
* annotations, which are placed anywhere on the frame for the same reason. A blur never does,
* and neither does an annotation saved before frame placement: the key is omitted entirely
* there, so their payload is unchanged (see `annotations/placement.ts`). */
export interface SceneAnnotation {
id: string;
startSec: number;
endSec: number;
/** See `SceneZoomRegion.clipIndex`. */
clipIndex?: number;
/** See `SceneZoomRegion.underTrim`. Annotations need no extra gate — they are already
* drawn only while `startSec <= t < endSec` — so this only carries the intent. */
underTrim?: boolean;
kind: "text" | "image" | "figure" | "blur";
/** Which box `x`/`y`/`w`/`h` — and `text.fontSizeRel` — are fractions of. Absent means
* `"screen"`: a blur, or an annotation saved before frame placement. */
space?: "frame";
/** Rect as fractions of the box named by `space` (x, y top-left). */
x: number;
y: number;
w: number;
h: number;
/** Paint order among annotations; the compositor draws ascending. */
zIndex: number;
/** Present for `kind: "text"`. Colours are CSS strings, parsed native-side like the
* background colour is (`parse_hex`); `"transparent"` means no fill. */
text?: {
content: string;
color: string;
backgroundColor: string;
/** Font size as a FRACTION of the height of the box named by `space`, like everything
* else in this struct — multiply by that box's height in output pixels. See
* `annotationScale.ts`: the preview applies the identical product against its own box,
* so preview and render agree at any resolution. The denominator MUST follow `space`:
* measuring the rect against the frame while sizing the text off the screen rect would
* hold a caption still and still shrink its text with the padding slider. */
fontSizeRel: number;
fontFamily: string;
fontWeight: "normal" | "bold";
fontStyle: "normal" | "italic";
textDecoration: "none" | "underline";
textAlign: "left" | "center" | "right";
/** Which edge of the drawn text block is pinned to the box. Omitted for
* annotations, which keep the historical centring — so their payload does not
* change shape at all. Captions send it because a centred block moves BOTH its
* edges as it grows, which made a subtitle drift every time its text wrapped. */
verticalAlign?: "top" | "center" | "bottom";
animation: string | null;
};
/** Present for `kind: "image"` — the authored `imageContent` (path or data URI). */
imagePath?: string;
/** Present for `kind: "figure"`. */
figure?: {
direction:
| "up"
| "down"
| "left"
| "right"
| "up-right"
| "up-left"
| "down-right"
| "down-left";
color: string;
strokeWidth: number;
};
/** Present for `kind: "blur"`. `freehandPoints` are fractions of the screen layer, same
* space as the rect. */
blur?: {
style: "blur" | "mosaic";
shape: "rectangle" | "oval" | "freehand";
color: "white" | "black";
intensity: number;
blockSize: number;
freehandPoints?: Array<{ x: number; y: number }>;
};
}
/** Normalized rect in 0..1 of the output frame (x, y top-left; width, height). */
export interface SceneRect {
x: number;
y: number;
width: number;
height: number;
}
/** Webcam layout, from the editor settings. */
export interface SceneLayout {
preset: LayoutPreset;
/**
* Webcam size as a fraction (0..1) of the canvas reference dimension, derived from
* `webcamSizeToFraction(settings.webcamSizePreset)`. Matches the web's canonical
* composite-layout helper (0.10 = small, 0.25 ≈ default, 0.50 = max). The native
* compositor must consume this directly as a fraction of the reference dimension —
* an earlier revision emitted `settings.webcamSizePreset / 16.7` (a multiplier
* where ~1 ≈ default PiP size); that unit was incorrect vs. the web pipeline and has
* been replaced here. If you are touching the Rust consumer of this field, treat the
* incoming value as a 0..1 fraction of the canvas reference dimension, NOT as a
* size-multiplier.
*/
webcamSize: number;
/**
* The mask shape the layout actually RESOLVED to, not the raw `webcamMaskShape`
* setting. Only picture-in-picture honours the setting; the block layouts cut a
* rectangle out of the welded block whatever the user last picked in the shape
* picker (which the UI hides there). Shipping the raw setting is what let a circle
* chosen in PiP follow the user into "Side by side" and round the camera off into a
* disc — see `computeCompositeLayout`, which is the one place that decides.
*/
webcamShape: "rectangle" | "circle" | "square" | "rounded";
webcamMirror: boolean;
/** Normalized position (0..1) for the webcam centre, or null to use the preset default. */
webcamPosition: { cx: number; cy: number } | null;
/** Webcam shrinks while a zoom region is active. */
webcamReactiveZoom: boolean;
/**
* Picture-in-picture: the corner or edge middle the camera is anchored to, at a constant
* margin from the border (`computeCompositeLayout`). The zoom-time shrink keeps that anchor
* fixed, so a corner camera keeps its margin to both edges.
*/
webcamAnchor: WebcamAnchor;
/** User-authored webcam framing, as fractions of the camera source. */
webcamCrop: { x: number; y: number; width: number; height: number };
/**
* Webcam rect resolved by the app (= `computeCompositeLayout(...).webcamRect`, pixels
* → fractions of the output frame, parity EXACTE entre preview et natif). When set, the
* native compositor consumes it directly for the base webcam placement instead of its
* own hardcoded PiP math; it still applies `webcamSize` (slider) + reactive-zoom scaling
* + Full Camera lerp on top. Absent (older payloads / passthrough) → the native side
* falls back to its legacy `preset_placements` for the affected preset.
*/
webcamRect?: SceneRect | null;
/**
* Screen rect resolved by the app (= `computeCompositeLayout(...).screenRect`, same
* fractions-of-the-output-frame convention as `webcamRect`). Already padded and
* already at the crop's aspect ratio, so the native compositor must consume it as-is
* — no `padding_scale`, no aspect fit. Without it the native side kept its hardcoded
* `preset_placements` screen box while honouring the app's camera box, which is what
* pushed the side-by-side camera past the edge of the scene.
*/
screenRect?: SceneRect | null;
/**
* Should the screen FILL its box, cropping the overflow (`object-fit: cover`), rather
* than fit inside it? True for the block layouts, whose screen box is an arbitrary-ratio
* slot. This is `computeCompositeLayout(...).screenCover`, which `frameRenderer` already
* honours on the web path; shipping it here is what lets the native compositor agree.
* Without it native stretched the source to fill the slot — most visible on a cropped
* clip, the crop pushing the source aspect further from the slot's.
*
* The slot also MASKS the screen: a zoom or a 3D tilt stays inside it, cropped, instead of
* spilling over the camera (`ScreenMask` in `frame_geometry.rs`).
*/
screenCover?: boolean;
/**
* The format is filled (`isFormatFillActive`): the screen box is the whole padded area,
* `screenCover` is on, and the window the cover cuts out follows the smoothed cursor
* instead of staying centred. Omitted when off, so other payloads are unchanged.
*/
screenFollow?: true;
/**
* One resolved layout per visible clip, index-aligned with `SceneDescription.clips`
* and `cropByClip`. The scalar fields above are the FIRST clip's entry (fallback for
* older payloads and the value native uses before a clip is active).
*
* Per clip because the screen source's SHAPE is per clip — a clip is a screen
* recording plus an optional camera and audio, and nothing forces two clips to share
* a recording size or ratio. Cropping is one more way that shape varies: a 16:9
* recording cropped to 9:16 must lay out exactly like one recorded natively in 9:16.
* Distinct from the scene's aspect ratio, which is global.
*
* Resolving one layout for the whole scene was therefore already wrong for a document
* mixing recording resolutions; cropping only made it reachable in a single document.
*/
layoutByClip?: Array<ResolvedClipLayout | null>;
/**
* Corner radius of the screen box, as a fraction of that box's own SHORT SIDE, when
* the preset imposes one (the block layouts frame screen and camera alike). Null →
* the native side falls back to the user's Roundness slider.
*
* A fraction, not pixels — see `roundnessFrac` for why the whole contract works this
* way. The denominator is the box rather than the frame on purpose: a corner radius
* belongs to the thing it rounds, so the rounding stays put when the box is resized.
*/
screenRadiusFrac?: number | null;
/**
* Corner radius of the webcam box, as a fraction of ITS own short side — same rule as
* `screenRadiusFrac`, resolved by the same `computeCompositeLayout` call, so "the
* block frames screen and camera alike" can actually hold. It could not before: the
* screen took the app's radius while the camera kept a second, independent table in
* Rust (`min * 0.5 | 0.3 | 0.12`, unclamped, keyed off the raw mask shape), so the two
* halves of one welded block were rounded by two formulas that could not agree.
*/
webcamRadiusFrac?: number | null;
}
/**
* The shape-dependent half of a layout, resolved for one clip. See `layoutByClip`.
*
* Radii are fractions of their own box's short side, exactly as the scalar
* `screenRadiusFrac`/`webcamRadiusFrac` above — no length crosses this contract in
* pixels, per-clip or not.
*/
export interface ResolvedClipLayout {
screenRect: SceneRect;
webcamRect: SceneRect | null;
screenRadiusFrac: number | null;
webcamRadiusFrac: number | null;
webcamShape: "rectangle" | "circle" | "square" | "rounded";
screenCover: boolean;
}
export interface SceneLayoutRegion {
/** Index of the clip whose SOURCE clock addresses this passage. */
clipIndex: number;
/** Source seconds visible after trims; these can be narrower than the original passage. */
startSec: number;
endSec: number;
preset: LayoutPreset;
layout: ResolvedClipLayout;
webcamReactiveZoom: boolean;
/** Previous resolved layout, preset and reactive flag: supplied together or all omitted. */
fromLayout?: ResolvedClipLayout;
fromPreset?: LayoutPreset;
fromWebcamReactiveZoom?: boolean;
/** Last contained-layout exit in source seconds. An ongoing zoom keeps that viewport until it settles. */
keepScreenClipFromSec?: number;
/** Original passage start in source seconds; trimming must not restart its animation. */
transitionStartSec: number;
/** Duration in source seconds, or 0 without fromLayout (including legacy-camera neighbours). */
transitionSec: number;
}
/** Frame-styling effects, from the editor settings. */
export interface SceneEffects {
/** 0..1 extra inset of the screen (padding). */
padding: number;
/** 0..1 blur of the wallpaper, 0 = sharp (`background_blur_steps` in frame_geometry.rs). */
blur: number;
/** 0..1 drop-shadow strength. */
shadow: number;
/**
* Roundness slider, as a fraction of a fixed 1080 px reference (`ROUNDNESS_REFERENCE_PX`).
*
* The native side multiplies it by the SCREEN's unit: the short side of the frame the
* screen would have in an output of its own ratio (`screen_unit_px`). So 24 means 24 px
* on a 1080p export whose format matches the recording, and the same share of the screen
* everywhere else. It used to be divided by the output's short side, which follows the
* source: a 4K take got corners half as round as a 1080p one.
*
* Every length crossing this contract is a fraction, never a pixel count, and that is
* load-bearing rather than stylistic: the native compositor rasterises the preview
* into a small contain-fitted frame and the export at full output size, so a pixel
* means two different things on the two sides of the boundary. Absolute values used to
* cross it and silently meant "render-target pixels" — which is why the preview drew
* the PiP circle as a shrunken blob while the export was correct, and why a 4K export
* got a proportionally weaker shadow than a 1080p one. A fraction has no unit to get
* wrong; the native side multiplies by whatever its reference measures right now.
*
* The slider itself stays in pixels for the user — the division happens here, once.
*
* Under a frame, the native side reads the slider's POSITION back from it (× the
* reference ÷ `ROUNDNESS_SLIDER_MAX_PX`) and maps it onto the range that
* frame wears well, in the frame's own unit — the same corner on every clip ratio.
*/
roundnessFrac: number;
/** 0..1 motion blur. */
motionBlur: number;
/**
* The frame drawn around the recording. Omitted for "none", like `webcamEffect`: the
* Rust side defaults the field (`SceneFrame::None`), and a scene without a frame then
* serializes exactly as it did before the field existed.
*/
frame?: Exclude<RecordingFrame, "none">;
/**
* Light or dark, for whichever frame is on — the window chrome and the three modelled
* devices alike. Omitted at "light", the Rust default.
*/
frameTheme?: Exclude<FrameTheme, "light">;
/**
* Defocus a 3D-tilted screen by its depth, sharp at the zoom focus. Inert on flat zooms:
* the native side only reads it where it draws a tilted plane.
*/
depthOfField: boolean;
}
/** Cursor rendering, from the editor settings. */
export interface SceneCursor {
show: boolean;
autoHide: boolean;
/** Direct scale (1 = default). */
size: number;
smoothing: number;
/** 0..1. */
motionBlur: number;
clickBounce: number;
/**
* Modelled 3D arrow (`scene.rs` `SceneCursor::model3d`, compositor mode 15). Only drawn for
* the default theme's arrow; any other cursor keeps its flat sprite.
*/
model3d: boolean;
/** The kinds drawn with the arrow sprite (`CURSOR_KINDS`, resolved app-side, see
* compositorViewService). */
asArrow: CursorKind[];
/** Each click gives the screen an impact, under every camera (`scene.rs`
* `SceneCursor::click_impact`). Omitted (not `false`) when off. */
clickImpact?: true;
clipToBounds: boolean;
/** Cursor theme id (sprite set). */
theme: string;
}
/** Everything native needs to compose the scene, serialized from one document. */
export interface SceneDescription {
/** Ordered clips (multiclip) with source trims — same shape the export already uses. */
clips: CompositorClipInput[];
layout: SceneLayout;
effects: SceneEffects;
background: SceneBackground;
zoomRegions: SceneZoomRegion[];
/**
* Annotations projected onto each clip's source time, ascending by `zIndex` so the
* compositor can draw them in order without re-sorting. Empty when none set.
*/
annotations: SceneAnnotation[];
/**
* "Full Camera" regions projected onto each clip's source time (one entry per
* source-time span after `projectRegionsToSourceTime`). Empty when none set.
*/
cameraFullscreenRegions: SceneCameraFullscreenRegion[];
/** Agent-written shader effects, omitted when there are none. */
shaderEffects?: SceneShaderEffect[];
layoutRegions?: SceneLayoutRegion[];
/**
* Speed regions projected onto each clip's source time (one entry per
* source-time span after `projectRegionsToSourceTime`). Empty when none set.
*
* ponytail: speed regions today live at `document.legacyEditor.speedRegions`
* — the new `timeline.speedRanges` schema field is `z.array(rangeSchema)` and
* `rangeSchema` is `{startSec, endSec, reason}`, which does NOT carry a `speed`
* value (see migrate.ts comment "speedRegions stay on the legacy editor envelope
* — axcut's rangeSchema doesn't carry a speed value, and Phase 1 timeline rewrite
* is when speed becomes a first-class timeline concept"). We read from
* `legacyEditor.speedRegions` (where the `speed` multiplier actually lives) so
* the native compositor gets a populated `speed`, matching the legacy web
* exporter's read site. When the schema rewrite lands, swap the source to
* `document.timeline.speedRanges` and keep the same projection call.
*/
speedRegions: SceneSpeedRegion[];
cursor: SceneCursor;
/**
* Audio finishing: the output trim, applied identically by the preview and by
* `finish_audio` (Rust).
*
* One field, and it takes some resisting to keep it that way. The preview plays the
* untouched SOURCE file, seeked; the export runs on the assembled timeline — trimmed,
* speed-adjusted, concatenated. A linear gain is the only operation that means the same
* thing on both. That is why the loudness normalisation is not a field here: it is a gain
* per voice FILE, measured over the whole file by the compositor, which the export and
* the preview both ask for (`loudness_gain_db`). Anything with memory diverges across
* cuts — the export's peak limiter is the one such stage, and it runs at export only —
* and even a plain delay diverges, because the preview would apply it in source seconds
* while this applies it in timeline seconds. A sync offset shipped here and was removed
* for exactly that.
*/
audio: {
gainDb: number;
};
/**
* Timeline audio tracks (issue #350), mixed over the assembled programme by
* `audio::mix_external_tracks`. One entry per contiguous stretch: a track split by a
* clip boundary contributes one entry per fragment (each picking the source up where
* the last left off), and a looping track one entry per repeat.
*
* `startSec` is the head on the trim-COMPRESSED output programme: the track's raw
* timeline head projected through the trims via `projectRawTimelineSecToPlayback`, so
* a cut ahead of the track pulls it earlier by the removed duration (exactly as the
* preview already plays it). Exact for trims; speed regions stay an approximation.
* `trimEndSec` is always concrete — the compositor preallocates the decode window
* from it — so it is resolved from the span and the source duration.
*/
audioTracks: Array<{
path: string;
startSec: number;
gainDb: number;
trimStartSec: number;
trimEndSec: number;
/** Ramp lengths at the entry's own edges, in seconds. A split or looping
* track carries them only on the pieces that touch the track's real
* start and end, so it fades once rather than at every cut or repeat. */
fadeInSec: number;
fadeOutSec: number;
/** A voiceover is voice: the export levels it like the recording's own audio. */
kind: AxcutAudioTrack["kind"];
}>;
/**
* The "mouse clicks" toggle (see `src/lib/ai-edition/clickSound.ts`): the recorded clicks as
* hits placed on the OUTPUT programme, summed by `audio::mix_click_hits`. Omitted when the
* sound is off, the take has no clicks, or the cues have not been read yet.
*
* Not entries of `audioTracks`. An imported track is placed and never stretched, so a click
* bed baked into one file falls behind every click past a speed region; these hits go through
* the same `projectRawTimelineSecToPlayback` imported audio tracks are laid out with.
*/
clickSound?: {
downPath: string;
upPath: string;
hits: Array<{ timeSec: number; gain: number; release: boolean }>;
};
/**
* Per-clip screen crop (fractions of the frame), or null for the identity
* (full-frame) crop. One entry per clip in the same order as `clips`, so a
* clip that owns its own cropRegion is rendered with that crop and a clip
* without one stays at the full frame. Replaces the old single global
* `crop` field, which lost per-clip crops on multi-clip documents.
*/
cropByClip: Array<{ x: number; y: number; width: number; height: number } | null>;
/** Output frame. `fps` null = use the first clip's source fps. */
output: { width: number; height: number; fps: number | null };
/** Webcam background effect. Omitted when the mode is "none". */
webcamEffect?: SceneWebcamEffect;
}
/**
* The webcam background effect, as the compositor needs it.
*
* Carries the MODE and its parameters only — never pixels. The per-pixel subject mask is
* produced by the segmentation running in the compositor process and reaches the shader as a
* texture. An earlier design baked the composite here and shipped it as a video track; the codec
* could not carry alpha, and preview and export drifted apart.
*/
export interface SceneWebcamEffect {
mode: WebcamBackgroundMode;
/** 0..1, only meaningful for "blur". */
blurIntensity: number;
/** Background behind the subject for "custom", parsed like `settings.wallpaper`. */
background?: SceneBackground;
}
/** Parse the settings wallpaper string into the discriminated SceneBackground union. */
function parseWallpaper(wallpaper: string) {
if (wallpaper.startsWith("#")) {
return { kind: "color", color: wallpaper } as const;
}
if (wallpaper.startsWith("linear-gradient(")) {
// parseCssGradient handles nested parens (rgba()/hsl() stops) and the
// "to bottom right" keyword directions that a flat comma split drops.
// It's anchored on a trailing ")", so it rejects strings this branch
// accepts (a trailing space is enough) — stay a gradient when it does,
// rather than falling through and handing the native side a CSS string
// as an image path.
const parsed = parseCssGradient(wallpaper);
return {
kind: "gradient",
angleDeg: resolveLinearGradientAngle(parsed?.descriptor ?? null),
stops: parsed?.stops.map((stop) => stop.color) ?? [],
offsets: parsed?.stops.map((stop) => stop.offset) ?? [],
} as const;
}
return { kind: "image", path: wallpaper } as const;
}
/**
* True when `settings.wallpaperMotion` has something to move: a linear gradient the compositor
* draws itself, or an image it loads. A colour has nothing to move, and any other string (a
* radial gradient, a colour function) reaches the compositor as an image path that does not
* load. The motion control asks this, not its own guess, so it is shown exactly when the export
* would move.
*/
export function wallpaperAcceptsMotion(wallpaper: string): boolean {
return (
parseWallpaper(wallpaper).kind === "gradient" || classifyWallpaper(wallpaper).kind === "image"
);
}
/** The screen background with the chosen motion attached, when there is one to attach. */
function sceneBackground(wallpaper: string, motion: WallpaperMotion): SceneBackground {
const background = parseWallpaper(wallpaper);
if (motion === "none" || background.kind === "color" || !wallpaperAcceptsMotion(wallpaper)) {
return background;
}
return { ...background, motion };
}
/**
* The ONE clip list every native-facing consumer must build from — trim-narrowed
* (`resolvePlaybackSegments`, so word-level cuts from the transcript editor actually reach
* native instead of only affecting the transcript panel's own strikethrough), sorted, and
* filtered to clips whose asset has a resolvable path. Shared by `buildSceneDescription`
* below, `ExportDialog.tsx`'s `buildNativeClipList` (native MP4 export), and
* `NativeCompositorOverlay.tsx`'s `nativeClips` (live preview) — previously these three each
* hand-rolled their own sort+filter, acknowledged as needing to be "kept in lock-step".
*/
/** Whether a clip's media can actually be read — the one rule that decides
* which clips make it into the programme. Shared with the audio-track
* projection, which must count exactly the clips the programme is built from
* or every track after a relinked-away clip lands late. */
function clipAssetIsResolvable(
clip: { assetId: string },
assetById: Map<string, { originalPath?: string }>,
): boolean {
return Boolean(assetById.get(clip.assetId)?.originalPath);
}
/**
* Returns `PlaybackSegment[]`, not `AxcutClip[]`: a held segment carries `heldSec`, and
* widening it away here is what kept the pause from ever reaching the compositor. Every
*/
export function resolveVisibleClips(document: AxcutDocument): PlaybackSegment[] {
const assetById = new Map(document.assets.map((a) => [a.id, a]));
return resolvePlaybackSegments(document.timeline.clips, document.timeline.trimRanges)
.sort((a, b) => a.timelineStartSec - b.timelineStartSec)
.filter((clip) => clipAssetIsResolvable(clip, assetById));
}
/**
* The clip list a native export renders: the trim-narrowed visible clips, mapped onto the native
* multiclip contract. `openscreen export` and getFrames share it, beside `buildSceneDescription`
* so that the clip stream and the scene agree.
* ponytail: the Export dialog keeps an identical copy, which its tests' module mocks rely on.
*/
export function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] {
const assetById = new Map(document.assets.map((a) => [a.id, a]));
return resolveVisibleClips(document).flatMap((clip) => {
const asset = assetById.get(clip.assetId);
if (!asset?.originalPath) {
return [];
}
const camera = assetCameraSource(asset);
// sourceEndSec is optional in the schema (unknown until probed) — fall back through
// the single canonical precedence used by every consumer (clip.probe → asset.duration
// → timeline-length guess). See `resolveClipSourceEndSec` for the full order.
const sourceEndSec = resolveClipSourceEndSec(clip, asset);
// ponytail: `hasAudio` stays optimistic for the same reason as in
// `buildSceneDescription` — nothing populates `asset.audio` yet, and the
// native side degrades cleanly on a stream-less file.
return [
{
screenPath: asset.originalPath,
webcamPath: camera.path,
sourceStartSec: clip.sourceStartSec,
sourceEndSec,
webcamOffsetSec: camera.offsetSec,
hasAudio: true,
},
];
});
}
/** A clip's screen source size in pixels: its recording × its crop (see `screenSourceSizeOf`). */
function screenSourceSize(
video: { width?: number; height?: number } | null | undefined,
crop: { width: number; height: number } | null | undefined,
) {
return {
width: Math.max(1, Math.round((video?.width || 1920) * (crop?.width ?? 1))),
height: Math.max(1, Math.round((video?.height || 1080) * (crop?.height ?? 1))),
};
}
/**
* The deepest zoom a clip takes before its recording blurs: `maxZoomScaleFor` its
* magnification at rest, frame pixels per source pixel, read on its screen box. The larger
* axis, so a slot that covers its box (block layouts) counts the pixels it really blows up.
*
* Measured on the default export, a frame with a 1080 px short side
* (`MAGNIFICATION_REFERENCE_PX`), not on `output`: `output` follows the largest clip, crop
* included, so a cropped take would look small there and be upscaled at export. A 1080p take
* at 50 % padding sits at 0.8 (limit 2.5×), a 2160p take at 0.4 (5×), half of a 1080p take
* at 1.6 (1.25×).
*/
function zoomScaleLimitOf(
rect: SceneRect | null,
output: { width: number; height: number },
source: { width: number; height: number },
): number {
if (!rect) return MAX_ZOOM_SCALE;
const toReference =
MAGNIFICATION_REFERENCE_PX / Math.max(1, Math.min(output.width, output.height));
const rest =
toReference *
Math.max(
(rect.width * output.width) / source.width,
(rect.height * output.height) / source.height,
);
return maxZoomScaleFor(rest);
}
/**
* The deepest zoom the region `regionId` can take, for the inspector: the same bound the
* scene applies, read on the clip the region belongs to. `MAX_ZOOM_SCALE` when the region
* is not on screen.
*/
export function zoomScaleLimit(document: AxcutDocument, regionId: string): number {
const region = (document.zoomRanges ?? []).find((z) => z.id === regionId);
if (!region) return MAX_ZOOM_SCALE;
const scene = buildSceneDescription(document);
const visibleClips = resolveVisibleClips(document);
// A region straddling a cut becomes one piece per clip, each with its own id: the same
// projection as the scene, and the tightest of their limits.
const pieces = projectRegionsToSource([region], visibleClips, document.timeline.clips, () => "");
let limit = MAX_ZOOM_SCALE;
for (const { clipIndex = 0 } of pieces) {
const clip = visibleClips[clipIndex];
if (!clip) continue;
const video = document.assets.find((a) => a.id === clip.assetId)?.video;
limit = Math.min(
limit,
zoomScaleLimitOf(
scene.layout.layoutByClip?.[clipIndex]?.screenRect ?? scene.layout.screenRect ?? null,
scene.output,
screenSourceSize(video, scene.cropByClip[clipIndex]),
),
);
}
return limit;
}
/**
* Where the footage under the annotation `annotationId` sits in the output frame, as fractions
* of it: the layout of the clip the annotation is on, exactly as the scene lays it out. What the
* inspector converts through when an annotation changes box (`annotations/placement.ts`). `null`
* when the annotation is on no clip (its own clip trimmed away, say): another clip's layout may
* differ, and converting through it would move the annotation.
*/
export function annotationFootageRect(
document: AxcutDocument,
annotationId: string,
): SceneRect | null {
const region = (document.annotations ?? []).find((a) => a.id === annotationId);
if (!region) return null;
const [piece] = projectRegionsToSource(
[region],
resolveVisibleClips(document),
document.timeline.clips,
() => "",
);
if (piece?.clipIndex === undefined) return null;
const scene = buildSceneDescription(document);
return (
scene.layout.layoutByClip?.[piece.clipIndex]?.screenRect ?? scene.layout.screenRect ?? null
);
}
type Size = { width: number; height: number };
/**
* The size the camera box is laid out from: the camera's own proportions, cropped.
*
* The order matters. `camera.width/height` comes FIRST because it is the only source
* every caller has: it is in the document, so the export dialog and the CLI runner read
* it exactly as the preview does. `probed` is second, as a fresher-than-disk override for
* the window before the backfill has written the dimensions: it is what a mounted <video>
* just reported, and only the preview can ever supply it. The hardcoded 4:3 is last and
* is only reached for a document that predates the field, opened with no camera mounted.
*
* That ordering is the fix: the box used to depend on WHO was asking rather than on what
* was recorded, so a 16:9 camera was framed 16:9 in the preview and 4:3 in the export.
* The scene and the editor's drag box both read the same answer now.
*/
export function webcamBoxSourceSize(
camera: { width?: number; height?: number } | null | undefined,
probed: Size | null,
crop: Size,
): Size {
const source =
camera?.width && camera?.height
? { width: camera.width, height: camera.height }
: (probed ?? { width: 960, height: 720 });
return {
width: Math.max(1, Math.round(source.width * crop.width)),
height: Math.max(1, Math.round(source.height * crop.height)),
};
}
/** Serialize a document into a {@link SceneDescription}. No per-frame math; the one side effect is
* handing the click cues to the preview, which has no document of its own to read them from. */
export function buildSceneDescription(
document: AxcutDocument,
webcamSourceSize: { width: number; height: number } | null = null,
): SceneDescription {
const settings = getEditorSettings(document);
const assetById = new Map(document.assets.map((a) => [a.id, a]));
// Timeline audio tracks (issue #350) → the compositor's mix list.
//
// Each STORED track is one clip-anchored fragment, already carrying its own
// advanced `offsetMs`, so a fragment maps to one contiguous decode window and
// the pieces of a split take play as one continuous take. Project each head
// onto the trim-compressed programme the mixer overlays on: the head is
// stored in RAW ruler seconds, and passing it verbatim delayed every track by
// the total trim duration ahead of it (issue #350).
//
// Projected onto the same clips the programme is assembled from —
// `resolveVisibleClips` drops clips whose asset has no resolvable
// `originalPath`, and a projection that counted a relinked-away clip the
// programme does not would land every following track past the real end.
// `projectRawTimelineSecToPlayback` subtracts the trims itself, so it needs
// the RAW clips behind that filter, not the already-compressed segments.
const projectedClips = document.timeline.clips.filter((clip) =>
clipAssetIsResolvable(clip, assetById),
);
// Speed regions on the RAW ruler. The programme these tracks mix onto has
// already been time-stretched by them (`stretch_clip_pcm_by_speed` runs before
// `mix_external_tracks`), so a projection blind to speed lands every track
// after a speed region at the wrong second. The tracks themselves are never
// stretched — a voiceover should not chipmunk because the video under it was
// sped up.
const rawSpeedRegions = readSpeedRegions(document);
// The one removed set, hoisted out of the map: every voiceover asks it the same
// question, and it does not depend on the track.
// Placed once: the projection below counts them, so a track after a pause lands where
const removed = removedRawSpans(projectedClips, document.timeline.trimRanges);
// Every take is mixed ONCE, from its head fragment, as its whole pill — never per stored
// fragment. The document keeps one fragment per clip a take covers: a voiceover walked per
// fragment would play on top of itself (`overlay_track_pcm` sums with `+=`), and a bed laid
// per fragment, each at its projected head with its ruler-advanced offset, went silent at the
// end of a slowed clip and came back late (see `takesByHeadFragment`).
const takes = takesByHeadFragment(document.audioTracks);
// Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.
// Synchronous: the cues were read from the take's sidecar when the sound was switched on.
// Its row lives under "Show cursor", like the click impact: a hidden cursor makes no sound.
const clickSound = clickSoundForDocument(document, {
clickSound: settings.cursorShow && settings.cursor.clickSound,
clickSoundGainDb: settings.cursor.clickSoundGainDb,
});
const audioTracks = document.audioTracks.flatMap((fragment) => {
const track = takes.get(fragment.id);
if (!track || track.muted) return [];
const asset = assetById.get(track.assetId);
if (!asset?.originalPath) return [];
const sourceDurationSec = asset.durationSec ?? track.durationSec;
const offsetSec = track.offsetMs / 1000;
const startSec = projectRawTimelineSecToPlayback(
projectedClips,
document.timeline.trimRanges,
track.startMs / 1000,
rawSpeedRegions,
);
// Length is measured WITHOUT speed, position WITH it — the two do different
// things to a track and must not be conflated.
//
// A trim REMOVES timeline: a track inside removed time has nowhere left to
// be (zero length, dropped), and one crossing a cut loses what the cut took.
// A speed region only COMPRESSES: the track still holds all its audio and
// still plays at 1x, so speeding the video up must not quietly cut the
// narration short. It changes where the track STARTS, because the programme
// ahead of it got shorter, and nothing else.
const trimmedSpanSec =
projectRawTimelineSecToPlayback(
projectedClips,
document.timeline.trimRanges,
track.endMs / 1000,
) -
projectRawTimelineSecToPlayback(
projectedClips,
document.timeline.trimRanges,
track.startMs / 1000,
);
const spanSec = trimmedSpanSec;
if (spanSec <= 0) return [];
const base = {
path: asset.originalPath,
gainDb: track.gainDb,
fadeInSec: track.fadeInMs / 1000,
fadeOutSec: track.fadeOutMs / 1000,
kind: track.kind,
};
// The window the file has left after the offset. Without a probed duration
// there is nothing to loop over and nothing to cap the tail with, so the
// span itself is the window — the mixer stops at the real end of the file.
const windowSec = sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : spanSec;
if (windowSec <= 0) return [];
// A cut under a VOICEOVER removes the words that were said there, not the tail of
// the take (issue #560). The transcript pane strikes those words through; if the
// mix went on playing them, shifted earlier, the red would be a lie.
//
// Music deliberately keeps the branch below: a bed plays through a cut and ends
// early, because slicing it at every edit is a musical regression, and a bed has no
// words whose redness has to be true. A LOOPING voiceover keeps it too — step 6 of
// #560 refuses that combination outright, and inventing semantics for something
// about to be banned would be the worse answer.
if (track.kind === "voiceover" && !track.loop) {
const rawSpanSec = Math.max(0, track.endMs / 1000 - track.startMs / 1000);
// Unprobed assets have no real duration to cap with; the RAW span is how much
// file the take covers, which is the honest fallback once the cuts are taken out.
const voWindowSec =
sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : rawSpanSec;
const kept = takeProgramme(track, removed)
.filter((piece) => piece.kind === "play")
.map((piece) => ({
...base,
startSec: projectRawTimelineSecToPlayback(
projectedClips,
document.timeline.trimRanges,
piece.rawStartSec,
rawSpeedRegions,
),
trimStartSec: piece.sourceStartSec,
trimEndSec: Math.min(offsetSec + voWindowSec, piece.sourceEndSec),
}))
.filter((entry) => entry.trimEndSec > entry.trimStartSec);
// The fades belong to the TAKE's edges, not to every piece a cut left behind.
return kept.map((entry, i) => {
// A piece plays at 1x, but its head and the next one's are projected through the
// speed regions: a sped-up stretch between them leaves the piece longer than the
// output gap, so it would still be playing when the next one starts (#1283). It
// stops where the next piece begins.
const next = kept[i + 1];
const trimEndSec = next
? Math.min(entry.trimEndSec, entry.trimStartSec + next.startSec - entry.startSec)
: entry.trimEndSec;
return {
...entry,
trimEndSec,
fadeInSec: i === 0 ? base.fadeInSec : 0,
fadeOutSec: i === kept.length - 1 ? base.fadeOutSec : 0,
};
});
}
if (!track.loop) {
return [
{
...base,
startSec,
trimStartSec: offsetSec,
// Always concrete: the compositor preallocates its decode window
// from it. Whichever runs out first — the span or the file.
trimEndSec: offsetSec + Math.min(windowSec, spanSec),
},
];
}
// A looping track is one mix entry per repeat: the mixer overlays entries
// independently, so the repeats are just more of them. The last one is cut
// short wherever the span ends.
const entries = [];
for (let played = 0; played < spanSec && entries.length < 1000; played += windowSec) {
const thisSec = Math.min(windowSec, spanSec - played);
entries.push({
...base,
startSec: startSec + played,
trimStartSec: offsetSec,
trimEndSec: offsetSec + thisSec,
// The fades belong to the track's edges, not to every repeat.
fadeInSec: played === 0 ? base.fadeInSec : 0,
fadeOutSec: played + thisSec >= spanSec ? base.fadeOutSec : 0,
});
}
return entries;
});
const visibleClips = resolveVisibleClips(document);
const clips: CompositorClipInput[] = visibleClips.flatMap((clip) => {
const asset = assetById.get(clip.assetId);
if (!asset?.originalPath) return [];
const camera = assetCameraSource(asset);
// ponytail: `asset.audio` exists in the schema but the probe pipeline never
// populates it, so there is no per-asset "is there a track?" signal to read
// yet. Every consumer downstream degrades on a stream-less file (audio.rs
// returns Ok(None)), so this stays optimistic. NOT "recordings always carry
// audio" — a capture made with no mic and no system audio has no audio
// stream at all (issue #348). Swap to `Boolean(asset.audio)` the day the
// probe fills it in.
return [
{
screenPath: asset.originalPath,
webcamPath: camera.path,
sourceStartSec: clip.sourceStartSec,
sourceEndSec: resolveClipSourceEndSec(clip, asset),
webcamOffsetSec: camera.offsetSec,
hasAudio: true,
// A held segment has an empty source window and exists only for the frames it
// holds; every other clip holds nothing.
},
];
});
const cropByClip = visibleClips.map(
(clip): { x: number; y: number; width: number; height: number } | null => {
const cropRegion = clip.cropRegion;
if (!cropRegion) return null;
if (
cropRegion.x === DEFAULT_CROP_REGION.x &&
cropRegion.y === DEFAULT_CROP_REGION.y &&
cropRegion.width === DEFAULT_CROP_REGION.width &&
cropRegion.height === DEFAULT_CROP_REGION.height
) {
return null;
}
return {
x: cropRegion.x,
y: cropRegion.y,
width: cropRegion.width,
height: cropRegion.height,
};
},
);
// Zoom + Full Camera + speed regions are authored in RAW virtual (timeline) ms
// in the document — the ruler where trims still occupy their space — but the
// compositor matches them against each frame's SOURCE time. `projectRegionsToSource`
// bridges the two through `timelineMap`: it resolves each region's RAW coordinate
// against every visible segment's OWN raw extent (via `document.timeline.clips`,
// the un-compressed layout) and maps the overlap to that segment's source range,
// tagging it with the segment's `clipIndex` in `visibleClips` (= the order of
// `Scene.clips`). A region whose source range a trim splits across two kept
// segments yields one entry per segment.
//
// BUG corrigé : ces projections utilisaient `visibleClips` (COMPRESSÉ, trims retirés)
// à la fois pour le recouvrement ET la source, alors que les régions sont posées en
// coordonnées RAW. Dès qu'un trim retirait Δs avant une région, la coordonnée RAW
// dépassait la position compressée de Δ → la région se déclenchait Δ trop tôt (offset
// visible en preview ET au rendu). On passe désormais le layout RAW pour le mapping
// raw→source et on ne garde `visibleClips` que pour l'ordre/`clipIndex`.
const projectedZoomRegions = projectRegionsToSource(
document.zoomRanges ?? [],
visibleClips,
document.timeline.clips,
() => createId("zoom"),
);
// Same raw→source projection as the zoom regions above, for the same reason: annotations are
// authored in RAW document time and the compositor matches each frame's SOURCE time.
//
// Captions join the annotations here rather than getting a bridge of their own: they are text
// with the same time base drawn through the same rasterizer, so the compositor already knows
// how to draw them. Skipping this would have left them visible in the DOM
// overlay and absent from the composited pixels — the exact preview/render gap the annotation
// work above closed. They are derived on the fly and never stored (see lib/ai-edition/captions).
//
// The reference box travels with each region: every caption carries `space: "frame"`, so the
// compositor measures it against the output frame. Subtitles have to sit where the viewer's
// frame ends, not where the footage does, or they slide inward the moment padding shrinks the
// screen rect (issue #396). Text, image and arrow annotations carry the same key for the same
// reason; a privacy blur never does (see `annotations/placement.ts`).
//
// The output aspect is what decides the caption column and the default inset (a
// caption 5% off the bottom of a 16:9 export sits under the platform's own chrome
// on a 9:16 one). It comes off `pickOutputDims`, hoisted above the webcam block that
// used to own it so there is exactly ONE caller — preview and export cannot pick a
// different column from each other.
const outputDims = pickOutputDims(document, settings.aspectRatio);
const captionAspect = outputDims.height > 0 ? outputDims.width / outputDims.height : 16 / 9;
const captionSettings = getCaptionSettings(document, captionAspect);
const captionRegions = captionCuesToTextRegions(
deriveCaptionCues(document, captionSettings, getCaptionTranslations(document)),
captionSettings,
captionAspect,
);
const projectedAnnotations = projectRegionsToSource(
[
...(document.annotations ?? []),
...(captionRegions as unknown as NonNullable<AxcutDocument["annotations"]>),
],
visibleClips,
document.timeline.clips,
() => createId("ann"),
);
const projectedCameraFullscreenRegions = projectRegionsToSource(
legacyCameraRegions(document),
visibleClips,
document.timeline.clips,
() => createId("camfull"),
);
// Speed regions carry an extra `speed` field the standard `rangeSchema` does not, so we
// can't read from `document.timeline.speedRanges` today (see SceneDescription.speedRegions
// comment). The legacy web exporter reads from `legacyEditor.speedRegions`; we mirror it.
// `projectRegionsToSource` accepts any `T extends { id; startMs; endMs }` and copies
// every other field verbatim via `{...region}` — so the `speed` field passes through,
// and the splitting-across-clips semantics match zoomRegions / cameraFullscreenRegions.
const projectedSpeedRegions = projectRegionsToSource(
readSpeedRegions<SpeedRegion>(document),
visibleClips,
document.timeline.clips,
() => createId("speed"),
);
// Shader effects: projected like the zooms, and each fragment told where its WHOLE pill sits
// on the programme clock, through the same projection the exporter plays the timeline with
// (trims skipped, speed applied). A pill cut by a trim or a clip boundary keeps one progress.
const effectsById = new Map((document.shaderEffects ?? []).map((effect) => [effect.id, effect]));
const programmeSec = (rawSec: number) =>
projectRawTimelineSecToPlayback(
projectedClips,
document.timeline.trimRanges,
rawSec,
rawSpeedRegions,
);
const pillSpanById = new Map<string, { programmeStartSec: number; programmeEndSec: number }>();
for (const pill of coalesceRegionsForRuler(document.shaderEffectRanges ?? [])) {
const span = {
programmeStartSec: programmeSec(pill.start),
programmeEndSec: programmeSec(pill.end),
};
for (const id of pill.ids) pillSpanById.set(id, span);
}
const projectedShaderEffects = projectRegionsToSource(
(document.shaderEffectRanges ?? []).map((range) => ({
...range,
...pillSpanById.get(range.id),
})),
visibleClips,
document.timeline.clips,
() => createId("fxr"),
);
const sceneShaderEffects: SceneShaderEffect[] = projectedShaderEffects.flatMap((region) => {
const effect = effectsById.get(region.effectId);
if (!effect) return [];
return [
{
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
clipIndex: region.clipIndex,
programmeStartSec: region.programmeStartSec ?? region.startMs / 1000,
programmeEndSec: region.programmeEndSec ?? region.endMs / 1000,
target: region.target,
contract: effect.contract,
code: effect.code,
params: effect.params.map(({ name, type }) => ({ name, type })),
values: Object.fromEntries(
effect.params.map((param) => [
param.name,
nativeShaderEffectValue(param, region.values[param.name]),
]),
),
},
];
});
// Webcam rect, single source of truth between preview & native :
// on résout le rect AVEC LA MÊME maths que `PreviewCanvas.computeCompositeLayout` et on
// l'envoie au natif dans `layout.webcamRect` (fractions du cadre de sortie). Le natif le
// consomme tel quel (voir `compositor.rs::preset_placements` bypass). La résolution ici se
// fait sur la résolution de sortie (= taille du canvas rendu) avec les unités sources du
// premier asset visible — la même convention que `pickOutputDims` + SCREEN_SOURCE_SIZE /
// WEBCAM_SOURCE_SIZE dans PreviewCanvas — ce qui garde preview/export/natif alignés.
// (`outputDims` est résolu plus haut, avec le bloc sous-titres qui en dépend aussi.)
// ponytail: when the active camera has been probed (real webcam dims cached by
// WebcamOverlay's loadedmetadata handler), use them so the box matches the actual
// camera aspect. Without this the box defaults to a hardcoded 4:3 (960x720) and the
// Rust `fit_cam_aspect` closure shrinks the real content inside the wrong-aspect box,
// leaving visible empty margin inside the PiP container (typical case: a 16:9 webcam
// shipped to a 4:3 box). The probed size is keyed by sourcePath and survives across
// re-mounts of the same camera — `webcamSourceSize` is the dimension snapshot the
// caller (NativeCompositorOverlay) currently knows about.
// The block layouts (side-by-side / top-bottom) inset the welded screen+camera
// block by the padding, so the rect we ship must be resolved against the same
// padded content area the preview uses — `compositor.rs` consumes an app-provided
// `webcamRect` verbatim (it only scale_frame's the SCREEN by padding), so an
// unpadded rect here would leave the camera behind while the screen moved.
const formatFill = isFormatFillActive(document);
const maxContentSize = paddedContentSize(
outputDims,
settings.padding,
settings.aspectRatio === "auto",
);
/**
* The screen source SHAPE of a clip = its recording's own dimensions × its crop.
*
* Both halves vary per clip and always have: a clip is a screen recording + an
* optional camera + optional audio, and nothing forces two clips to have been
* recorded at the same size or ratio. Cropping is simply one more way that shape
* varies — a 16:9 recording cropped to 9:16 must lay out exactly like a clip
* recorded natively in 9:16. This is NOT the scene's aspect ratio, which is global
* (`outputDims`); it is per clip.
*
* So the layout has to be resolved per clip. Resolving it once for the whole scene
* was already wrong for a document mixing recording resolutions — the crop only
* made the existing defect visible, by letting one document hold two shapes.
*/
const screenSourceSizeOf = (clip: AxcutClip, index: number) =>
// index-aligned; null = identity crop
screenSourceSize(assetById.get(clip.assetId)?.video, cropByClip[index]);
/** Does THIS clip have a camera to lay out? Same expression as the `webcamPath` sent
* with the clip above, so the layout and the decoder can never disagree about it.
* Note this is NOT `hasAnyClipWithCamera` (which gates the Layout panel): that one
* ignores `visible` on purpose, so the panel stays reachable to un-hide a camera. */
const clipHasCamera = (clip: AxcutClip) =>
assetCameraSource(assetById.get(clip.assetId)).path !== "";
/**
* The layout preset is GLOBAL — one panel for the whole timeline — but the camera is
* per clip: a project mixes a screen+webcam recording with a plain import that has
* none. A clip with no camera must lay out as if the preset were "no-webcam".
*
* Skipping this is not merely a stray thumbnail: the block presets (`dual-frame`,
* `vertical-stack`) size the SCREEN off the block, so a camera-less clip kept the
* screen squeezed into its half with nothing beside it. `has_webcam` (native) only
* gates the camera's own draw — it cannot give the screen its frame back.
*/
/**
* The camera source SHAPE of a clip, resolved the same way and for the same reasons
* as `screenSourceSizeOf` above: per clip, because two clips need not have been
* recorded with the same camera.
*/
const webcamSourceSizeOf = (clip: AxcutClip) =>
webcamBoxSourceSize(
assetById.get(clip.assetId)?.cameraTrack,
webcamSourceSize,
settings.webcamCropRegion,
);
const layoutForClip = (
screenSize: { width: number; height: number },
hasCamera: boolean,
camSize: { width: number; height: number },
requestedPreset: LayoutPreset = settings.webcamLayoutPreset,
) => {
const preset = resolveWebcamLayoutPreset(requestedPreset, hasCamera);
return computeCompositeLayout({
canvasSize: outputDims,
maxContentSize,
// Filled, the screen takes the whole padded area; the compositor cuts the window.
screenSize: formatFill ? maxContentSize : screenSize,
webcamSize: preset === "no-webcam" ? null : camSize,
layoutPreset: preset,
webcamSizePreset: settings.webcamSizePreset,
webcamAnchor: settings.webcamAnchor,
webcamMaskShape: settings.webcamMaskShape,
webcamRoundness: settings.webcamRoundness,
frame: settings.frame,
});
};
const toFrameFractions = (r: RenderRect) => ({
x: r.x / outputDims.width,
y: r.y / outputDims.height,
width: r.width / outputDims.width,
height: r.height / outputDims.height,
});
// Corner radii leave as a fraction of the box they round (see `screenRadiusFrac`).
// Both the radius and the box are measured in output pixels here, so the ratio is
// scale-invariant — the clamp `computeCompositeLayout` applies in absolute pixels
// still lands exactly where it did, at any render size.
const radiusFractionOf = (box: RenderRect | null | undefined, radius: number | undefined) => {
const shortSide = box ? Math.min(box.width, box.height) : 0;
return box && shortSide > 0 && radius != null ? radius / shortSide : null;
};
const resolvedLayoutOf = (layout: ReturnType<typeof layoutForClip>) =>
layout
? {
screenRect: toFrameFractions(layout.screenRect),
webcamRect: layout.webcamRect ? toFrameFractions(layout.webcamRect) : null,
screenRadiusFrac: radiusFractionOf(layout.screenRect, layout.screenBorderRadius),
webcamRadiusFrac: radiusFractionOf(layout.webcamRect, layout.webcamRect?.borderRadius),
webcamShape: layout.webcamRect?.maskShape ?? settings.webcamMaskShape,
screenCover: formatFill || (layout.screenCover ?? false),
}
: null;
// One resolved layout per visible clip, index-aligned with `clips` / `cropByClip`.
// `for_clip_window` (Rust) selects the entry for the clip being composed, so the
// draw path keeps reading a single `layout` and needs no per-clip branch of its own.
const layoutByClip = visibleClips.map((clip, index) =>
resolvedLayoutOf(
layoutForClip(screenSourceSizeOf(clip, index), clipHasCamera(clip), webcamSourceSizeOf(clip)),
),
);
const storedLayouts = materializeLayouts(document);
const projectedLayouts = document.layoutRegions?.length
? projectRegionsToSource(storedLayouts, visibleClips, document.timeline.clips, () =>
createId("layout"),
)
: [];
const layoutRegions: SceneLayoutRegion[] = projectedLayouts.flatMap((region) => {
if (region.clipIndex === undefined) return [];
const clipIndex = region.clipIndex;
const clip = visibleClips[clipIndex];
if (!clip) return [];
const requested = region.legacyCamera?.basePreset ?? region.preset;
const hasCamera = clipHasCamera(clip);
const preset = resolveWebcamLayoutPreset(requested, hasCamera);
const resolve = (p: LayoutPreset) =>
resolvedLayoutOf(
layoutForClip(screenSourceSizeOf(clip, clipIndex), hasCamera, webcamSourceSizeOf(clip), p),
);
const layout = resolve(preset);
if (!layout) return [];
const previous = storedLayouts.find(
(r) =>
r.clipId === region.clipId &&
r.sourceEndSec !== undefined &&
region.sourceStartSec !== undefined &&
Math.abs(r.sourceEndSec - region.sourceStartSec) < 0.001,
);
const fromLayout =
!region.legacyCamera && previous && !previous.legacyCamera ? resolve(previous.preset) : null;
// Carry this history through later free passages and trim projections. Native can
// then keep a zoom already in flight inside its viewport, without constraining a new one.
const keepScreenClipFromSec = storedLayouts.reduce<number | undefined>((last, candidate) => {
if (
candidate.clipId !== region.clipId ||
candidate.sourceEndSec === undefined ||
region.sourceStartSec === undefined ||
candidate.sourceEndSec > region.sourceStartSec + 0.0001 ||
!(
formatFill ||
(hasCamera && isWebcamBlockLayout(candidate.legacyCamera?.basePreset ?? candidate.preset))
)
)
return last;
return Math.max(last ?? -Infinity, candidate.sourceEndSec);
}, undefined);
return [
{
clipIndex: region.clipIndex,
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
preset,
layout,
webcamReactiveZoom: resolveWebcamReactiveZoom(preset, settings.webcamReactiveZoom),
...(keepScreenClipFromSec === undefined ? {} : { keepScreenClipFromSec }),
...(fromLayout && previous
? {
fromLayout,
fromPreset: resolveWebcamLayoutPreset(previous.preset, hasCamera),
fromWebcamReactiveZoom: resolveWebcamReactiveZoom(
previous.preset,
settings.webcamReactiveZoom,
),
}
: {}),
transitionStartSec: region.sourceStartSec ?? region.startMs / 1000,
transitionSec: fromLayout
? Math.min(
LAYOUT_TRANSITION_SECONDS,
((region.sourceEndSec ?? region.endMs / 1000) -
(region.sourceStartSec ?? region.startMs / 1000)) /
2,
)
: 0,
},
];
});
// Scalar fields stay the FIRST clip's layout: they are the fallback for a payload
// without `layoutByClip`, and the value native starts from before any clip is active.
const computedLayout = visibleClips[0]
? layoutForClip(
screenSourceSizeOf(visibleClips[0], 0),
clipHasCamera(visibleClips[0]),
webcamSourceSizeOf(visibleClips[0]),
)
: null;
const webcamRect = computedLayout?.webcamRect
? toFrameFractions(computedLayout.webcamRect)
: null;
const screenRect = computedLayout ? toFrameFractions(computedLayout.screenRect) : null;
// The deepest zoom each clip takes before its recording blurs (`zoomScaleLimitOf`).
const zoomLimitByClip = visibleClips.map((clip, index) =>
zoomScaleLimitOf(
layoutByClip[index]?.screenRect ?? screenRect,
outputDims,
screenSourceSizeOf(clip, index),
),
);
return {
clips,
...(layoutRegions.length ? { layoutRegions } : {}),
layout: {
preset: settings.webcamLayoutPreset,
// web-consistent 0..1 fraction of the canvas reference dimension
// (see `SceneLayout.webcamSize` for the consumer-facing semantics).
webcamSize: webcamSizeToFraction(settings.webcamSizePreset),
// The RESOLVED shape, not the raw setting — the block layouts always cut a
// rectangle, whatever shape the user last picked under picture-in-picture.
webcamShape: computedLayout?.webcamRect?.maskShape ?? settings.webcamMaskShape,
webcamMirror: settings.webcamMirrored,
// The centre of the rect the anchor resolved to. Native places the camera from
// `webcamRect`; this only keeps the field's meaning for anything that reads it.
webcamPosition: webcamRect
? { cx: webcamRect.x + webcamRect.width / 2, cy: webcamRect.y + webcamRect.height / 2 }
: null,
// Gated by the preset: the block layouts size their camera off the screen
// box, so it must never shrink mid-zoom (the UI hides the toggle too).
webcamReactiveZoom: resolveWebcamReactiveZoom(
settings.webcamLayoutPreset,
settings.webcamReactiveZoom,
),
webcamAnchor: settings.webcamAnchor,
webcamCrop: settings.webcamCropRegion,
webcamRect,
screenRect,
screenRadiusFrac: radiusFractionOf(
computedLayout?.screenRect,
computedLayout?.screenBorderRadius,
),
screenCover: formatFill || (computedLayout?.screenCover ?? false),
...(formatFill ? { screenFollow: true as const } : {}),
webcamRadiusFrac: radiusFractionOf(
computedLayout?.webcamRect,
computedLayout?.webcamRect?.borderRadius,
),
layoutByClip,
},
effects: {
padding: settings.padding / 100,
blur: settings.backgroundBlur,
shadow: settings.shadowIntensity,
// The slider is in pixels of a 1080 reference, whatever the source resolution —
// see `roundnessFrac`. No padding and no frame: the footage meets the scene's edges,
// with no background to round against, so the corners go square and the stored value
// waits for the padding to come back. Under a frame the frame meets the edges, and
// the footage keeps its corners. A block layout's screen carries its own radius
// (`screenRadiusFrac`), so this only ever squares Picture in Picture and camera-less
// clips.
roundnessFrac:
settings.padding > 0 || settings.frame !== "none"
? settings.borderRadius / ROUNDNESS_REFERENCE_PX
: 0,
motionBlur: settings.motionBlurAmount,
// Omitted at their defaults, like `webcamEffect`: the Rust side defaults both fields,
// so a project with no frame serializes exactly as it did before they existed.
...(settings.frame !== "none" ? { frame: settings.frame } : {}),
...(settings.frame !== "none" && settings.frameTheme !== "light"
? { frameTheme: settings.frameTheme }
: {}),
depthOfField: settings.depthOfField,
},
cursor: {
show: settings.cursorShow,
autoHide: settings.cursorAutoHide,
size: settings.cursor.size,
smoothing: settings.cursor.smoothing,
motionBlur: settings.cursor.motionBlur,
clickBounce: settings.cursor.clickBounce,
model3d: settings.cursor.model3d,
asArrow: settings.cursor.asArrow,
...(settings.cursor.clickImpact ? { clickImpact: true as const } : {}),
// The "Keep inside frame" option is gone: the cursor always overflows into the
// background. The field stays because the compositor's scene requires it.
clipToBounds: false,
theme: settings.cursorTheme,
},
audio: {
gainDb: settings.audioGainDb,
},
audioTracks,
...(clickSound ? { clickSound } : {}),
background: sceneBackground(settings.wallpaper, settings.wallpaperMotion),
zoomRegions: projectedZoomRegions.map((region) => ({
id: region.id,
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
// `ZOOM_DEPTH_SCALES` et rien d'autre. Cette ligne portait sa propre formule,
// `depth / 2 + 0.5`, qui ne coïncide avec la table qu'à la profondeur 2 — et le focus,
// lui, est borné avec la table (`getFocusBoundsForScale`, via `getZoomScale`). Les deux
// échelles se contredisaient donc : à la profondeur 3 le gimbal butait à 1/(2×1.8) =
// 0.2778 pendant que le compositeur découpait une demi-fenêtre de 1/(2×2.0) = 0.25, si
// bien que la fenêtre source s'arrêtait 2.78 % avant le bord — 53 px sur 1920
// définitivement hors d'atteinte, gimbal à fond dans le coin. Le symptôme rapporté :
// « le zoom coupe les bords extérieurs ».
//
// ponytail: et c'est `getZoomScale` en entier, pas la table seule. La ligne
// lisait `region.customScale ?? ZOOM_DEPTH_SCALES[depth]`, donc sans le clamp
// [1.0, 5.0] que l'UI applique. Or `zoomRegionSchema` n'exige de `customScale`
// que d'être positif : un document portant `customScale: 12` est valide, rendait
// 5× dans l'aperçu web et 12× ici. Même désaccord que ci-dessus, un cran plus bas.
//
// Bounded per clip: crop and zoom multiply, and past `MAX_SOURCE_MAGNIFICATION` the
// recording blurs. The inspector greys those levels out; this catches a level set
// before a crop or a format change made it too deep.
scale: Math.min(
getZoomScale(region),
zoomLimitByClip[region.clipIndex ?? 0] ?? MAX_ZOOM_SCALE,
),
focusX: region.focus.cx,
focusY: region.focus.cy,
// The global Auto-Focus toggle OVERRIDES each region's own mode rather than merely
// seeding it — that's what `settings.zoom.focusMode.lockedDisclaimer` promises the
// user ("turn it off to set focus mode per zoom"), and it's what makes the toolbar
// button a one-click "make every zoom follow the cursor".
focusMode: settings.autoFocusAll ? "auto" : (region.focusMode ?? null),
rotation: region.rotationPreset ?? null,
clipIndex: region.clipIndex,
...(region.underTrim ? { underTrim: true } : {}),
...(region.hideCursor ? { hideCursor: true } : {}),
})),
annotations: projectedAnnotations
.map((region) => {
const style = region.style;
// Omitted rather than sent as null/undefined when absent: an annotation still on the
// footage keeps emitting the exact keys it always has.
const space = region.space;
// Same treatment, same reason: only captions pin an edge. That includes the
// caption annotations `openscreen captions` writes, whose box is the editor's
// caption box: bottom-anchored, a caption that wraps grows upward from the inset
// instead of past the frame's edge.
const verticalAlign =
(region as { verticalAlign?: "top" | "bottom" }).verticalAlign ??
(region.annotationSource === "auto-caption" ? "bottom" : undefined);
const base = {
id: region.id,
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
clipIndex: region.clipIndex,
...(region.underTrim ? { underTrim: true as const } : {}),
kind: region.type,
...(space ? { space } : {}),
// Authored as percentages of the box named by `space` — the screen rect unless
// this is a caption; the native side wants fractions either way.
x: region.position.x / 100,
y: region.position.y / 100,
w: region.size.width / 100,
h: region.size.height / 100,
zIndex: region.zIndex,
} as const;
if (region.type === "text") {
return {
...base,
text: {
// `content` first, and with `||` rather than `??`. The inspector's textarea
// writes to `content`; `textContent` is the parallel slot, which
// `addAnnotation` initialises to "". Since "" is neither null nor undefined,
// `textContent ?? content` returned that empty string for every annotation
// created since — so the compositor was handed nothing to draw and text
// vanished from the preview the moment the DOM overlay stopped painting it.
content: region.content || region.textContent || "",
color: style.color,
backgroundColor: style.backgroundColor,
fontSizeRel: annotationFontSizeFraction(style.fontSize),
// The one place every drawn text passes: a family that does not ship
// (a project saved with any other name) draws in the default rather
// than in whatever the machine falls back to.
fontFamily: resolveTextFontFamily(style.fontFamily),
fontWeight: style.fontWeight,
fontStyle: style.fontStyle,
textDecoration: style.textDecoration,
textAlign: style.textAlign,
...(verticalAlign ? { verticalAlign } : {}),
animation: style.textAnimation ?? null,
},
};
}
if (region.type === "image") {
// `content` first: that is the field the live overlay reads (it checks
// `content.startsWith("data:image")`), `imageContent` being the parallel slot
// older documents used. Reading them the other way round would render an image
// the preview isn't showing.
return { ...base, imagePath: region.content || region.imageContent || "" };
}
if (region.type === "figure") {
const figure = region.figureData;
return {
...base,
figure: {
direction: figure?.arrowDirection ?? "right",
color: figure?.color ?? "#34B27B",
strokeWidth: figure?.strokeWidth ?? 4,
},
};
}
const blur = region.blurData;
return {
...base,
blur: {
style: blur?.type ?? "mosaic",
shape: blur?.shape ?? "rectangle",
color: blur?.color ?? "white",
intensity: blur?.intensity ?? 12,
blockSize: blur?.blockSize ?? 12,
...(blur?.freehandPoints
? {
freehandPoints: blur.freehandPoints.map((p) => ({
x: p.x / 100,
y: p.y / 100,
})),
}
: {}),
},
};
})
// Ascending zIndex so the compositor paints in order without sorting per frame.
.sort((a, b) => a.zIndex - b.zIndex),
cameraFullscreenRegions: projectedCameraFullscreenRegions.map((region) => ({
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
clipIndex: region.clipIndex,
...(region.underTrim ? { underTrim: true } : {}),
...(region.continuesBefore ? { continuesBefore: true } : {}),
...(region.continuesAfter ? { continuesAfter: true } : {}),
})),
// Speed is the one modifier with nothing to show for itself on a parked playhead: a
// still frame has no rate. So the entries under a trim are dropped here rather than
// shipped inert — `speed_at` (regions.rs) matches on clipIndex + time with no window
// to bound it, and the export's frame count is derived from these spans. Nothing to
// gain, an arithmetic to put at risk.
speedRegions: projectedSpeedRegions
.filter((region) => !region.underTrim)
.map((region) => ({
startSec: region.startMs / 1000,
endSec: region.endMs / 1000,
speed: region.speed,
clipIndex: region.clipIndex,
})),
cropByClip,
...(sceneShaderEffects.length > 0 ? { shaderEffects: sceneShaderEffects } : {}),
output: { ...pickOutputDims(document, settings.aspectRatio), fps: null },
// Omitted rather than sent as `{mode:"none"}`: the Rust side defaults the field, and
// every project without a webcam effect would otherwise carry it for nothing.
...(settings.webcamBackgroundMode !== "none"
? {
webcamEffect: {
mode: settings.webcamBackgroundMode,
blurIntensity: settings.webcamBlurIntensity,
background: parseWallpaper(settings.webcamWallpaper),
},
}
: {}),
};
}