Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

14.8 KB
Raw Blame
/**
 * Type declarations for the Rust napi-rs native compositor addon
 * (`compositor_view.node`). Built separately and loaded by
 * `electron/native-bridge/services/compositorViewService.ts`. Until the
 * prebuilt `.node` binary is present the service logs once and falls back to
 * safe no-ops — the type contract here is the only contract renderer code
 * relies on.
 *
 * The compositor renders OFFSCREEN at the resolution given by `createView`'s
 * `rect.width` / `rect.height`; the renderer polls `readFrame` on a timer and
 * paints the result into an HTML `<canvas>`. There is no OS window — the rect
 * is purely a target preview resolution, and the `x` / `y` fields are
 * vestigial (ignored native-side; the TS side keeps them on the wire so the
 * `CompositorViewRect` shape is unchanged and existing callers don't need a
 * refactor for fields they never used).
 */

export interface CompositorViewRect {
	x: number;
	y: number;
	width: number;
	height: number;
}

export type CompositorParamValue = boolean | number | string;

/** A self-describing preview frame from `readFrame`: pixels plus everything needed
 *  to interpret them (`width`/`height`) and to decide whether to repaint (`gen`).
 *  `gen` is a monotonic per-frame generation (≥ 1); the renderer keeps the last one
 *  it painted and passes it back as `sinceGen` so an unchanged frame is never
 *  re-delivered. `data` is RGBA8, `width * height * 4` bytes. */
export interface NativeFramePacket {
	gen: number;
	width: number;
	height: number;
	data: Buffer;
	/** The footage in this frame: its TL, TR, BR, BL corners (x, y as fractions of the frame,
	 *  eight numbers), where a privacy blur's gimbal goes. Absent before anything is composed. */
	footage?: number[] | null;
	/** The footage maps from those corners by their homography (real camera), not bilinearly. */
	footageProjective?: boolean;
	/** Where the view was when it composed this frame: the active clip in `scene.clips` and the
	 *  screen frame's time in its source file. Absent from an older `.node`. */
	clipIndex?: number;
	sourceTimeSec?: number;
}

/** A preview frame left in a shared GPU texture instead of copied into RAM (Windows, hardware
 *  backend — see `setSharedFrames`). `handle` is the texture's NT handle the way Electron's
 *  `sharedTexture.importSharedTexture` takes it: 8 bytes, little-endian, valid in this process
 *  only. Its `slot` stays reserved until `releaseSharedFrame(id, slot, gen)`. */
export interface NativeSharedFramePacket {
	gen: number;
	slot: number;
	handle: Buffer;
	width: number;
	height: number;
	footage?: number[] | null;
	footageProjective?: boolean;
	/** Where the view was when it composed this frame: the active clip in `scene.clips` and the
	 *  screen frame's time in its source file. Absent from an older `.node`. */
	clipIndex?: number;
	sourceTimeSec?: number;
}

/** What `checkShaderEffect` learns of an agent-written effect (`docs/shader-effects.md`, E.3). */
export interface ShaderEffectCheck {
	/** The error as the agent must read it (line, column, excerpt); absent when accepted. */
	error?: string | null;
	/** The indicative cost tier: never a refusal. */
	tier?: "light" | "medium" | "heavy" | "very-heavy" | null;
	/** `frame()` reads per pixel, loops multiplied by their bound. */
	frameReads?: number | null;
	/** Loop iterations per pixel. */
	iterations?: number | null;
}

export interface ExportStats {
	frames: number;
	wallS: number;
	fps: number;
	/** Duration of the exported video (seconds) — distinct from `wallS` (real render time). */
	videoDurationS: number;
}

/** Bilan d'un export GIF natif (slice 1). Mêmes champs que `ExportStats`
 *  plus la taille du fichier sur disque — format petit, mesure utile. */
export interface GifExportStats {
	frames: number;
	wallS: number;
	fps: number;
	videoDurationS: number;
	/** Size of the final `.gif` file in bytes, measured after the encoder
	 *  drops (i.e. after the GIF89a trailer flush). */
	fileBytes: number;
}

/** Outcome of a container-only remux — see `remuxSeekable`. */
export interface RemuxStats {
	/** Packets copied verbatim, all streams combined. */
	packets: number;
	/** Streams kept in the output (video/audio/subtitle; anything else is dropped). */
	streams: number;
	wallS: number;
}

/** Sortie GIF native : taille, cadence, loop, dither. Tout optionnel —
 *  absent → 854×480, 12 fps, boucle infinie, dithering Floyd-Steinberg. */
export interface GifParamsInput {
	width?: number;
	height?: number;
	fps?: number;
	/** How many times the GIF plays: `0` or omitted = forever, `1` = once, `n` = n times. */
	loopCount?: number;
	/** Floyd-Steinberg error diffusion at quantization. On by default. */
	dither?: boolean;
}

/** Output size/framerate/codec the app wants. All optional — omitted → 1920x1080 / first
 *  clip's fps / h264. `width`/`height` are rounded to the nearest even number (NV12 4:2:0). */
export interface ExportParamsInput {
	width?: number;
	height?: number;
	fps?: number;
	/** "h264" | "h265". Anything else (e.g. "vp9", no AMF hardware equivalent) fails the export
	 *  with a clear error instead of silently falling back to h264. */
	codec?: string;
	/** Target video bitrate in bits/s, from the output size and frame rate. Omitted or 0 → the
	 *  pipeline's own fallback, which ignores the frame rate (8 Mb/s at 1080p). */
	bitrate?: number;
}

/** One timeline clip for the native multiclip export (screen + webcam files + source trim). */
export interface ClipInput {
	screenPath: string;
	webcamPath: string;
	sourceStartSec: number;
	sourceEndSec: number;
	/** webcam source time = screen source time − this. */
	webcamOffsetSec: number;
}

/** Which backend the compositor will run on. `"cpu"` = WARP rasterisation + software
 *  decode/encode, used when no usable D3D11 GPU is present: correct output, but roughly
 *  8 fps preview with all effects and minutes-long exports. `"none"` = no D3D11 device at
 *  all, so the view will fail with its own, more specific message. */
export type CompositorBackend = "hardware" | "cpu" | "none";

/** Whether this machine can segment the camera, and if not, what is missing. Mirrored in
 *  `src/native/contracts.ts` for the renderer, the same way `CompositorBackend` is — the addon
 *  boundary and the IPC boundary each own their shape. */
export type SegmentationSupport = "ready" | "no-runtime" | "no-model" | "none";

export interface CompositorViewAddon {
	/** What this machine offers, asked without allocating a view — the export dialog
	 *  needs the answer before any preview exists. Cached native-side. */
	probeBackend(): CompositorBackend;
	/** Checks an effect's WGSL: validation, literal loop bounds, the real compile for this machine's
	 *  backend (FXC on Windows, Metal on macOS, SPIR-V on Linux), and the cost tier. `paramsJson`
	 *  is the declared parameters, `[{ "name": "strength", "type": "number" }, …]`. Runs on the
	 *  libuv pool: a Metal compile takes 120 ms and more. Absent from older builds. */
	checkShaderEffect?(code: string, paramsJson: string): Promise<ShaderEffectCheck>;

	/** Whether the ONNX Runtime library is where the app staged it, and therefore whether a
	 *  segmentation mask can be produced at all. Asked of the machine rather than guessed from
	 *  the platform: upstream publishes no ONNX build for Intel Macs, and a dev checkout or a
	 *  `--dir` build has none either. */
	segmentationRuntimeAvailable(): boolean;

	/** Subject mask for ONE frame, outside any view — the crop thumbnail shows the camera
	 *  background with it. `rgba` is already at the model's size (256x144 RGBA8); resolves one
	 *  byte per pixel, 0 = background, 255 = subject. Optional: an older `.node` predates it. */
	segmentFrame?(modelPath: string, rgba: Buffer): Promise<Buffer>;

	/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the
	 *  target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side).
	 *  No HWND/native-window-handle is passed: there's no OS window to parent to. The
	 *  renderer reads frames back via `readFrame` and paints them into a canvas.
	 *
	 *  Optional screen/webcam/cursor paths (F3 — the app's real recording, two separate H264
	 *  files); omitted → the POC fixture. */
	createView(
		rect: CompositorViewRect,
		screenPath?: string,
		webcamPath?: string,
		cursorPath?: string,
	): number;
	setRect(id: number, rect: CompositorViewRect): void;
	/** Returns the most recently rendered frame as a self-describing packet
	 *  (`{ gen, width, height, data }`) IF its generation is newer than `sinceGen`.
	 *  `data` is a raw RGBA pixel buffer (length = `width * height * 4`, byte order
	 *  RGBA — what `ImageData` / `putImageData` expect).
	 *
	 *  Returns `null` — nothing to paint — when the view is unknown, no frame has
	 *  been composed yet, OR the caller already holds the current generation
	 *  (`gen <= sinceGen`). That last case is the idle path (preview paused on a
	 *  still frame): `null` comes back WITHOUT cloning the buffer or crossing IPC.
	 *  Pass `sinceGen = 0` to force delivery of the current frame. */
	readFrame(id: number, sinceGen: number): NativeFramePacket | null;
	/** Deliver this view's frames as shared GPU textures (`readSharedFrame`) rather than RAM
	 *  pixels (`readFrame`). Returns `false` where that cannot work (not Windows, software
	 *  backend): the view keeps reading back. Optional: an older `.node` predates it. */
	setSharedFrames?(id: number, enabled: boolean): boolean;
	/** The latest frame left in the view's shared texture ring, if newer than `sinceGen`.
	 *  `null` too while the view reads back to RAM. Throws the render thread's fatal error,
	 *  like `readFrame`. */
	readSharedFrame?(id: number, sinceGen: number): NativeSharedFramePacket | null;
	/** Chromium let go of frame `gen` in `slot`, in every process: the render thread may write
	 *  that slot again. A no-op for a destroyed view. */
	releaseSharedFrame?(id: number, slot: number, gen: number): void;
	setParam(id: number, key: string, value: CompositorParamValue): void;
	setPlaying(id: number, playing: boolean): void;
	/** Seeks the view to source-media `seconds` for the active clip. */
	presentTime(id: number, seconds: number): void;
	/** Installs the app scene (JSON `SceneDescription`) — layout preset etc. drive the render
	 *  instead of the fixture. Invalid JSON is ignored native-side. */
	setScene(id: number, sceneJson: string): void;
	setActiveClip(
		id: number,
		screenPath: string,
		webcamPath: string,
		webcamOffsetSec: number,
		clipIndex: number,
		sourceTimeSec: number,
	): void;
	destroyView(id: number): void;
	/** Renders the fixture to `outPath` (C8), auto-pausing live previews. `onProgress`
	 *  (frames encoded so far) is optional and called at most ~10/s from the render
	 *  thread — cheap: the encode loop already ticks a progress hook every frame, this
	 *  just forwards it (throttled) instead of the previous no-op. */
	export(outPath: string, onProgress?: (frames: number) => void): Promise<ExportStats>;
	/** Renders the real timeline (ordered clips + trims) to `outPath`, auto-pausing previews.
	 *  `sceneJson` — same `SceneDescription` as the live preview (background/layout/webcam/cursor/
	 *  effects); omitted or invalid → nothing configured is applied (not a masking fallback).
	 *  `params` — output size/fps/codec; omitted → 1920x1080/first clip's fps/h264.
	 *  `onProgress` (frames encoded so far) is optional, throttled to ~10/s.
	 *  `control` (`createMp4ExportControl`) — cancelling it stops the render between two frames
	 *  and rejects with `MP4_EXPORT_CANCELLED`, leaving no file. */
	exportMulti(
		clips: ClipInput[],
		outPath: string,
		sceneJson?: string,
		params?: ExportParamsInput,
		onProgress?: (frames: number) => void,
		control?: object,
	): Promise<ExportStats>;
	/** Opaque napi External, like the GIF one. Absent from a `.node` whose `exportMulti` ignores
	 *  a control: that addon's MP4 export cannot be cancelled. */
	createMp4ExportControl?(): object;
	cancelMp4Export?(control: object): boolean;
	/** Frames of the timeline composed as `exportMulti` composes them, without encoding a video:
	 *  getFrames when the export is missing or stale. Same clips and scene as `exportMulti`, sized
	 *  by the scene's output. `frames` are indices on the `fps` grid, and one past the timeline's
	 *  end renders nothing. JPEG, at most `maxWidth` wide. Absent from older builds. */
	renderFrames?(
		clips: ClipInput[],
		sceneJson: string | undefined,
		fps: number,
		frames: number[],
		maxWidth: number,
	): Promise<Array<{ index: number; width: number; height: number; jpeg: Buffer }>>;
	/** Native GIF export. Identical inputs to `exportMulti` — same clips, same
	 *  scene — because it is the same render: both drive `walk_composited_timeline`
	 *  in the compositor crate and differ only in the encoder. Cursor, background,
	 *  layout and webcam all come from the scene, so there is no GIF-specific
	 *  input. No codec pick: GIF is one codec. `params` defaults to 854×480,
	 *  12 fps, infinite loop, dithered (`GifExportParams::default`).
	 *  `onProgress(frames)` is throttled to ~10/s like the MP4 path. */
	exportGif(
		clips: ClipInput[],
		outPath: string,
		sceneJson?: string,
		params?: GifParamsInput,
		onProgress?: (frames: number) => void,
		control?: object,
	): Promise<GifExportStats>;
	/** Opaque napi External: kept in main, never sent to a renderer. */
	createGifExportControl?(): object;
	cancelGifExport?(control: object): boolean;

	/** Stream-copy `inputPath` to `outputPath` through the matroska muxer, rebuilding the
	 *  container (real `Duration` computed from the packet timestamps, plus `Cues` and
	 *  `SeekHead`) without re-encoding a single frame.
	 *
	 *  Optional at the type level: a `.node` built before this export exists in the wild
	 *  (dev trees keep a stale binary until the next `build-linux-compositor-addon.mjs`),
	 *  and the caller degrades to "leave the file alone" rather than failing the save. */
	remuxSeekable?(inputPath: string, outputPath: string): Promise<RemuxStats>;

	/** Loudness-normalisation gain in dB that the export applies to this voice file (the
	 *  recording's own audio, or a voiceover take), measured over the whole file. The
	 *  preview applies the same number so it plays the voice at the exported level.
	 *  0 for a file with no audio, only silence, or that cannot be read.
	 *
	 *  Optional for the same reason as `remuxSeekable`: a stale `.node` predates it. */
	loudnessGainDb?(path: string): Promise<number>;
}

/**
 * The napi-rs addon is `require()`d from an absolute path resolved at runtime
 * (see `compositorViewService`). Declare the module shape here so the
 * `require()` return type can be constrained to `CompositorViewAddon` without
 * `any`. The wildcard pattern lets us match both the packaged
 * `electron/native/bin/<arch>/compositor_view.node` and the locally built
 * `electron/native/compositor-view/build/compositor_view.node` paths.
 */
declare module "*compositor_view.node" {
	const addon: CompositorViewAddon;
	export = addon;
}