Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

32.7 KB
Raw Blame
import { resolveTemporalContext } from "@hypit/temporal/markup";
import { assertEmptyElement as empty, assertAttributes as allowed, textAttribute as text, optionalTextAttribute as optionalText, type StructuredElement, type StructuredSurfaceHandler, type SurfaceComponentDraft, type SurfaceRecordDraft, type SurfaceResolvedReference, type MarkupAttributeValue } from "@hypit/hypit/markup";
import { sameType, type CanonicalValue, type TypeRef } from "@hypit/hypit/protocol";
import { blobTypes } from "@hypit/hypit/blob";
import { compositionTypes } from "@hypit/composition";
import { sealGraphFragment } from "@hypit/hypit/author";
import type { FragmentOperation } from "@hypit/hypit/author";
import { mediaTypes } from "@hypit/media";
import { spatialTypes } from "@hypit/spatial";
import { recipeType } from "@hypit/hypit/recipe";
import type { Recipe } from "@hypit/hypit/recipe";
import { temporalTypes } from "@hypit/temporal";
import { resolveTemporalWindowReference, temporalWindowAttributeNames } from "@hypit/temporal/markup";

import {
  decodeMediaFit,
  decodeMediaFramePaint,
  decodeVisualClipSpec,
  decodeMediaSampleSpec,
} from "./author.js";
import { sealVisualClipMotion } from "./motion.js";
import { sealVisualTrackHeader } from "./program.js";
import { assertVisualSourceTimeSpec } from "./source-time.js";
import { visualTrackProducers, visualTrackTypes } from "./manifest.js";
import type {
  MediaSamplingMotion,
  VisualClipMotion,
  VisualMotionPosition,
  VisualSourceTimeBounds,
  VisualSourceTimePoint,
  VisualSourceTimeRelation,
  VisualSourceTimeSpec,
} from "./types.js";

const input = (name: string) => ({ kind: "fragment-input" as const, name });
const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });

function numberValue(element: StructuredElement, name: string, fallback?: number): number {
  const raw = optionalText(element, name);
  if (raw === undefined && fallback !== undefined) return fallback;
  const value = Number(raw);
  if (!Number.isFinite(value)) throw new Error(`${element.name}.${name} must be a finite number.`);
  return value;
}

function integerValue(element: StructuredElement, name: string): number {
  const value = numberValue(element, name);
  if (!Number.isSafeInteger(value)) throw new Error(`${element.name}.${name} must be an integer.`);
  return value;
}

function motionPosition(element: StructuredElement): VisualMotionPosition {
  const source = text(element, "at");
  if (source === "start") return { kind: "start", offsetFrames: 0 };
  if (source === "end") return { kind: "end", offsetFrames: 0 };
  const percentage = /^(\d+(?:\.\d+)?)%$/u.exec(source);
  if (percentage !== null) return { kind: "progress", value: Number(percentage[1]) / 100 };
  const fromStart = /^(?:start\+)?(\d+)f$/u.exec(source);
  if (fromStart !== null) return { kind: "start", offsetFrames: Number(fromStart[1]) };
  const fromEnd = /^end-(\d+)f$/u.exec(source);
  if (fromEnd !== null) return { kind: "end", offsetFrames: -Number(fromEnd[1]) };
  throw new Error(`${element.name}.at must be start, end, a percentage, an exact frame from start, or end-Nf.`);
}

export function decodeVisualPoseKeyframe(element: StructuredElement): VisualClipMotion["keyframes"][number] {
  allowed(element, ["at", "x", "y", "scale", "scale-x", "scale-y", "rotate", "opacity", "origin-x", "origin-y", "easing"]);
  empty(element);
  const scale = optionalText(element, "scale");
  if (scale !== undefined && (element.attributes["scale-x"] !== undefined || element.attributes["scale-y"] !== undefined)) {
    throw new Error(`${element.name}.scale cannot be combined with scale-x or scale-y.`);
  }
  const easing = optionalText(element, "easing");
  if (easing !== undefined && !["linear", "ease-in", "ease-out", "ease-in-out"].includes(easing)) {
    throw new Error(`${element.name}.easing is invalid.`);
  }
  const uniformScale = scale === undefined ? undefined : Number(scale);
  if (uniformScale !== undefined && !Number.isFinite(uniformScale)) throw new Error(`${element.name}.scale must be finite.`);
  return {
    at: motionPosition(element),
    translateX: numberValue(element, "x", 0),
    translateY: numberValue(element, "y", 0),
    scaleX: uniformScale ?? numberValue(element, "scale-x", 1),
    scaleY: uniformScale ?? numberValue(element, "scale-y", 1),
    rotationDeg: numberValue(element, "rotate", 0),
    opacity: numberValue(element, "opacity", 1),
    originX: numberValue(element, "origin-x", 0.5),
    originY: numberValue(element, "origin-y", 0.5),
    ...(easing === undefined ? {} : { easing: easing as "linear" | "ease-in" | "ease-out" | "ease-in-out" }),
  };
}

function inlineMotion(element: StructuredElement): VisualClipMotion | undefined {
  const keyframes = element.children.flatMap((child) => child.kind === "element"
    && (child.name.endsWith(":Pose") || child.name === "Pose") ? [decodeVisualPoseKeyframe(child)] : []);
  return keyframes.length === 0 ? undefined : sealVisualClipMotion({ keyframes });
}

export const decodeVisualMotionSurface: StructuredSurfaceHandler = ({ element }) => {
  allowed(element, ["id"]);
  const id = text(element, "id");
  for (const child of element.children) {
    if (child.kind === "text") {
      if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Pose children.`);
      continue;
    }
    if (!child.name.endsWith(":Pose") && child.name !== "Pose") {
      throw new Error(`${element.name} accepts only Pose children.`);
    }
  }
  const motion = inlineMotion(element);
  if (motion === undefined) throw new Error(`${element.name} requires at least two Pose children.`);
  return {
    records: [{ id, type: visualTrackTypes.motion, value: { kind: "inline", value: motion as never }, range: element.range }],
    components: [], fragments: [], exports: [id],
  };
};

function reference(
  raw: MarkupAttributeValue | undefined,
  label: string,
  expected: TypeRef,
  resolve: (path: string) => SurfaceResolvedReference | undefined,
): SurfaceResolvedReference {
  if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
  const value = resolve(raw.path);
  if (value === undefined || !sameType(value.type, expected)) throw new Error(`${label} has the wrong Type.`);
  return value;
}

function recipe(value: SurfaceResolvedReference, label: string): Recipe {
  if (!sameType(value.type, recipeType) || value.record?.value.kind !== "inline") {
    throw new Error(`${label} must be an authored SVS Recipe.`);
  }
  return value.record.value.value as unknown as Recipe;
}

const FIT_ATTRIBUTES = ["fit", "frame-x", "frame-y", "content-x", "content-y", "fit-offset-x", "fit-offset-y", "fit-constraint"] as const;
const NUMERIC_CLIP_ATTRIBUTES = new Set(["frame-x", "frame-y", "content-x", "content-y", "fit-offset-x", "fit-offset-y"]);

function sourceRecipe(element: StructuredElement, treatment: Recipe): Recipe {
  const properties: Record<string, CanonicalValue> = { ...treatment.properties };
  for (const name of FIT_ATTRIBUTES) {
    const raw = optionalText(element, name);
    if (raw !== undefined) properties[name] = NUMERIC_CLIP_ATTRIBUTES.has(name) ? Number(raw) : raw;
  }
  return { path: `${treatment.path}:source`, properties };
}

const SOURCE_TIME_MAP_ATTRIBUTES = [
  "target-from", "target-until", "target-at", "source-at", "rate",
  "source-from", "source-until", "wrap-from", "wrap-until",
] as const;

function sourceTimePoint(raw: string | undefined, fallback: "start" | "end", label: string): VisualSourceTimePoint {
  const value = raw ?? fallback;
  if (value === "start") return { edge: "start", offsetFrames: 0 };
  if (value === "end") return { edge: "end", offsetFrames: 0 };
  const fromStart = /^(?:start\+)?(\d+)f$/u.exec(value);
  if (fromStart !== null) return { edge: "start", offsetFrames: Number(fromStart[1]) };
  const fromEnd = /^end-(\d+)f$/u.exec(value);
  if (fromEnd !== null) return { edge: "end", offsetFrames: -Number(fromEnd[1]) };
  throw new Error(`${label} must be start, end, Nf, start+Nf or end-Nf.`);
}

function sourceTimeBounds(
  element: StructuredElement,
  prefix: "target" | "source" | "wrap",
  required = false,
): VisualSourceTimeBounds | undefined {
  const from = optionalText(element, `${prefix}-from`);
  const until = optionalText(element, `${prefix}-until`);
  if (!required && from === undefined && until === undefined) return undefined;
  return {
    from: sourceTimePoint(from, "start", `${element.name}.${prefix}-from`),
    until: sourceTimePoint(until, "end", `${element.name}.${prefix}-until`),
  };
}

function divisor(left: number, right: number): number {
  let a = Math.abs(left);
  let b = Math.abs(right);
  while (b !== 0) [a, b] = [b, a % b];
  return a;
}

function sourceTimeRate(raw: string, label: string): { readonly numerator: number; readonly denominator: number } {
  const fraction = /^(-?\d+)\/([1-9]\d*)$/u.exec(raw);
  if (fraction !== null) {
    const numerator = Number(fraction[1]);
    const denominator = Number(fraction[2]);
    if (!Number.isSafeInteger(numerator) || !Number.isSafeInteger(denominator)) throw new Error(`${label} is outside safe arithmetic.`);
    const common = divisor(numerator, denominator);
    return { numerator: numerator / common, denominator: denominator / common };
  }
  const decimal = /^(-?)(\d+)(?:\.(\d+))?$/u.exec(raw);
  if (decimal === null) throw new Error(`${label} must be an exact integer, decimal or fraction.`);
  const fractional = decimal[3] ?? "";
  const denominator = 10 ** fractional.length;
  const magnitude = Number(decimal[2]) * denominator + (fractional.length === 0 ? 0 : Number(fractional));
  const numerator = decimal[1] === "-" ? -magnitude : magnitude;
  if (!Number.isSafeInteger(numerator) || !Number.isSafeInteger(denominator)) throw new Error(`${label} is outside safe arithmetic.`);
  const common = divisor(numerator, denominator);
  return { numerator: numerator / common, denominator: denominator / common };
}

export function decodeVisualSourceTimeRelation(element: StructuredElement): VisualSourceTimeRelation {
  allowed(element, SOURCE_TIME_MAP_ATTRIBUTES);
  empty(element);
  const target = sourceTimeBounds(element, "target", true)!;
  const source = sourceTimeBounds(element, "source", true)!;
  const hasRateRelation = element.attributes.rate !== undefined
    || element.attributes["target-at"] !== undefined
    || element.attributes["source-at"] !== undefined
    || element.attributes["wrap-from"] !== undefined
    || element.attributes["wrap-until"] !== undefined;
  if (!hasRateRelation) return { kind: "fit", target, source };
  const wrap = sourceTimeBounds(element, "wrap");
  const relation: VisualSourceTimeRelation = {
    kind: "rate",
    target,
    targetAt: sourceTimePoint(optionalText(element, "target-at"), "start", `${element.name}.target-at`),
    sourceAt: sourceTimePoint(optionalText(element, "source-at"), "start", `${element.name}.source-at`),
    rate: sourceTimeRate(optionalText(element, "rate") ?? "1", `${element.name}.rate`),
    source,
    ...(wrap === undefined ? {} : { wrap }),
  };
  return relation;
}

function inlineSourceTime(element: StructuredElement): VisualSourceTimeSpec | undefined {
  const relations = element.children.flatMap((child) => child.kind === "element"
    && (child.name.endsWith(":Map") || child.name === "Map") ? [decodeVisualSourceTimeRelation(child)] : []);
  if (relations.length === 0) return undefined;
  const value = { relations };
  assertVisualSourceTimeSpec(value);
  return value;
}

function sourceTimeFrom(
  element: StructuredElement,
  resolve: (path: string) => SurfaceResolvedReference | undefined,
): VisualSourceTimeSpec | undefined {
  const inline = inlineSourceTime(element);
  if (element.attributes["source-time"] === undefined) return inline;
  if (inline !== undefined) throw new Error(`${element.name} cannot combine source-time={SourceTime} with inline Map children.`);
  const value = reference(element.attributes["source-time"], `${element.name}.source-time`, visualTrackTypes.sourceTime, resolve);
  if (value.record?.value.kind !== "inline") throw new Error(`${element.name}.source-time must resolve to an inline SourceTime value.`);
  const spec = value.record.value.value as unknown as VisualSourceTimeSpec;
  assertVisualSourceTimeSpec(spec);
  return spec;
}

export const decodeVisualSourceTimeSurface: StructuredSurfaceHandler = ({ element }) => {
  allowed(element, ["id"]);
  const id = text(element, "id");
  const relations: VisualSourceTimeRelation[] = [];
  for (const child of element.children) {
    if (child.kind === "text") {
      if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Map children.`);
      continue;
    }
    if (!child.name.endsWith(":Map") && child.name !== "Map") throw new Error(`${element.name} accepts only Map children.`);
    relations.push(decodeVisualSourceTimeRelation(child));
  }
  const value = { relations };
  assertVisualSourceTimeSpec(value);
  return {
    records: [{ id, type: visualTrackTypes.sourceTime, value: { kind: "inline", value: value as never }, range: element.range }],
    components: [], fragments: [], exports: [id],
  };
};

function treatmentRecipe(
  element: StructuredElement,
  resolve: (path: string) => SurfaceResolvedReference | undefined,
): Recipe {
  if (element.attributes.treatment === undefined) return { path: `${element.name}:default-treatment`, properties: {} };
  return recipe(reference(element.attributes.treatment, `${element.name}.treatment`, recipeType, resolve), `${element.name}.treatment`);
}

type FragmentLayer =
  | { readonly kind: "paint"; readonly specName: string }
  | { readonly kind: "still"; readonly sourceName: string; readonly extentName: string; readonly placementKind: "fit" | "mapping"; readonly placementName: string; readonly specName: string }
  | { readonly kind: "timed" | "surface"; readonly sourceName: string; readonly placementKind: "fit" | "mapping"; readonly placementName: string; readonly specName: string };

type FragmentClip = {
  readonly suffix: string;
  readonly windowName: string;
  readonly frameName: string;
  readonly specName: string;
  readonly clipPathName?: string;
  readonly motionName?: string;
  readonly layers: readonly FragmentLayer[];
};

function appendLayers(operations: FragmentOperation[], prefix: string, layers: readonly FragmentLayer[]): string {
  const emptyId = `${prefix}:layers:empty`;
  operations.push({ id: emptyId, producer: visualTrackProducers.createLayers, inputs: {}, result: { kind: "output", name: "layers" } });
  let current = emptyId;
  for (const [index, layer] of layers.entries()) {
    const id = `${prefix}:layers:${String(index + 1).padStart(4, "0")}`;
    const common = { layers: operation(current), spec: input(layer.specName) };
    if (layer.kind === "paint") {
      operations.push({ id, producer: visualTrackProducers.appendPaintLayer, inputs: common, result: { kind: "output", name: "layers" } });
    } else if (layer.kind === "still") {
      operations.push({ id, producer: layer.placementKind === "fit"
        ? visualTrackProducers.appendStillLayer : visualTrackProducers.appendMappedStillLayer, inputs: {
        ...common, source: input(layer.sourceName), extent: input(layer.extentName),
        [layer.placementKind === "fit" ? "fit" : "mapping"]: input(layer.placementName),
      }, result: { kind: "output", name: "layers" } });
    } else {
      const producer = layer.kind === "timed"
        ? layer.placementKind === "fit" ? visualTrackProducers.appendTimedLayer : visualTrackProducers.appendMappedTimedLayer
        : layer.placementKind === "fit" ? visualTrackProducers.appendSurfaceLayer : visualTrackProducers.appendMappedSurfaceLayer;
      operations.push({ id, producer, inputs: {
        ...common, source: input(layer.sourceName),
        [layer.placementKind === "fit" ? "fit" : "mapping"]: input(layer.placementName),
      }, result: { kind: "output", name: "layers" } });
    }
    current = id;
  }
  return current;
}

function createVisualTrackSurfaceFragment(inputTypes: readonly { readonly name: string; readonly type: TypeRef }[], clips: readonly FragmentClip[]) {
  const operations: FragmentOperation[] = [
    { id: "track:set:empty", producer: visualTrackProducers.createSet, inputs: {}, result: { kind: "output", name: "set" } },
  ];
  let set = "track:set:empty";
  for (const clip of clips) {
    const prefix = `clip:${clip.suffix}`;
    const layers = appendLayers(operations, prefix, clip.layers);
    const appendId = `${prefix}:append`;
    let spec: FragmentOperation["inputs"][string] = input(clip.specName);
    if (clip.clipPathName !== undefined) {
      const bindId = `${prefix}:bind-clip-path`;
      operations.push({ id: bindId, producer: visualTrackProducers.bindClipPath, inputs: {
        spec, path: input(clip.clipPathName),
      }, result: { kind: "output", name: "spec" } });
      spec = operation(bindId);
    }
    if (clip.motionName !== undefined) {
      const bindId = `${prefix}:bind-motion`;
      operations.push({ id: bindId, producer: visualTrackProducers.bindMotion, inputs: {
        spec, motion: input(clip.motionName),
      }, result: { kind: "output", name: "spec" } });
      spec = operation(bindId);
    }
    const common = {
      set: operation(set), header: input("header"), timeline: input("timeline"),
      layers: operation(layers), frame: input(clip.frameName), spec, window: input(clip.windowName),
    };
    operations.push({ id: appendId, producer: visualTrackProducers.appendClip,
      inputs: common, result: { kind: "output", name: "set" } });
    set = appendId;
  }
  operations.push(
    { id: "track:finalize", producer: visualTrackProducers.finalize, inputs: { set: operation(set), header: input("header"), timeline: input("timeline") }, result: { kind: "output", name: "program" } },
    { id: "track:visual", producer: visualTrackProducers.projectVisual, inputs: { timeline: input("timeline"), program: operation("track:finalize") }, result: { kind: "output", name: "track" } },
  );
  return sealGraphFragment({
    inputs: inputTypes,
    operations,
    exports: [
      { name: "program", type: visualTrackTypes.program, root: operation("track:finalize") },
      { name: "visual", type: compositionTypes.visualTrack, root: operation("track:visual") },
    ],
  });
}

type SurfaceBuilder = {
  readonly records: SurfaceRecordDraft[];
  readonly inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }>;
  readonly inputTypes: { name: string; type: TypeRef }[];
  addReference(name: string, value: SurfaceResolvedReference): void;
  addRecord(name: string, id: string, type: TypeRef, value: unknown, range: StructuredElement["range"]): void;
  addValue(name: string, value: SurfaceResolvedReference["ref"], type: TypeRef): void;
};

function builder(): SurfaceBuilder {
  const records: SurfaceRecordDraft[] = [];
  const inputs: SurfaceBuilder["inputs"] = {};
  const inputTypes: SurfaceBuilder["inputTypes"] = [];
  return {
    records,
    inputs,
    inputTypes,
    addReference(name, value) {
      if (inputs[name] !== undefined) throw new Error(`Media Surface repeats input ${name}.`);
      inputs[name] = value.ref;
      inputTypes.push({ name, type: value.type });
    },
    addRecord(name, id, type, value, range) {
      if (inputs[name] !== undefined) throw new Error(`Media Surface repeats input ${name}.`);
      records.push({ id, type, value: { kind: "inline", value: value as never }, range });
      inputs[name] = { kind: "record", id };
      inputTypes.push({ name, type });
    },
    addValue(name, value, type) {
      if (inputs[name] !== undefined) throw new Error(`Media Surface repeats input ${name}.`);
      inputs[name] = value;
      inputTypes.push({ name, type });
    },
  };
}

function suffix(index: number): string {
  return String(index).padStart(4, "0");
}

type SourceLayerContext = {
  readonly trackId: string;
  readonly unitSuffix: string;
  readonly defaultRecipe: Recipe;
  readonly sourceElement: StructuredElement;
  readonly source?: DeclaredVisualSource;
  readonly sourceTime?: VisualSourceTimeSpec;
  readonly ignoreSiblingChildren?: boolean;
  readonly defaultLayerId?: string;
  readonly mapping?: SurfaceResolvedReference;
};

type DeclaredVisualSource =
  | { readonly kind: "image"; readonly value: SurfaceResolvedReference; readonly extent: SurfaceResolvedReference }
  | { readonly kind: "media"; readonly value: SurfaceResolvedReference }
  | { readonly kind: "surface"; readonly value: SurfaceResolvedReference };

const VISUAL_SOURCE_ATTRIBUTES = ["image", "media", "surface"] as const;

function declaredVisualSource(
  element: StructuredElement,
  resolve: (path: string) => SurfaceResolvedReference | undefined,
  required = false,
): DeclaredVisualSource | undefined {
  const present = VISUAL_SOURCE_ATTRIBUTES.filter((name) => element.attributes[name] !== undefined);
  if (present.length === 0) {
    if (element.attributes.extent !== undefined) throw new Error(`${element.name}.extent requires image={Artifact}.`);
    if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
    if (required) throw new Error(`${element.name} requires exactly one of image, media or surface.`);
    return undefined;
  }
  if (present.length !== 1) throw new Error(`${element.name} requires exactly one of image, media or surface.`);
  const kind = present[0]!;
  if (kind === "image") {
    if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
    return {
      kind,
      value: reference(element.attributes.image, `${element.name}.image`, blobTypes.blob, resolve),
      extent: reference(element.attributes.extent, `${element.name}.extent`, spatialTypes.extent, resolve),
    };
  }
  if (element.attributes.extent !== undefined) throw new Error(`${element.name}.extent is only valid with image={Artifact}.`);
  if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
  return kind === "media"
    ? { kind, value: reference(element.attributes.media, `${element.name}.media`, mediaTypes.synchronized, resolve) }
    : { kind, value: reference(element.attributes.surface, `${element.name}.surface`, mediaTypes.compositableSurface, resolve) };
}

export function decodeMediaSamplingKeyframe(element: StructuredElement): MediaSamplingMotion["keyframes"][number] {
  allowed(element, ["at", "zoom", "x", "y", "rotate", "easing"]);
  empty(element);
  const at = text(element, "at");
  const percentage = /^(\d+(?:\.\d+)?)%$/u.exec(at);
  const atProgress = at === "start" ? 0 : at === "end" ? 1
    : percentage === null ? Number.NaN : Number(percentage[1]) / 100;
  if (!Number.isFinite(atProgress) || atProgress < 0 || atProgress > 1) {
    throw new Error(`${element.name}.at must be start, end or a percentage inside 0%..100%.`);
  }
  const easing = optionalText(element, "easing");
  if (easing !== undefined && !["linear", "ease-in", "ease-out", "ease-in-out"].includes(easing)) {
    throw new Error(`${element.name}.easing is invalid.`);
  }
  return {
    atProgress, zoom: numberValue(element, "zoom", 1), offsetX: numberValue(element, "x", 0),
    offsetY: numberValue(element, "y", 0), rotationDeg: numberValue(element, "rotate", 0),
    ...(easing === undefined ? {} : { easing: easing as "linear" | "ease-in" | "ease-out" | "ease-in-out" }),
  };
}

function decodeSamplingMotion(element: StructuredElement, ignoreSiblingChildren = false): MediaSamplingMotion | undefined {
  const keyframes = element.children.flatMap((child) => {
    if (child.kind === "text") {
      if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Sampling children.`);
      return [];
    }
    if (!child.name.endsWith(":Sampling") && child.name !== "Sampling") {
      if (ignoreSiblingChildren) return [];
      throw new Error(`${element.name} accepts only Sampling children.`);
    }
    return [decodeMediaSamplingKeyframe(child)];
  });
  return keyframes.length === 0 ? undefined : { keyframes };
}

function sourceLayer(
  state: SurfaceBuilder,
  context: SourceLayerContext,
  layerIndex: number,
  resolve: (path: string) => SurfaceResolvedReference | undefined,
): FragmentLayer {
  const element = context.sourceElement;
  const source = context.source ?? declaredVisualSource(element, resolve, true)!;
  const appearance = element.attributes.appearance === undefined
    ? context.defaultRecipe
    : recipe(reference(element.attributes.appearance, `${element.name}.appearance`, recipeType, resolve), `${element.name}.appearance`);
  const motion = decodeSamplingMotion(element, context.ignoreSiblingChildren);
  const layerSuffix = `${context.unitSuffix}-layer-${suffix(layerIndex)}`;
  const layerId = context.defaultLayerId ?? optionalText(element, "id") ?? `${context.trackId}.${layerSuffix}`;
  const placementKind = context.mapping === undefined ? "fit" : "mapping";
  const placementName = `${layerSuffix}-${placementKind}`;
  const specName = `${layerSuffix}-spec`;
  if (context.mapping === undefined) {
    state.addRecord(placementName, `${context.trackId}.${layerSuffix}.fit`, spatialTypes.fit, decodeMediaFit(appearance), element.range);
  } else {
    state.addReference(placementName, context.mapping);
  }
  const sourceKind = source.kind === "image" ? "still"
    : source.kind === "surface" ? "surface" : "timed";
  const spec = decodeMediaSampleSpec(
    appearance,
    layerId,
    sourceKind,
    motion,
    context.source !== undefined,
    context.sourceTime,
  );
  state.addRecord(specName, `${context.trackId}.${layerSuffix}.spec`, visualTrackTypes.sampleLayerSpec, spec, element.range);
  const sourceName = `${layerSuffix}-source`;
  state.addReference(sourceName, source.value);
  if (sourceKind === "still") {
    const extentName = `${layerSuffix}-extent`;
    state.addReference(extentName, (source as Extract<DeclaredVisualSource, { readonly kind: "image" }>).extent);
    return { kind: "still", sourceName, extentName, placementKind, placementName, specName };
  }
  return { kind: sourceKind, sourceName, placementKind, placementName, specName };
}

function unitLayers(
  state: SurfaceBuilder,
  input: {
    readonly trackId: string;
    readonly unitSuffix: string;
    readonly element: StructuredElement;
    readonly appearance: Recipe;
    readonly directSource: DeclaredVisualSource;
    readonly sourceTime?: VisualSourceTimeSpec;
    readonly mapping?: SurfaceResolvedReference;
    readonly allowFramePaint: boolean;
  },
  resolve: (path: string) => SurfaceResolvedReference | undefined,
): readonly FragmentLayer[] {
  const layers: FragmentLayer[] = [];
  if (input.allowFramePaint) {
    const framePaint = decodeMediaFramePaint(input.appearance, `${input.trackId}.${input.unitSuffix}.frame-paint`);
    if (framePaint !== undefined) {
      const name = `${input.unitSuffix}-frame-paint`;
      state.addRecord(name, `${input.trackId}.${input.unitSuffix}.frame-paint`, visualTrackTypes.paintLayerSpec, framePaint, input.element.range);
      layers.push({ kind: "paint", specName: name });
    }
  }
  layers.push(sourceLayer(state, {
    trackId: input.trackId, unitSuffix: input.unitSuffix, defaultRecipe: input.appearance,
    sourceElement: input.element, source: input.directSource,
    ...(input.mapping === undefined ? {} : { mapping: input.mapping }),
    ...(input.sourceTime === undefined ? {} : { sourceTime: input.sourceTime }),
    ignoreSiblingChildren: true,
    defaultLayerId: "content",
  }, 1, resolve));
  return layers;
}

function validateUnitChildren(element: StructuredElement): void {
  for (const child of element.children) {
    if (child.kind === "text") {
      if (child.value.trim().length > 0) throw new Error(`${element.name} cannot contain text.`);
      continue;
    }
    const samplingChild = child.name.endsWith(":Sampling") || child.name === "Sampling";
    const poseChild = child.name.endsWith(":Pose") || child.name === "Pose";
    const mapChild = child.name.endsWith(":Map") || child.name === "Map";
    if (samplingChild || poseChild || mapChild) continue;
    throw new Error(`${element.name} has unsupported child ${child.name}.`);
  }
}

export const decodeVisualTrackSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
  allowed(element, ["id", "timeline"]);
  const trackId = text(element, "id");
  const state = builder();
  const context = resolveTemporalContext({ element, resolveReference });
  state.addReference("timeline", context.timeline);
  const headerId = `${trackId}.header`;
  state.addRecord("header", headerId, visualTrackTypes.header,
    sealVisualTrackHeader({ id: trackId }), element.range);
  const clips: FragmentClip[] = [];
  let clipIndex = 0;
  for (const child of element.children) {
    if (child.kind === "text") {
      if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Clip children.`);
      continue;
    }
    if (child.name.endsWith(":Clip") || child.name === "Clip") {
      clipIndex += 1;
      const clipSuffix = suffix(clipIndex);
      allowed(child, [
        "id", "frame", "treatment", "motion", "z",
        "image", "media", "surface", "extent",
        "clip", "source-time", "mapping", ...FIT_ATTRIBUTES, ...temporalWindowAttributeNames,
      ]);
      const id = optionalText(child, "id") ?? `${trackId}.clip.${clipSuffix}`;
      const treatment = treatmentRecipe(child, resolveReference);
      const sourceAppearance = sourceRecipe(child, treatment);
      const clipPathName = child.attributes.clip === undefined ? undefined : `clip-${clipSuffix}-clip-path`;
      if (clipPathName !== undefined) {
        if (treatment.properties.clip !== undefined) throw new Error(`${child.name} cannot combine clip={Path} with treatment clip.`);
        state.addReference(clipPathName, reference(child.attributes.clip, `${child.name}.clip`, spatialTypes.path, resolveReference));
      }
      const authoredMotion = inlineMotion(child);
      if (authoredMotion !== undefined && child.attributes.motion !== undefined) {
        throw new Error(`${child.name} cannot combine motion={Motion} with Pose children.`);
      }
      const motionName = authoredMotion === undefined && child.attributes.motion === undefined ? undefined : `clip-${clipSuffix}-motion`;
      if (motionName !== undefined) {
        if (authoredMotion !== undefined) {
          state.addRecord(motionName, `${trackId}.clip.${clipSuffix}.motion`, visualTrackTypes.motion, authoredMotion, child.range);
        } else {
          state.addReference(motionName, reference(child.attributes.motion, `${child.name}.motion`, visualTrackTypes.motion, resolveReference));
        }
      }
      const window = resolveTemporalWindowReference({ element: child, resolveReference });
      const directSource = declaredVisualSource(child, resolveReference, true)!;
      const sourceTime = sourceTimeFrom(child, resolveReference);
      const mapping = child.attributes.mapping === undefined ? undefined
        : reference(child.attributes.mapping, `${child.name}.mapping`, spatialTypes.map2D, resolveReference);
      if (mapping !== undefined && FIT_ATTRIBUTES.some((name) => child.attributes[name] !== undefined)) {
        throw new Error(`${child.name} cannot combine mapping={SpatialMap2D} with fit attributes.`);
      }
      validateUnitChildren(child);
      const layers = unitLayers(state, { trackId, unitSuffix: `clip-${clipSuffix}`, element: child, appearance: sourceAppearance,
        directSource, ...(sourceTime === undefined ? {} : { sourceTime }), ...(mapping === undefined ? {} : { mapping }),
        allowFramePaint: true }, resolveReference);
      const specName = `clip-${clipSuffix}-spec`;
      const windowName = `clip-${clipSuffix}-window`;
      state.addValue(windowName, window.ref, temporalTypes.window);
      state.addRecord(specName, `${trackId}.clip.${clipSuffix}.spec`, visualTrackTypes.clipSpec,
        decodeVisualClipSpec(treatment, { id, z: integerValue(child, "z") }), child.range);
      const frameName = `clip-${clipSuffix}-frame`;
      state.addReference(frameName, reference(child.attributes.frame, `${child.name}.frame`, spatialTypes.frame, resolveReference));
      clips.push({ suffix: clipSuffix, windowName, frameName, specName,
        ...(clipPathName === undefined ? {} : { clipPathName }),
        ...(motionName === undefined ? {} : { motionName }), layers });
      continue;
    }
    throw new Error(`${element.name} accepts only Clip children.`);
  }
  if (clips.length === 0) {
    throw new Error(`${element.name} requires at least one Clip.`);
  }
  const fragment = createVisualTrackSurfaceFragment(state.inputTypes, clips);
  return {
    records: state.records,
    components: [{
      id: trackId,
      fragment: fragment.id,
      inputs: state.inputs,
      outputs: { visual: `${trackId}.visual`, program: `${trackId}.program` },
      range: element.range,
    }],
    fragments: [fragment],
    exports: [
      `${trackId}.program`,
      `${trackId}.visual`,
    ],
  };
};