Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(auto-zoom): plan zooms as shots from recorded clicks

EtienneLescotb639cf1beb42
diff --git a/electron/app-settings.ts b/electron/app-settings.ts
index 510943ea4..2cc1cee25 100644
--- a/electron/app-settings.ts
+++ b/electron/app-settings.ts
@@ -14,7 +14,7 @@ export interface RecordingPreferences {
/** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */
hideDesktopIcons: boolean;
/**
- * Whether the editor decorates a fresh take with cursor-dwell zooms when it imports it.
+ * Whether the editor decorates a fresh take with automatic zooms when it imports it.
*
* Absent from a settings file written before this preference existed, which is why
* `parseRecording` reads a missing key as the default rather than as "off": the
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 7ce5f4d80..bd19f1059 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -652,7 +652,7 @@ export interface RecordingPrefs {
systemAudioEnabled: boolean;
cursorCaptureMode: CursorCaptureMode;
hideDesktopIcons: boolean;
- /** Whether a fresh take gets cursor-dwell zooms on import. Persisted; defaults on. */
+ /** Whether a fresh take gets automatic zooms on import. Persisted; defaults on. */
autoZoomEnabled: boolean;
}
const defaultRecordingPrefs: RecordingPrefs = {
@@ -3870,7 +3870,7 @@ export function registerIpcHandlers(
// do it. Publishing first opens a window where `getCurrentRecordingSession` hands
// the editor a take whose `.cursor.json` is not on disk yet, and the editor's
// fresh-take auto-zoom reads that file the moment it imports -- an empty read there
- // is indistinguishable from a take with no dwell, so the zooms are silently
+ // is indistinguishable from a take with no click, so the zooms are silently
// skipped.
await writePendingCursorTelemetry(screenVideoPath);
setCurrentRecordingSessionState(session);
diff --git a/src/components/ai-edition/recordingImport.test.ts b/src/components/ai-edition/recordingImport.test.ts
index 6ef1a4c62..4b1a314a6 100644
--- a/src/components/ai-edition/recordingImport.test.ts
+++ b/src/components/ai-edition/recordingImport.test.ts
@@ -1,5 +1,6 @@
// @vitest-environment jsdom
import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { CursorTelemetryPoint } from "@/components/video-editor/types";
import { replaceTimeline as replaceTimelineOp } from "@/lib/ai-edition/document/timeline";
import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema";
import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
@@ -258,19 +259,14 @@ describe("what the recording import leaves on the undo stack", () => {
});
});
-function dwell(
- centerMs: number,
- cx: number,
- cy: number,
- count = 6,
- spanMs = 900,
-): Array<{ timeMs: number; cx: number; cy: number }> {
- const step = spanMs / (count - 1);
- return Array.from({ length: count }, (_, i) => ({
- timeMs: centerMs - spanMs / 2 + i * step,
- cx,
- cy,
- }));
+// A click at `atMs`, in a take whose pointer moves on for 5.5 s more: the click is not its last.
+function clickAt(atMs: number, cx: number, cy: number): CursorTelemetryPoint[] {
+ return [
+ { timeMs: atMs - 300, cx, cy },
+ { timeMs: atMs, cx, cy, interactionType: "click" },
+ { timeMs: atMs + 300, cx, cy, interactionType: "mouseup" },
+ { timeMs: atMs + 5500, cx: 0.5, cy: 0.5 },
+ ];
}
const RECORDING_PATH = "C:\\recordings\\rec.mp4";
@@ -336,7 +332,7 @@ describe("fresh-recording auto-zoom", () => {
expect(consumeFreshRecordingAutoZoomPending()).toBe(false);
});
- // A system-cursor take writes no `.cursor.json`, so there is never a dwell to
+ // A system-cursor take writes no `.cursor.json`, so there is never a click to
// find. Marking it pending anyway armed the hand-off for a take that can never
// spend it.
it("does not mark a system-cursor take as waiting", async () => {
@@ -351,26 +347,26 @@ describe("fresh-recording auto-zoom", () => {
expect(consumeFreshRecordingAutoZoomPending()).toBe(true);
});
- it("applies cursor-dwell zooms once, after duration is known", async () => {
+ it("applies click zooms once, after duration is known", async () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
const next = await applyPendingFreshRecordingAutoZooms(documentWithClip(), {
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_test`,
});
expect(next.zoomRanges).toHaveLength(1);
expect(next.zoomRanges[0]).toMatchObject({
- startMs: 3424,
- endMs: 5424,
+ startMs: 3500,
+ endMs: 5500,
focusMode: "auto",
});
expect(await applyPendingFreshRecordingAutoZooms(next)).toBe(next);
});
- it("applies cursor-dwell zooms by default without requiring an enabled flag", async () => {
+ it("applies click zooms by default without requiring an enabled flag", async () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
const document = documentWithClip();
const next = await applyPendingFreshRecordingAutoZooms(document, {
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
createId: (prefix) => `${prefix}_test`,
});
expect(next.zoomRanges).toHaveLength(1);
@@ -384,7 +380,7 @@ describe("fresh-recording auto-zoom", () => {
stubRecordingPrefs({ autoZoomEnabled: false });
const document = documentWithClip();
const next = await applyPendingFreshRecordingAutoZooms(document, {
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
});
expect(next.zoomRanges).toEqual([]);
expect(consumeFreshRecordingAutoZoomPending()).toBe(false);
@@ -399,7 +395,7 @@ describe("fresh-recording auto-zoom", () => {
stubRecordingPrefsBridge(() => new Promise(() => undefined));
const next = await applyPendingFreshRecordingAutoZooms(documentWithClip(), {
prefsTimeoutMs: 10,
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
createId: (prefix) => `${prefix}_test`,
});
expect(next.zoomRanges).toHaveLength(1);
@@ -417,7 +413,7 @@ describe("fresh-recording auto-zoom", () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
stubRecordingPrefsBridge(getRecordingPrefs);
const next = await applyPendingFreshRecordingAutoZooms(documentWithClip(), {
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
createId: (prefix) => `${prefix}_test`,
});
expect(next.zoomRanges).toHaveLength(1);
@@ -435,7 +431,7 @@ describe("fresh-recording auto-zoom", () => {
return { autoZoomEnabled: true };
});
const next = await applyPendingFreshRecordingAutoZooms(documentWithClip(), {
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
createId: (prefix) => `${prefix}_test`,
});
expect(next).toBe(other);
@@ -446,7 +442,7 @@ describe("fresh-recording auto-zoom", () => {
// session, so the sidecar is on disk by the time a take can be imported. An empty
// read is therefore the take's real answer, not a mid-flush one, and leaving the
// hand-off armed would let it fire on the next document loaded in this window.
- it("consumes the hand-off when the sidecar holds no dwell", async () => {
+ it("consumes the hand-off when the sidecar holds no click", async () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
const document = documentWithClip();
const first = await applyPendingFreshRecordingAutoZooms(document, {
@@ -462,13 +458,13 @@ describe("fresh-recording auto-zoom", () => {
const document = createEmptyDocument({ projectId: "p_empty", title: "Recording" });
const next = await applyPendingFreshRecordingAutoZooms(document, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.5, 0.5),
+ getTelemetry: async () => clickAt(4000, 0.5, 0.5),
});
expect(next).toBe(document);
expect(consumeFreshRecordingAutoZoomPending()).toBe(true);
});
- it("consumes the hand-off when the cursor never sits still", async () => {
+ it("consumes the hand-off when the pointer only moves, never clicks", async () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
const document = documentWithClip();
const moving = Array.from({ length: 8 }, (_, i) => ({
@@ -485,7 +481,7 @@ describe("fresh-recording auto-zoom", () => {
});
// The one read that is not an answer: an error says nothing about whether the take
- // has a dwell, so the hand-off survives it.
+ // has a click, so the hand-off survives it.
it("keeps pending when telemetry read throws", async () => {
markFreshRecordingAutoZoomPending(RECORDING_PATH);
const document = documentWithClip();
@@ -504,7 +500,7 @@ describe("fresh-recording auto-zoom", () => {
const other = documentWithClip();
const next = await applyPendingFreshRecordingAutoZooms(other, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
});
expect(next).toBe(other);
expect(consumeFreshRecordingAutoZoomPending()).toBe(true);
@@ -547,10 +543,10 @@ describe("fresh-recording auto-zoom", () => {
markFreshRecordingAutoZoomPending(fresh.assets[0].originalPath);
const next = await applyPendingFreshRecordingAutoZooms(withLater, {
enabled: true,
- getTelemetry: async (videoPath) => (videoPath === laterPath ? dwell(14000, 0.4, 0.6) : []),
+ getTelemetry: async (videoPath) => (videoPath === laterPath ? clickAt(14000, 0.4, 0.6) : []),
createId: (prefix) => `${prefix}_later`,
});
- // The point of the test: the later clip's dwell is not read at all, so the fresh
+ // The point of the test: the later clip's click is not read at all, so the fresh
// take is never decorated with another asset's telemetry. Its own sidecar was
// empty, which is a real answer, so the hand-off is spent either way.
expect(next).toBe(withLater);
@@ -564,7 +560,7 @@ describe("fresh-recording auto-zoom", () => {
placeholder.assets[0].durationSec = undefined;
const first = await applyPendingFreshRecordingAutoZooms(placeholder, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
});
expect(first).toBe(placeholder);
expect(first.zoomRanges).toEqual([]);
@@ -596,7 +592,7 @@ describe("fresh-recording auto-zoom", () => {
enabled: true,
getTelemetry: async () => {
await gate;
- return dwell(4000, 0.4, 0.6);
+ return clickAt(4000, 0.4, 0.6);
},
createId: (prefix) => `${prefix}_live`,
});
@@ -621,7 +617,7 @@ describe("fresh-recording auto-zoom", () => {
await expect(
maybeSaveFreshRecordingAutoZooms(document, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_once`,
}),
).resolves.toBe(true);
@@ -635,7 +631,7 @@ describe("fresh-recording auto-zoom", () => {
await expect(
maybeSaveFreshRecordingAutoZooms(cleared, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
}),
).resolves.toBe(false);
expect(useProjectStore.getState().document?.zoomRanges).toEqual([]);
@@ -680,7 +676,7 @@ describe("fresh-recording auto-zoom", () => {
enabled: true,
getTelemetry: async () => {
await telemetryGate;
- return dwell(4000, 0.4, 0.6);
+ return clickAt(4000, 0.4, 0.6);
},
createId: (prefix) => `${prefix}_race`,
});
@@ -723,7 +719,7 @@ describe("fresh-recording auto-zoom", () => {
userSaveLanded = true;
await useProjectStore.getState().saveDocument(trimmed, { history: true });
}
- return dwell(4000, 0.4, 0.6);
+ return clickAt(4000, 0.4, 0.6);
},
createId: (prefix) => `${prefix}_contended`,
});
@@ -786,7 +782,7 @@ describe("fresh-recording auto-zoom", () => {
markFreshRecordingAutoZoomPending(stale.assets[0].originalPath);
const autoZoom = maybeSaveFreshRecordingAutoZooms(stale, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_after`,
});
await autoZoomStartedGate;
@@ -803,7 +799,7 @@ describe("fresh-recording auto-zoom", () => {
await expect(
maybeSaveFreshRecordingAutoZooms(wiped as AxcutDocument, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_rebase`,
}),
).resolves.toBe(true);
@@ -828,7 +824,7 @@ describe("fresh-recording auto-zoom", () => {
await expect(
maybeSaveFreshRecordingAutoZooms(doc, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_stuck`,
saveTimeoutMs: 10,
waitTimeoutMs: 10,
@@ -845,7 +841,7 @@ describe("fresh-recording auto-zoom", () => {
await expect(
maybeSaveFreshRecordingAutoZooms(doc, {
enabled: true,
- getTelemetry: async () => dwell(4000, 0.4, 0.6),
+ getTelemetry: async () => clickAt(4000, 0.4, 0.6),
createId: (prefix) => `${prefix}_after`,
saveTimeoutMs: 1_000,
waitTimeoutMs: 10,
diff --git a/src/components/ai-edition/recordingImport.ts b/src/components/ai-edition/recordingImport.ts
index 7a530dfda..b6b3e119a 100644
--- a/src/components/ai-edition/recordingImport.ts
+++ b/src/components/ai-edition/recordingImport.ts
@@ -34,7 +34,7 @@ import { nativeBridgeClient } from "@/native/client";
// `autoZoomEnabled`, default on). The ai-edition import only seeded a clip, so
// the wand still worked but a new take landed un-zoomed. This flag is the
// one-shot hand-off: set as soon as the asset is on the document, then taken on
-// the first pass that can give a real answer -- zooms written, no dwell in the
+// the first pass that can give a real answer -- zooms written, no click in the
// sidecar, or the toggle is off. What keeps it set is only ever "not enough of
// the document yet" (no clips, still the placeholder duration), and the
// `loadedmetadata` write is what comes back when that resolves.
@@ -194,7 +194,7 @@ export async function applyPendingFreshRecordingAutoZooms(
return await inner(videoPath);
} catch {
// An unreadable sidecar is NOT the same answer as an empty one: it says
- // nothing about whether this take has a dwell, so pending survives it and a
+ // nothing about whether this take has a click, so pending survives it and a
// later `loadedmetadata` for the same take gets another go.
telemetryFailed = true;
return [];
@@ -222,7 +222,7 @@ export async function applyPendingFreshRecordingAutoZooms(
if (collected.suggestions.length === 0) {
// A real answer, not a race: the stop handler awaits `writePendingCursorTelemetry`
// BEFORE it publishes the session, so by the time the editor can import the take
- // its sidecar is on disk. An empty read therefore means this take has no dwell to
+ // its sidecar is on disk. An empty read therefore means this take has no click to
// zoom, and no amount of retrying changes that -- consume the hand-off rather than
// leaving it armed for the next document loaded in this window.
if (!telemetryFailed) clearFreshRecordingAutoZoomPending();
@@ -265,7 +265,7 @@ export async function maybeSaveFreshRecordingAutoZooms(
// after `waitForDocumentSaves` has seen that writer finish.
if (current !== latest) return "contended";
// Nothing to write: the pass returned the document it was given, because a
- // guard refused it or the take has no dwell.
+ // guard refused it or the take has no click.
if (next === latest) return false;
const saved = await saveWithDeadline(
useProjectStore.getState().saveDocument(next, { history: true }),
@@ -349,7 +349,7 @@ export async function importPendingRecording(
// Except for a system-cursor take, which writes no `.cursor.json` at all: the
// toggle stays on in prefs (it is only disabled in the UI while that mode is
// picked), so without this the flag is set for a recording that can never produce
- // a dwell. What governs is the mode THIS take was recorded in, not the current
+ // a click. What governs is the mode THIS take was recorded in, not the current
// preference.
if (cursorCaptureMode !== "system") {
markFreshRecordingAutoZoomPending(screenPath);
diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx
index 1dc4cc361..634effef8 100644
--- a/src/components/ai-edition/v4/V4Timeline.tsx
+++ b/src/components/ai-edition/v4/V4Timeline.tsx
@@ -1533,8 +1533,8 @@ export function V4Timeline({
];
// Auto-enhance option 1 — the deterministic cursor-telemetry auto-zoom
- // (ported from main; NOT AI). Reads the recorded cursor movement and drops
- // zoom-ins on the dwell moments.
+ // (ported from main; NOT AI). Reads the clicks recorded with each take and
+ // plans zooms around them.
//
// Telemetry belongs to a RECORDING, not to a clip: it is fetched per asset and read in
// that asset's source time. Projecting it onto the ruler is `buildAutoZoomSuggestionsForClips`'
diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json
index 85b898011..c3c5df4e9 100644
--- a/src/i18n/locales/ar/timeline.json
+++ b/src/i18n/locales/ar/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "تحسين تلقائي",
"automaticZooms": "تكبيرات تلقائية",
- "automaticZoomsHint": "من حركة المؤشر المسجلة",
+ "automaticZoomsHint": "من النقرات المسجلة",
"smartZoomsAndCuts": "قصات ذكية",
"smartZoomsAndCutsHint": "باستخدام الذكاء الاصطناعي",
"timelineTools": "أدوات المخطط الزمني",
@@ -76,7 +76,7 @@
"dropToAdd": "أفلت للإضافة إلى المخطط الزمني",
"importRecordingFirst": "استورد تسجيلاً أولاً",
"noAutoZoomMoments": "لم يتم العثور على لحظات تكبير تلقائي",
- "noAutoZoomMomentsDescription": "لا يحتوي هذا التسجيل على بيانات حركة مؤشر، أو أن التكبيرات الحالية تغطي بالفعل اللحظات المزدحمة.",
+ "noAutoZoomMomentsDescription": "لا يحتوي هذا التسجيل على نقرات مسجلة، أو لا توجد نقرة تتسع لتكبير.",
"addedAutoZoom": "تمت إضافة {{count}} تكبير تلقائي",
"addedAutoZoomPlural": "تمت إضافة {{count}} تكبيرات تلقائية",
"autoZoomFailed": "فشل التكبير التلقائي",
diff --git a/src/i18n/locales/cs/timeline.json b/src/i18n/locales/cs/timeline.json
index b601092ab..033a978be 100644
--- a/src/i18n/locales/cs/timeline.json
+++ b/src/i18n/locales/cs/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Automatické vylepšení",
"automaticZooms": "Automatická přiblížení",
- "automaticZoomsHint": "Z zaznamenaného pohybu kurzoru",
+ "automaticZoomsHint": "Ze zaznamenaných kliknutí",
"smartZoomsAndCuts": "Chytré střihy",
"smartZoomsAndCutsHint": "S AI",
"timelineTools": "Nástroje časové osy",
@@ -76,7 +76,7 @@
"dropToAdd": "Přetáhněte pro přidání na časovou osu",
"importRecordingFirst": "Nejprve importujte nahrávku",
"noAutoZoomMoments": "Žádné momenty pro automatické přiblížení nenalezeny",
- "noAutoZoomMomentsDescription": "Tato nahrávka nemá data o pohybu kurzoru nebo existující přiblížení již pokrývají aktivní momenty.",
+ "noAutoZoomMomentsDescription": "Tato nahrávka neobsahuje žádná zaznamenaná kliknutí, nebo žádné, u kterého je místo pro přiblížení.",
"addedAutoZoom": "Přidáno {{count}} automatické přiblížení",
"addedAutoZoomPlural": "Přidáno {{count}} automatických přiblížení",
"autoZoomFailed": "Automatické přiblížení selhalo",
diff --git a/src/i18n/locales/de/timeline.json b/src/i18n/locales/de/timeline.json
index f98539dca..28b5ef036 100644
--- a/src/i18n/locales/de/timeline.json
+++ b/src/i18n/locales/de/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Automatisch verbessern",
"automaticZooms": "Automatische Zooms",
- "automaticZoomsHint": "Aus der aufgezeichneten Cursorbewegung",
+ "automaticZoomsHint": "Aus den aufgezeichneten Klicks",
"smartZoomsAndCuts": "Smarte Schnitte",
"smartZoomsAndCutsHint": "Mit KI",
"timelineTools": "Timeline-Werkzeuge",
@@ -76,7 +76,7 @@
"dropToAdd": "Ablegen, um zur Timeline hinzuzufügen",
"importRecordingFirst": "Importiere zuerst eine Aufnahme",
"noAutoZoomMoments": "Keine Auto-Zoom-Momente gefunden",
- "noAutoZoomMomentsDescription": "Diese Aufnahme enthält keine Cursordaten, oder bestehende Zooms decken die aktiven Momente bereits ab.",
+ "noAutoZoomMomentsDescription": "Diese Aufnahme enthält keine aufgezeichneten Klicks oder keinen, der Platz für einen Zoom lässt.",
"addedAutoZoom": "{{count}} automatischer Zoom hinzugefügt",
"addedAutoZoomPlural": "{{count}} automatische Zooms hinzugefügt",
"autoZoomFailed": "Auto-Zoom fehlgeschlagen",
diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json
index 8736be751..5a0267fb2 100644
--- a/src/i18n/locales/en/timeline.json
+++ b/src/i18n/locales/en/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Auto-enhance",
"automaticZooms": "Automatic zooms",
- "automaticZoomsHint": "From recorded cursor movement",
+ "automaticZoomsHint": "From recorded clicks",
"smartZoomsAndCuts": "Smart cuts",
"smartZoomsAndCutsHint": "With AI",
"timelineTools": "Timeline tools",
@@ -76,7 +76,7 @@
"dropToAdd": "Drop to add to timeline",
"importRecordingFirst": "Import a recording first",
"noAutoZoomMoments": "No auto-zoom moments found",
- "noAutoZoomMomentsDescription": "This recording has no cursor movement data, or existing zooms already cover the busy moments.",
+ "noAutoZoomMomentsDescription": "This recording has no recorded clicks, or none with room for a zoom.",
"addedAutoZoom": "Added {{count}} automatic zoom",
"addedAutoZoomPlural": "Added {{count}} automatic zooms",
"autoZoomFailed": "Auto-zoom failed",
diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json
index d3c259602..355eb3a76 100644
--- a/src/i18n/locales/es/timeline.json
+++ b/src/i18n/locales/es/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Mejora automática",
"automaticZooms": "Zooms automáticos",
- "automaticZoomsHint": "Del movimiento del cursor grabado",
+ "automaticZoomsHint": "De los clics grabados",
"smartZoomsAndCuts": "Cortes inteligentes",
"smartZoomsAndCutsHint": "Con IA",
"timelineTools": "Herramientas de la línea de tiempo",
@@ -76,7 +76,7 @@
"dropToAdd": "Suelta para añadir a la línea de tiempo",
"importRecordingFirst": "Importa una grabación primero",
"noAutoZoomMoments": "No se encontraron momentos para zoom automático",
- "noAutoZoomMomentsDescription": "Esta grabación no tiene datos de movimiento del cursor, o los zooms existentes ya cubren los momentos con actividad.",
+ "noAutoZoomMomentsDescription": "Esta grabación no tiene clics registrados, o ninguno deja espacio para un zoom.",
"addedAutoZoom": "Se añadió {{count}} zoom automático",
"addedAutoZoomPlural": "Se añadieron {{count}} zooms automáticos",
"autoZoomFailed": "Error en el zoom automático",
diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json
index 41ca79de3..b74ccb471 100644
--- a/src/i18n/locales/fr/timeline.json
+++ b/src/i18n/locales/fr/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Amélioration auto",
"automaticZooms": "Zooms automatiques",
- "automaticZoomsHint": "Basé sur le mouvement du curseur enregistré",
+ "automaticZoomsHint": "À partir des clics enregistrés",
"smartZoomsAndCuts": "Coupes intelligentes",
"smartZoomsAndCutsHint": "Avec l'IA",
"timelineTools": "Outils de la timeline",
@@ -76,7 +76,7 @@
"dropToAdd": "Déposez pour ajouter à la timeline",
"importRecordingFirst": "Importez d'abord un enregistrement",
"noAutoZoomMoments": "Aucun moment de zoom automatique trouvé",
- "noAutoZoomMomentsDescription": "Cet enregistrement n'a pas de données de mouvement du curseur, ou les zooms existants couvrent déjà les moments importants.",
+ "noAutoZoomMomentsDescription": "Cet enregistrement ne contient aucun clic enregistré, ou aucun qui laisse la place à un zoom.",
"addedAutoZoom": "{{count}} zoom automatique ajouté",
"addedAutoZoomPlural": "{{count}} zooms automatiques ajoutés",
"autoZoomFailed": "Échec du zoom automatique",
diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json
index 3cccfbaa0..b2fc550e9 100644
--- a/src/i18n/locales/it/timeline.json
+++ b/src/i18n/locales/it/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Miglioramento automatico",
"automaticZooms": "Zoom automatici",
- "automaticZoomsHint": "Dal movimento del cursore registrato",
+ "automaticZoomsHint": "Dai clic registrati",
"smartZoomsAndCuts": "Tagli intelligenti",
"smartZoomsAndCutsHint": "Con l'IA",
"timelineTools": "Strumenti della timeline",
@@ -76,7 +76,7 @@
"dropToAdd": "Rilascia per aggiungere alla timeline",
"importRecordingFirst": "Importa prima una registrazione",
"noAutoZoomMoments": "Nessun momento per lo zoom automatico trovato",
- "noAutoZoomMomentsDescription": "Questa registrazione non ha dati sul movimento del cursore, oppure gli zoom esistenti coprono già i momenti principali.",
+ "noAutoZoomMomentsDescription": "Questa registrazione non contiene clic registrati, oppure nessuno lascia spazio a uno zoom.",
"addedAutoZoom": "Aggiunto {{count}} zoom automatico",
"addedAutoZoomPlural": "Aggiunti {{count}} zoom automatici",
"autoZoomFailed": "Zoom automatico non riuscito",
diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json
index 2f71ce5f0..7d1647935 100644
--- a/src/i18n/locales/ja-JP/timeline.json
+++ b/src/i18n/locales/ja-JP/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "自動強化",
"automaticZooms": "自動ズーム",
- "automaticZoomsHint": "記録されたカーソルの動きから",
+ "automaticZoomsHint": "記録されたクリックから",
"smartZoomsAndCuts": "スマートカット",
"smartZoomsAndCutsHint": "AIを使用",
"timelineTools": "タイムラインツール",
@@ -76,7 +76,7 @@
"dropToAdd": "ドロップしてタイムラインに追加",
"importRecordingFirst": "先に録画をインポートしてください",
"noAutoZoomMoments": "自動ズームの瞬間が見つかりません",
- "noAutoZoomMomentsDescription": "この録画にはカーソルの動きのデータがないか、既存のズームがすでに重要な瞬間をカバーしています。",
+ "noAutoZoomMomentsDescription": "この録画には記録されたクリックがないか、ズームを入れる余地のあるクリックがありません。",
"addedAutoZoom": "自動ズームを {{count}} 件追加しました",
"addedAutoZoomPlural": "自動ズームを {{count}} 件追加しました",
"autoZoomFailed": "自動ズームに失敗しました",
diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json
index 198a06a06..9052eb096 100644
--- a/src/i18n/locales/ko-KR/timeline.json
+++ b/src/i18n/locales/ko-KR/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "자동 향상",
"automaticZooms": "자동 줌",
- "automaticZoomsHint": "녹화된 커서 움직임 기반",
+ "automaticZoomsHint": "녹화된 클릭 기반",
"smartZoomsAndCuts": "스마트 컷",
"smartZoomsAndCutsHint": "AI 사용",
"timelineTools": "타임라인 도구",
@@ -76,7 +76,7 @@
"dropToAdd": "타임라인에 추가하려면 놓으세요",
"importRecordingFirst": "먼저 녹화 파일을 가져오세요",
"noAutoZoomMoments": "자동 줌 순간을 찾을 수 없음",
- "noAutoZoomMomentsDescription": "이 녹화에는 커서 이동 데이터가 없거나 기존 줌이 이미 주요 순간을 다루고 있습니다.",
+ "noAutoZoomMomentsDescription": "이 녹화에는 기록된 클릭이 없거나, 줌을 넣을 여유가 있는 클릭이 없습니다.",
"addedAutoZoom": "자동 줌 {{count}}개가 추가되었습니다",
"addedAutoZoomPlural": "자동 줌 {{count}}개가 추가되었습니다",
"autoZoomFailed": "자동 줌 실패",
diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json
index 773e85770..0165dc7db 100644
--- a/src/i18n/locales/pt-BR/timeline.json
+++ b/src/i18n/locales/pt-BR/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Melhoria automática",
"automaticZooms": "Zooms automáticos",
- "automaticZoomsHint": "Do movimento do cursor gravado",
+ "automaticZoomsHint": "Dos cliques gravados",
"smartZoomsAndCuts": "Cortes inteligentes",
"smartZoomsAndCutsHint": "Com IA",
"timelineTools": "Ferramentas da linha do tempo",
@@ -76,7 +76,7 @@
"dropToAdd": "Solte para adicionar à linha do tempo",
"importRecordingFirst": "Importe uma gravação primeiro",
"noAutoZoomMoments": "Nenhum momento de zoom automático encontrado",
- "noAutoZoomMomentsDescription": "Esta gravação não tem dados de movimento do cursor, ou os zooms existentes já cobrem os momentos de destaque.",
+ "noAutoZoomMomentsDescription": "Esta gravação não tem cliques registrados, ou nenhum deixa espaço para um zoom.",
"addedAutoZoom": "{{count}} zoom automático adicionado",
"addedAutoZoomPlural": "{{count}} zooms automáticos adicionados",
"autoZoomFailed": "Falha no zoom automático",
diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json
index 10b870651..cd19dc201 100644
--- a/src/i18n/locales/ru/timeline.json
+++ b/src/i18n/locales/ru/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Авто-улучшение",
"automaticZooms": "Автоматические зумы",
- "automaticZoomsHint": "На основе записанного движения курсора",
+ "automaticZoomsHint": "На основе записанных кликов",
"smartZoomsAndCuts": "Умные вырезки",
"smartZoomsAndCutsHint": "С помощью ИИ",
"timelineTools": "Инструменты таймлайна",
@@ -76,7 +76,7 @@
"dropToAdd": "Отпустите, чтобы добавить на таймлайн",
"importRecordingFirst": "Сначала импортируйте запись",
"noAutoZoomMoments": "Моменты для автозума не найдены",
- "noAutoZoomMomentsDescription": "В этой записи нет данных о движении курсора, либо существующие зумы уже покрывают активные моменты.",
+ "noAutoZoomMomentsDescription": "В этой записи нет записанных кликов, или ни для одного из них нет места для зума.",
"addedAutoZoom": "Добавлен {{count}} автоматический зум",
"addedAutoZoomPlural": "Добавлено {{count}} автоматических зумов",
"autoZoomFailed": "Не удалось выполнить автозум",
diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json
index 3ea8d518b..39807b729 100644
--- a/src/i18n/locales/tr/timeline.json
+++ b/src/i18n/locales/tr/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Otomatik iyileştirme",
"automaticZooms": "Otomatik yakınlaştırmalar",
- "automaticZoomsHint": "Kaydedilen imleç hareketinden",
+ "automaticZoomsHint": "Kaydedilen tıklamalardan",
"smartZoomsAndCuts": "Akıllı kırpma",
"smartZoomsAndCutsHint": "Yapay zeka ile",
"timelineTools": "Zaman çizelgesi araçları",
@@ -76,7 +76,7 @@
"dropToAdd": "Zaman çizelgesine eklemek için bırakın",
"importRecordingFirst": "Önce bir kayıt içe aktarın",
"noAutoZoomMoments": "Otomatik yakınlaştırma anı bulunamadı",
- "noAutoZoomMomentsDescription": "Bu kayıtta imleç hareket verisi yok veya mevcut yakınlaştırmalar zaten yoğun anları kapsıyor.",
+ "noAutoZoomMomentsDescription": "Bu kayıtta kaydedilmiş tıklama yok ya da hiçbiri için yakınlaştırmaya yer yok.",
"addedAutoZoom": "{{count}} otomatik yakınlaştırma eklendi",
"addedAutoZoomPlural": "{{count}} otomatik yakınlaştırma eklendi",
"autoZoomFailed": "Otomatik yakınlaştırma başarısız oldu",
diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json
index 79c271c20..b4abac5e2 100644
--- a/src/i18n/locales/vi/timeline.json
+++ b/src/i18n/locales/vi/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "Tự động nâng cao",
"automaticZooms": "Thu phóng tự động",
- "automaticZoomsHint": "Từ chuyển động con trỏ đã ghi",
+ "automaticZoomsHint": "Từ các cú nhấp đã ghi",
"smartZoomsAndCuts": "Cắt thông minh",
"smartZoomsAndCutsHint": "Với AI",
"timelineTools": "Công cụ dòng thời gian",
@@ -76,7 +76,7 @@
"dropToAdd": "Thả để thêm vào dòng thời gian",
"importRecordingFirst": "Hãy nhập một bản ghi trước",
"noAutoZoomMoments": "Không tìm thấy khoảnh khắc thu phóng tự động",
- "noAutoZoomMomentsDescription": "Bản ghi này không có dữ liệu chuyển động con trỏ, hoặc các thu phóng hiện có đã bao phủ các khoảnh khắc bận rộn.",
+ "noAutoZoomMomentsDescription": "Bản ghi này không có cú nhấp nào được ghi lại, hoặc không có cú nhấp nào còn chỗ cho thu phóng.",
"addedAutoZoom": "Đã thêm {{count}} thu phóng tự động",
"addedAutoZoomPlural": "Đã thêm {{count}} thu phóng tự động",
"autoZoomFailed": "Thu phóng tự động thất bại",
diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json
index 81c62e474..63b0e0c41 100644
--- a/src/i18n/locales/zh-CN/timeline.json
+++ b/src/i18n/locales/zh-CN/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "自动增强",
"automaticZooms": "自动缩放",
- "automaticZoomsHint": "基于录制的光标移动",
+ "automaticZoomsHint": "基于录制的点击",
"smartZoomsAndCuts": "智能剪切",
"smartZoomsAndCutsHint": "使用 AI",
"timelineTools": "时间轴工具",
@@ -76,7 +76,7 @@
"dropToAdd": "拖放以添加到时间轴",
"importRecordingFirst": "请先导入录制内容",
"noAutoZoomMoments": "未找到自动缩放时刻",
- "noAutoZoomMomentsDescription": "此录制没有光标移动数据,或现有缩放已覆盖繁忙时刻。",
+ "noAutoZoomMomentsDescription": "此录制没有记录到点击,或没有点击留有缩放的空间。",
"addedAutoZoom": "已添加 {{count}} 个自动缩放",
"addedAutoZoomPlural": "已添加 {{count}} 个自动缩放",
"autoZoomFailed": "自动缩放失败",
diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json
index e15055444..de6dc2845 100644
--- a/src/i18n/locales/zh-TW/timeline.json
+++ b/src/i18n/locales/zh-TW/timeline.json
@@ -63,7 +63,7 @@
"toolbar": {
"autoEnhance": "自動加強",
"automaticZooms": "自動縮放",
- "automaticZoomsHint": "根據錄製的游標移動",
+ "automaticZoomsHint": "根據錄製的點擊",
"smartZoomsAndCuts": "智慧剪輯",
"smartZoomsAndCutsHint": "使用 AI",
"timelineTools": "時間軸工具",
@@ -76,7 +76,7 @@
"dropToAdd": "拖放以新增至時間軸",
"importRecordingFirst": "請先匯入錄製內容",
"noAutoZoomMoments": "找不到自動縮放時刻",
- "noAutoZoomMomentsDescription": "此錄製內容沒有游標移動資料,或現有縮放已涵蓋忙碌時刻。",
+ "noAutoZoomMomentsDescription": "此錄製內容沒有記錄到點擊,或沒有點擊留有縮放的空間。",
"addedAutoZoom": "已新增 {{count}} 個自動縮放",
"addedAutoZoomPlural": "已新增 {{count}} 個自動縮放",
"autoZoomFailed": "自動縮放失敗",
diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts
index b82f8dcf4..b72b924cb 100644
--- a/src/lib/ai-edition/store/useTimeline.test.ts
+++ b/src/lib/ai-edition/store/useTimeline.test.ts
@@ -1066,7 +1066,7 @@ describe("useTimeline save failures", () => {
let added: number | undefined;
await act(async () => {
added = await result.current.addZoomsBulk([
- { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 } },
+ { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 }, depth: 3 },
]);
});
@@ -2252,7 +2252,7 @@ describe("useTimeline.addZoomsBulk reads the document at write time", () => {
let added: number | undefined;
await act(async () => {
added = await addZoomsBulk([
- { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 } },
+ { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 }, depth: 3 },
]);
});
@@ -2286,7 +2286,7 @@ describe("useTimeline.addZoomsBulk reads the document at write time", () => {
let added: number | undefined;
await act(async () => {
added = await addZoomsBulk([
- { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 } },
+ { span: { start: 1000, end: 2000 }, focus: { cx: 0.5, cy: 0.5 }, depth: 3 },
]);
});
diff --git a/src/lib/ai-edition/timeline/apply-auto-zooms.test.ts b/src/lib/ai-edition/timeline/apply-auto-zooms.test.ts
index 99226150b..32de4396b 100644
--- a/src/lib/ai-edition/timeline/apply-auto-zooms.test.ts
+++ b/src/lib/ai-edition/timeline/apply-auto-zooms.test.ts
@@ -7,19 +7,14 @@ import {
collectAutoZoomSuggestionsForLatestDocument,
} from "./apply-auto-zooms";
-function dwell(
- centerMs: number,
- cx: number,
- cy: number,
- count = 6,
- spanMs = 900,
-): CursorTelemetryPoint[] {
- const step = spanMs / (count - 1);
- return Array.from({ length: count }, (_, i) => ({
- timeMs: centerMs - spanMs / 2 + i * step,
- cx,
- cy,
- }));
+// A click at `atMs`, in a take whose pointer moves on until 9.5 s: the click is not its last.
+function clickAt(atMs: number, cx: number, cy: number): CursorTelemetryPoint[] {
+ return [
+ { timeMs: atMs - 300, cx, cy },
+ { timeMs: atMs, cx, cy, interactionType: "click" },
+ { timeMs: atMs + 300, cx, cy, interactionType: "mouseup" },
+ { timeMs: 9500, cx: 0.5, cy: 0.5 },
+ ];
}
function documentWithClip(durationSec = 10): AxcutDocument {
@@ -60,13 +55,11 @@ describe("collectAutoZoomSuggestionsForDocument", () => {
it("builds suggestions from the asset's cursor sidecar", async () => {
const document = documentWithClip();
const suggestions = await collectAutoZoomSuggestionsForDocument(document, async () =>
- dwell(4000, 0.4, 0.6),
+ clickAt(4000, 0.4, 0.6),
);
- expect(suggestions).toHaveLength(1);
- // Centred on the dwell would be 3000..5000; the wide opening holds it to 3424.
- expect(suggestions[0].span.start).toBe(3424);
- expect(suggestions[0].span.end).toBe(5424);
- expect(suggestions[0].focus.cx).toBeCloseTo(0.4, 5);
+ expect(suggestions).toEqual([
+ { span: { start: 3500, end: 5500 }, focus: { cx: 0.4, cy: 0.6 }, depth: 3 },
+ ]);
});
it("asks for telemetry on the asset path, not a file URL", async () => {
@@ -82,7 +75,48 @@ describe("collectAutoZoomSuggestionsForDocument", () => {
it("returns nothing without a clip window", async () => {
const document = createEmptyDocument({ projectId: "p1", title: "Empty" });
const suggestions = await collectAutoZoomSuggestionsForDocument(document, async () =>
- dwell(1000, 0.5, 0.5),
+ clickAt(4000, 0.5, 0.5),
+ );
+ expect(suggestions).toEqual([]);
+ });
+
+ it("plans every take on the timeline in one pass, so their zooms keep clear of each other", async () => {
+ // Two takes back to back: each click's zoom is fine alone, but the pair would zoom out
+ // at the cut and straight back in. Planned per take, both would stay.
+ const first = documentWithClip();
+ const document: AxcutDocument = {
+ ...first,
+ assets: [
+ ...first.assets,
+ { ...first.assets[0], id: "asset_2", originalPath: "C:\\recordings\\second.mp4" },
+ ],
+ timeline: {
+ ...first.timeline,
+ clips: [
+ ...first.timeline.clips,
+ {
+ ...first.timeline.clips[0],
+ id: "clip_2",
+ assetId: "asset_2",
+ timelineStartSec: 10,
+ timelineEndSec: 20,
+ },
+ ],
+ },
+ };
+ const suggestions = await collectAutoZoomSuggestionsForDocument(document, async (videoPath) =>
+ videoPath.endsWith("second.mp4") ? clickAt(1000, 0.5, 0.5) : clickAt(7000, 0.5, 0.5),
+ );
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 6500, end: 8500 }]);
+ });
+
+ it("keeps clear of the zooms already on the timeline", async () => {
+ const document: AxcutDocument = {
+ ...documentWithClip(),
+ zoomRanges: [{ id: "z1", startMs: 3000, endMs: 5000, depth: 3, focus: { cx: 0.5, cy: 0.5 } }],
+ };
+ const suggestions = await collectAutoZoomSuggestionsForDocument(document, async () =>
+ clickAt(6500, 0.4, 0.6),
);
expect(suggestions).toEqual([]);
});
@@ -98,7 +132,7 @@ describe("appendAutoZoomSuggestions", () => {
const document = documentWithClip();
const next = appendAutoZoomSuggestions(
document,
- [{ span: { start: 3000, end: 5000 }, focus: { cx: 0.4, cy: 0.6 } }],
+ [{ span: { start: 3000, end: 5000 }, focus: { cx: 0.4, cy: 0.6 }, depth: 2 }],
(prefix) => `${prefix}_fixed`,
);
expect(next).not.toBe(document);
@@ -106,7 +140,7 @@ describe("appendAutoZoomSuggestions", () => {
expect(next.zoomRanges[0]).toMatchObject({
startMs: 3000,
endMs: 5000,
- depth: 3,
+ depth: 2,
focusMode: "auto",
clipId: "clip_1",
});
diff --git a/src/lib/ai-edition/timeline/apply-auto-zooms.ts b/src/lib/ai-edition/timeline/apply-auto-zooms.ts
index ae349f63d..bc9d02196 100644
--- a/src/lib/ai-edition/timeline/apply-auto-zooms.ts
+++ b/src/lib/ai-edition/timeline/apply-auto-zooms.ts
@@ -6,10 +6,9 @@ import type { CursorTelemetryPoint } from "@/components/video-editor/types";
import { createId } from "../document/ids";
import type { AxcutAsset, AxcutDocument } from "../schema";
import { anchorRegionsWithDerivedMs } from "./timelineMap";
+import { effectiveZoomScale } from "./zoom-scale";
import { type AutoZoomSuggestion, buildAutoZoomSuggestionsForClips } from "./zoom-suggestions";
-export const AUTO_ZOOM_DEFAULT_DURATION_MS = 2000;
-
/** Which assets to read telemetry for. The wand takes every video on the document;
* the fresh-recording import narrows it to the take it is pending on, so a project
* that already holds other footage does not pay an IPC round trip per asset. */
@@ -20,10 +19,6 @@ export async function collectAutoZoomSuggestionsForDocument(
getTelemetry: (videoPath: string) => Promise<CursorTelemetryPoint[] | null | undefined>,
includeAsset: AutoZoomAssetFilter = () => true,
): Promise<AutoZoomSuggestion[]> {
- const existingRegions = document.zoomRanges.map((region) => ({
- startMs: region.startMs,
- endMs: region.endMs,
- }));
const assetsWithClips = document.assets.filter(
(asset) =>
asset.kind === "video" &&
@@ -31,19 +26,23 @@ export async function collectAutoZoomSuggestionsForDocument(
includeAsset(asset) &&
document.timeline.clips.some((clip) => clip.assetId === asset.id),
);
- const perSource = await Promise.all(
- assetsWithClips.map(async (asset) => {
- const telemetry = (await getTelemetry(asset.originalPath)) ?? [];
- return buildAutoZoomSuggestionsForClips({
- cursorTelemetry: telemetry,
- assetId: asset.id,
- clips: document.timeline.clips,
- existingRegions,
- defaultDurationMs: AUTO_ZOOM_DEFAULT_DURATION_MS,
- });
- }),
+ // Read in parallel, planned in one pass: two recordings' zooms must keep clear of each other too.
+ const telemetryByAssetId = new Map(
+ await Promise.all(
+ assetsWithClips.map(
+ async (asset) => [asset.id, (await getTelemetry(asset.originalPath)) ?? []] as const,
+ ),
+ ),
);
- return perSource.flat();
+ return buildAutoZoomSuggestionsForClips({
+ telemetryByAssetId,
+ clips: document.timeline.clips,
+ existingRegions: document.zoomRanges.map((region) => ({
+ startMs: region.startMs,
+ endMs: region.endMs,
+ scale: effectiveZoomScale(region),
+ })),
+ });
}
/**
@@ -110,7 +109,7 @@ export function appendAutoZoomSuggestions(
id: makeId("zoom"),
startMs: Math.round(suggestion.span.start),
endMs: Math.round(suggestion.span.end),
- depth: 3 as const,
+ depth: suggestion.depth,
focus: { cx: suggestion.focus.cx, cy: suggestion.focus.cy },
focusMode: "auto" as const,
},
diff --git a/src/lib/ai-edition/timeline/cursor-track.ts b/src/lib/ai-edition/timeline/cursor-track.ts
index aec4324a4..b82db3648 100644
--- a/src/lib/ai-edition/timeline/cursor-track.ts
+++ b/src/lib/ai-edition/timeline/cursor-track.ts
@@ -3,14 +3,14 @@
//
// ponytail: this is an OBSERVATION, not an interpretation, and the distinction
// is the whole point of the module. Its predecessor handed the model a list of
-// "dwell moments" computed by the same stillness detector that drives the magic
+// "dwell moments" computed by the same stillness detector that then drove the magic
// wand. That reads as helpful and is not: it caps the model at the detector's
// recall. Measured on a real 66s screencast, the detector reports 6 of 6
// annotated interest zones but 8 false positives out of 16 — and it is blind by
// construction to the one zone where the user traced slowly across an image
// while narrating, because the cursor genuinely travelled 30% of the frame. A
-// model fed that digest can never zoom there, however good it is. So the wand
-// keeps its detector, the model gets the track, and the bench compares them.
+// model fed that digest can never zoom there, however good it is. So the model
+// gets the track, and the bench compares the two. (The wand has since moved to clicks.)
//
// Downsampling is resolution, not interpretation: every kept point is a real
// sample, nothing is summarised, and every pointer-shape change survives the
diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
index 9d23aca68..14e7258ef 100644
--- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
+++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
@@ -1,27 +1,7 @@
import { describe, expect, it } from "vitest";
import type { CursorTelemetryPoint } from "@/components/video-editor/types";
import type { AxcutClip } from "../schema";
-import {
- buildAutoZoomSuggestions,
- buildAutoZoomSuggestionsForClips,
- detectZoomDwellCandidates,
-} from "./zoom-suggestions";
-
-// A dwell = many samples clustered in time at (nearly) the same position.
-function dwell(
- centerMs: number,
- cx: number,
- cy: number,
- count = 6,
- spanMs = 900,
-): CursorTelemetryPoint[] {
- const step = spanMs / (count - 1);
- return Array.from({ length: count }, (_, i) => ({
- timeMs: centerMs - spanMs / 2 + i * step,
- cx,
- cy,
- }));
-}
+import { buildAutoZoomSuggestions, buildAutoZoomSuggestionsForClips } from "./zoom-suggestions";
// A click = a single sample carrying recorded interaction metadata (issue #699).
function click(
@@ -33,122 +13,90 @@ function click(
return { timeMs, cx, cy, interactionType };
}
-describe("detectZoomDwellCandidates", () => {
- it("finds a dwell where the cursor sits still", () => {
- const candidates = detectZoomDwellCandidates(dwell(1000, 0.4, 0.6));
- expect(candidates).toHaveLength(1);
- expect(candidates[0].focus.cx).toBeCloseTo(0.4, 5);
- expect(candidates[0].focus.cy).toBeCloseTo(0.6, 5);
- });
+// The pointer parked in one place, with no click.
+function park(centerMs: number, cx: number, cy: number, count = 6, spanMs = 900) {
+ const step = spanMs / (count - 1);
+ return Array.from({ length: count }, (_, i) => ({
+ timeMs: centerMs - spanMs / 2 + i * step,
+ cx,
+ cy,
+ }));
+}
- it("ignores a fast sweep across the screen (no dwell)", () => {
- const samples: CursorTelemetryPoint[] = Array.from({ length: 10 }, (_, i) => ({
- timeMs: i * 100,
- cx: i / 10,
- cy: i / 10,
- }));
- expect(detectZoomDwellCandidates(samples)).toHaveLength(0);
- });
-});
+const plan = (cursorTelemetry: CursorTelemetryPoint[], totalMs = 5000) =>
+ buildAutoZoomSuggestions({ cursorTelemetry, totalMs, existingRegions: [] });
-describe("buildAutoZoomSuggestions", () => {
+describe("buildAutoZoomSuggestions: which clicks count", () => {
it("ignores a click made while the cursor is hidden", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [{ ...click(2000, 0.5, 0.5), visible: false }],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toEqual([]);
+ expect(plan([{ ...click(2000, 0.5, 0.5), visible: false }])).toEqual([]);
});
it("ignores a visible mouse-down when the cursor hides during that drag", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
- { ...click(1000, 0.5, 0.5), visible: true },
- { timeMs: 1016, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
- { timeMs: 1600, cx: 0.7, cy: 0.5, visible: false, interactionType: "mouseup" },
- ],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
+ const suggestions = plan([
+ { ...click(1000, 0.5, 0.5), visible: true },
+ { timeMs: 1016, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
+ { timeMs: 1600, cx: 0.7, cy: 0.5, visible: false, interactionType: "mouseup" },
+ ]);
expect(suggestions).toEqual([]);
});
it("keeps a click released before the cursor later hides", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
- { ...click(1000, 0.5, 0.5), visible: true },
- { timeMs: 1050, cx: 0.5, cy: 0.5, visible: true, interactionType: "mouseup" },
- { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
- ],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 });
+ const suggestions = plan([
+ { ...click(1000, 0.5, 0.5), visible: true },
+ { timeMs: 1050, cx: 0.5, cy: 0.5, visible: true, interactionType: "mouseup" },
+ { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
+ ]);
+ expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.5, cy: 0.5 }]);
});
it("keeps a later visible click after a hidden drag", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
+ const suggestions = plan(
+ [
{ ...click(1000, 0.5, 0.5), visible: true },
{ timeMs: 1100, cx: 0.6, cy: 0.5, visible: false, interactionType: "move" },
{ timeMs: 1600, cx: 0.7, cy: 0.5, visible: true, interactionType: "mouseup" },
{ ...click(4000, 0.2, 0.8), visible: true },
],
- totalMs: 6000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 });
+ 6000,
+ );
+ expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.2, cy: 0.8 }]);
});
it("keeps a legacy click without mouseup when the cursor hides much later", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
+ const suggestions = plan(
+ [
{ ...click(1000, 0.2, 0.8), visible: true },
{ timeMs: 1050, cx: 0.3, cy: 0.7, visible: true, interactionType: "move" },
{ timeMs: 2000, cx: 0.7, cy: 0.3, visible: false, interactionType: "move" },
],
- totalMs: 4000,
- existingRegions: [],
- defaultDurationMs: 1000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 });
+ 4000,
+ );
+ expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.2, cy: 0.8 }]);
});
it("rejects a legacy click when the cursor hides on the next sampler tick", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
+ const suggestions = plan(
+ [
{ ...click(1000, 0.2, 0.8), visible: true },
{ timeMs: 1033, cx: 0.25, cy: 0.75, visible: false, interactionType: "move" },
],
- totalMs: 4000,
- existingRegions: [],
- defaultDurationMs: 1000,
- });
+ 4000,
+ );
expect(suggestions).toEqual([]);
});
it("keeps a tap whose release shares its press sample when the cursor hides later", () => {
// The take records mouse-ups, but the second tap was released within one sampler tick.
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
+ const suggestions = plan(
+ [
{ ...click(500, 0.8, 0.2), visible: true },
{ timeMs: 600, cx: 0.8, cy: 0.2, visible: true, interactionType: "mouseup" },
- { ...click(3000, 0.2, 0.8), visible: true },
- { timeMs: 3500, cx: 0.2, cy: 0.8, visible: false, interactionType: "move" },
+ { ...click(9000, 0.2, 0.8), visible: true },
+ { timeMs: 9500, cx: 0.2, cy: 0.8, visible: false, interactionType: "move" },
],
- totalMs: 6000,
- existingRegions: [],
- defaultDurationMs: 1000,
- });
- expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([
+ 12000,
+ );
+ expect(suggestions.map((s) => s.focus)).toEqual([
{ cx: 0.8, cy: 0.2 },
{ cx: 0.2, cy: 0.8 },
]);
@@ -156,320 +104,163 @@ describe("buildAutoZoomSuggestions", () => {
it("keeps a click whose mouse-up sample is already hidden", () => {
// Released, then typing hid the pointer before the next sampler tick.
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
+ const suggestions = plan(
+ [
{ ...click(1000, 0.5, 0.5), visible: true },
{ timeMs: 1033, cx: 0.5, cy: 0.5, visible: false, interactionType: "mouseup" },
],
- totalMs: 4000,
- existingRegions: [],
- defaultDurationMs: 1000,
- });
- expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([{ cx: 0.5, cy: 0.5 }]);
- });
-
- it("does not join visible samples across a hidden interval into a dwell", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
- { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true },
- { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true },
- { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false },
- { timeMs: 1800, cx: 0.5, cy: 0.5, visible: true },
- { timeMs: 2000, cx: 0.5, cy: 0.5, visible: true },
- ],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toEqual([]);
- });
-
- it("ignores a dwell recorded while the cursor is hidden", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: dwell(2000, 0.5, 0.5).map((sample) => ({
- ...sample,
- visible: false,
- })),
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toEqual([]);
- });
-
- it("keeps a visible dwell beside hidden cursor samples", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
- ...dwell(1000, 0.2, 0.8),
- { timeMs: 2000, cx: 0.2, cy: 0.8, visible: false },
- ],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus.cx).toBeCloseTo(0.2);
- expect(suggestions[0].focus.cy).toBeCloseTo(0.8);
+ 4000,
+ );
+ expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.5, cy: 0.5 }]);
});
- it("returns a centered span around each accepted dwell", () => {
- const telemetry = dwell(2000, 0.5, 0.5);
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: telemetry,
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- // centred on ~2000ms with a 2000ms default → ~1000..3000
- expect(suggestions[0].span.start).toBe(1000);
- expect(suggestions[0].span.end).toBe(3000);
- });
-
- it("drops candidates overlapping an existing zoom region", () => {
- const telemetry = dwell(2000, 0.5, 0.5);
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: telemetry,
- totalMs: 5000,
- existingRegions: [{ startMs: 1500, endMs: 2500 }],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(0);
- });
-
- it("spaces two dwells and returns both when far apart", () => {
- const telemetry = [...dwell(1500, 0.2, 0.2), ...dwell(6000, 0.8, 0.8)];
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: telemetry,
- totalMs: 9000,
- existingRegions: [],
- defaultDurationMs: 1500,
- });
- expect(suggestions.length).toBe(2);
- });
-
- it("returns nothing without telemetry", () => {
- expect(
- buildAutoZoomSuggestions({
- cursorTelemetry: [],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- }),
- ).toEqual([]);
+ it("accepts every click kind the telemetry records", () => {
+ for (const kind of ["click", "double-click", "right-click", "middle-click"] as const) {
+ const suggestions = plan([click(2000, 0.3, 0.7, kind), { timeMs: 4000, cx: 0.4, cy: 0.6 }]);
+ expect(
+ suggestions.map((s) => s.focus),
+ kind,
+ ).toEqual([{ cx: 0.3, cy: 0.7 }]);
+ }
});
-});
-describe("recorded clicks (issue #699)", () => {
- // A fast sweep forms no dwell; before #699 this take produced no suggestion at all.
- const sweepWithClickAt = (clickIndex: number): CursorTelemetryPoint[] =>
- Array.from({ length: 10 }, (_, i) => ({
+ it("ignores move and mouseup: they are not clicks", () => {
+ const telemetry: CursorTelemetryPoint[] = Array.from({ length: 10 }, (_, i) => ({
timeMs: i * 500,
cx: i / 10,
cy: i / 10,
- interactionType: i === clickIndex ? "click" : "move",
+ interactionType: i === 4 ? "mouseup" : "move",
}));
-
- it("zooms on a click made while the pointer is still moving", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: sweepWithClickAt(4),
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.4, cy: 0.4 });
- expect(suggestions[0].span).toEqual({ start: 1000, end: 3000 });
+ expect(plan(telemetry)).toEqual([]);
});
- it("focuses the zoom on the click sample, not the pointer's average position", () => {
- // The pointer flew to a toolbar button, clicked, flew back.
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [
- { timeMs: 1000, cx: 0.1, cy: 0.9 },
- click(1200, 0.8, 0.2),
- { timeMs: 1400, cx: 0.12, cy: 0.88 },
- ],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.8, cy: 0.2 });
- expect(suggestions[0].span).toEqual({ start: 200, end: 2200 });
+ it("places no zoom where the pointer only sits still", () => {
+ expect(plan(park(2000, 0.5, 0.5))).toEqual([]);
});
- it("accepts every click kind the telemetry records", () => {
- for (const kind of ["click", "double-click", "right-click", "middle-click"] as const) {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [click(2000, 0.3, 0.7, kind), { timeMs: 4000, cx: 0.4, cy: 0.6 }],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions, kind).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.3, cy: 0.7 });
+ it("returns nothing without telemetry, or for a lone move or mouseup", () => {
+ expect(plan([])).toEqual([]);
+ for (const kind of ["move", "mouseup"] as const) {
+ expect(plan([{ timeMs: 2000, cx: 0.3, cy: 0.7, interactionType: kind }]), kind).toEqual([]);
}
});
+});
- it("ignores move and mouseup — they are not clicks", () => {
+describe("buildAutoZoomSuggestions: shots", () => {
+ it("is fully in half a second before the click and holds 1.5 s after it", () => {
+ // A fast sweep: the click is made while the pointer is still moving.
const telemetry: CursorTelemetryPoint[] = Array.from({ length: 10 }, (_, i) => ({
timeMs: i * 500,
cx: i / 10,
cy: i / 10,
- interactionType: i === 4 ? "mouseup" : "move",
+ interactionType: i === 4 ? "click" : "move",
}));
- expect(
- buildAutoZoomSuggestions({
- cursorTelemetry: telemetry,
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- }),
- ).toEqual([]);
+ expect(plan(telemetry)).toEqual([
+ { span: { start: 1500, end: 3500 }, focus: { cx: 0.4, cy: 0.4 }, depth: 3 },
+ ]);
});
- it("folds a nearby dwell into the click so one moment yields ONE zoom", () => {
- // The dwell run 1550..2270ms centres on 1910ms — 490ms from the click at 2400ms,
- // inside SUGGESTION_SPACING_MS. The click's own time and position anchor the zoom.
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [...dwell(2000, 0.2, 0.8), click(2400, 0.3, 0.75)],
- totalMs: 6000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
+ it("focuses the zoom on the click, not on the pointer's path", () => {
+ // The pointer flew to a toolbar button, clicked, flew back.
+ const suggestions = plan([
+ { timeMs: 1000, cx: 0.1, cy: 0.9 },
+ click(1200, 0.8, 0.2),
+ { timeMs: 1400, cx: 0.12, cy: 0.88 },
+ ]);
+ expect(suggestions.map((s) => [s.span, s.focus])).toEqual([
+ [
+ { start: 700, end: 2700 },
+ { cx: 0.8, cy: 0.2 },
+ ],
+ ]);
+ });
+
+ it("frames a burst of nearby clicks with one zoom that holds through it", () => {
+ const suggestions = plan(
+ [click(2000, 0.3, 0.3), click(3500, 0.36, 0.33), click(5000, 0.4, 0.35)],
+ 9000,
+ );
expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.3, cy: 0.75 });
- expect(suggestions[0].span).toEqual({ start: 1400, end: 3400 });
+ expect(suggestions[0].span).toEqual({ start: 1500, end: 6500 });
+ expect(suggestions[0].depth).toBe(3);
+ expect(suggestions[0].focus.cx).toBeCloseTo(0.35, 5);
+ expect(suggestions[0].focus.cy).toBeCloseTo(0.325, 5);
});
- it("keeps a dwell that sits clear of the click, and both survive", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [...dwell(2000, 0.2, 0.8), click(6000, 0.9, 0.1)],
- totalMs: 9000,
- existingRegions: [],
- defaultDurationMs: 1500,
- });
- // Nobody conflicts here, so the output order is an acceptance-order detail;
- // compare the SET of spans and the click-anchored focus.
- const spans = suggestions.map((s) => s.span).sort((a, b) => a.start - b.start);
- expect(spans).toEqual([
- { start: 1250, end: 2750 },
- { start: 5250, end: 6750 },
- ]);
- expect(suggestions.find((s) => s.span.start === 5250)?.focus).toEqual({ cx: 0.9, cy: 0.1 });
- expect(suggestions.find((s) => s.span.start === 1250)?.focus.cx).toBeCloseTo(0.2, 5);
+ it("zooms in less when the clicks it frames spread wider", () => {
+ // With a 12% margin each side, 0.35 of the frame fits at 1.5× and 0.5 only at 1.25×.
+ const wider = plan([click(2000, 0.2, 0.5), click(3000, 0.55, 0.5)], 9000);
+ expect(wider.map((s) => [s.span, s.depth])).toEqual([[{ start: 1500, end: 4500 }, 2]]);
+ const widest = plan([click(2000, 0.2, 0.5), click(3000, 0.7, 0.5)], 9000);
+ expect(widest.map((s) => s.depth)).toEqual([1]);
});
- it("collapses a rapid burst of clicks into the first click's zoom", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [click(1000, 0.2, 0.2), click(1500, 0.25, 0.25)],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.2 });
- expect(suggestions[0].span).toEqual({ start: 0, end: 2000 });
- });
-
- it("still drops a click that overlaps an existing zoom region", () => {
- expect(
- buildAutoZoomSuggestions({
- cursorTelemetry: [click(2000, 0.5, 0.5), { timeMs: 4000, cx: 0.6, cy: 0.4 }],
- totalMs: 5000,
- existingRegions: [{ startMs: 1500, endMs: 2500 }],
- defaultDurationMs: 2000,
- }),
- ).toEqual([]);
- });
-
- it("keeps the dwell-only behaviour when the take records no clicks", () => {
- // A dwell annotated only with move/mouseup: no click candidate, exactly the
- // pre-#699 result — and the mouseup does not break the still run either.
- const dwellRun = dwell(2000, 0.5, 0.5).map((sample, i) => ({
- ...sample,
- interactionType: (i === 3 ? "mouseup" : "move") as "mouseup" | "move",
- }));
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: dwellRun,
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].span).toEqual({ start: 1000, end: 3000 });
+ it("never zooms straight back: of two clashing zooms, the one with more clicks stays", () => {
+ // Too far apart to share a zoom, too close in time for a wide shot between them.
+ const suggestions = plan(
+ [click(2000, 0.1, 0.1), click(2600, 0.12, 0.12), click(3200, 0.9, 0.9)],
+ 9000,
+ );
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 1500, end: 4100 }]);
});
- it("clamps a click near the start or end of the take to the ruler", () => {
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [click(100, 0.1, 0.1), click(4800, 0.9, 0.9)],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- });
+ it("holds one zoom through a gap too short for a wide shot, when its clicks fit together", () => {
+ // 4 s apart: two groups, but coming back out and in within 2 s would read as a yo-yo.
+ const suggestions = plan([click(2000, 0.3, 0.3), click(6000, 0.34, 0.3)], 9000);
+ expect(suggestions.map((s) => [s.span, s.depth])).toEqual([[{ start: 1500, end: 7500 }, 3]]);
+ });
+
+ it("leaves a real wide shot between two zooms", () => {
+ const suggestions = plan([click(2000, 0.1, 0.1), click(9000, 0.9, 0.9)], 12000);
expect(suggestions.map((s) => s.span)).toEqual([
- { start: 0, end: 2000 },
- { start: 3000, end: 5000 },
+ { start: 1500, end: 3500 },
+ { start: 8500, end: 10500 },
]);
});
- it("keeps the dwell fallback when an existing region rejects only the nearby click", () => {
- // The click at 2700ms sits inside SUGGESTION_SPACING_MS of the dwell at 1000ms,
- // but the existing region 2300..2400 overlaps only the CLICK's span. The click
- // is rejected, and a rejected click must leave the dwell standing: one zoom at
- // the dwell, not zero. (The first cut of this PR removed the dwell up front and
- // returned nothing here.)
- const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [...dwell(1000, 0.2, 0.8), click(2700, 0.6, 0.4)],
- totalMs: 5000,
- existingRegions: [{ startMs: 2300, endMs: 2400 }],
- defaultDurationMs: 1000,
+ it("keeps clear of a zoom already on the timeline, its ramps and a wide shot included", () => {
+ const existingRegions = [{ startMs: 3000, endMs: 5000 }];
+ const near = buildAutoZoomSuggestions({
+ cursorTelemetry: [click(7000, 0.5, 0.5)],
+ totalMs: 12000,
+ existingRegions,
});
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].span).toEqual({ start: 500, end: 1500 });
- expect(suggestions[0].focus.cx).toBeCloseTo(0.2, 5);
- expect(suggestions[0].focus.cy).toBeCloseTo(0.8, 5);
+ expect(near).toEqual([]);
+ const clear = buildAutoZoomSuggestions({
+ cursorTelemetry: [click(9000, 0.5, 0.5)],
+ totalMs: 12000,
+ existingRegions,
+ });
+ expect(clear.map((s) => s.span)).toEqual([{ start: 8500, end: 10500 }]);
});
- it("lets a second click survive the first being rejected by an existing region", () => {
- // Two clicks within spacing of each other; only the first's span overlaps the
- // existing region. The rejected first click must not suppress the second.
+ it("keeps clear of a placed zoom that reaches past the next one", () => {
+ // Two placed zooms overlap: the long one still covers the click after the short one ends.
const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [click(1000, 0.2, 0.2), click(2400, 0.8, 0.8)],
- totalMs: 5000,
- existingRegions: [{ startMs: 700, endMs: 1300 }],
- defaultDurationMs: 1000,
+ cursorTelemetry: [click(12000, 0.5, 0.5)],
+ totalMs: 30000,
+ existingRegions: [
+ { startMs: 0, endMs: 20000 },
+ { startMs: 5000, endMs: 8000 },
+ ],
});
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.8, cy: 0.8 });
- expect(suggestions[0].span).toEqual({ start: 1900, end: 2900 });
+ expect(suggestions).toEqual([]);
});
- it("zooms a take whose telemetry is a single recorded click", () => {
+ it("measures an existing zoom's ramps at its own depth", () => {
+ // A 5× zoom takes 1.5 s to come back out, so the wide shot after it starts later.
const suggestions = buildAutoZoomSuggestions({
- cursorTelemetry: [click(2000, 0.3, 0.7)],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
+ cursorTelemetry: [click(9000, 0.5, 0.5)],
+ totalMs: 12000,
+ existingRegions: [{ startMs: 3000, endMs: 5000, scale: 5 }],
});
- expect(suggestions).toHaveLength(1);
- expect(suggestions[0].focus).toEqual({ cx: 0.3, cy: 0.7 });
- expect(suggestions[0].span).toEqual({ start: 1000, end: 3000 });
+ expect(suggestions).toEqual([]);
});
- it("returns nothing for a single move or mouseup sample", () => {
- for (const kind of ["move", "mouseup"] as const) {
- expect(
- buildAutoZoomSuggestions({
- cursorTelemetry: [{ timeMs: 2000, cx: 0.3, cy: 0.7, interactionType: kind }],
- totalMs: 5000,
- existingRegions: [],
- defaultDurationMs: 2000,
- }),
- kind,
- ).toEqual([]);
- }
+ it("keeps a zoom for a click at either end of the axis", () => {
+ expect(plan([click(100, 0.1, 0.1)]).map((s) => s.span)).toEqual([{ start: 0, end: 1800 }]);
+ // Near the end, the zoom arrives earlier rather than holding too briefly.
+ expect(plan([click(4800, 0.9, 0.9)]).map((s) => s.span)).toEqual([{ start: 3200, end: 5000 }]);
});
});
@@ -491,151 +282,119 @@ describe("buildAutoZoomSuggestionsForClips", () => {
origin: "user",
reason: "",
});
+ const telemetryOf = (samples: CursorTelemetryPoint[], assetId = "a1") =>
+ new Map([[assetId, samples]]);
// The bug this covers: telemetry is in the recording's SOURCE time, zoom regions are
// authored in RAW TIMELINE ms, and the two only coincide for a single clip starting at
// 0. Every other layout put all the zooms on the first clip's stretch of ruler.
- it("gives a dwell to EVERY clip that replays it, not just the first", () => {
+ it("gives a click to EVERY clip that replays it, not just the first", () => {
// One recording, laid down twice: source 0-10 at ruler 0-10, then again at 10-20.
- const clips = [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)];
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(4000, 0.3, 0.7),
- assetId: "a1",
- clips,
+ telemetryByAssetId: telemetryOf([click(4000, 0.3, 0.7), { timeMs: 9500, cx: 0.4, cy: 0.6 }]),
+ clips: [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)],
existingRegions: [],
- defaultDurationMs: 2000,
});
expect(suggestions.map((s) => s.span)).toEqual([
- { start: 3424, end: 5424 }, // clip_1: source 4s sits at ruler 4s, after the wide opening
- { start: 13000, end: 15000 }, // clip_2: the SAME source 4s sits at ruler 14s
+ { start: 3500, end: 5500 }, // clip_1: source 4s sits at ruler 4s
+ { start: 13500, end: 15500 }, // clip_2: the SAME source 4s sits at ruler 14s
]);
for (const suggestion of suggestions) {
- expect(suggestion.focus.cx).toBeCloseTo(0.3, 5);
- expect(suggestion.focus.cy).toBeCloseTo(0.7, 5);
+ expect(suggestion.focus).toEqual({ cx: 0.3, cy: 0.7 });
}
});
- it("shifts a dwell by the clip's own source in-point", () => {
+ it("shifts a click by the clip's own source in-point", () => {
// A clip that starts 30s into the recording: source 34s is ruler 4s.
- const clips = [clip("clip_1", "a1", 30, 40, 0)];
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(34000, 0.5, 0.5),
- assetId: "a1",
- clips,
+ telemetryByAssetId: telemetryOf([
+ click(34000, 0.5, 0.5),
+ { timeMs: 60000, cx: 0.6, cy: 0.4 },
+ ]),
+ clips: [clip("clip_1", "a1", 30, 40, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 3424, end: 5424 }]);
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 3500, end: 5500 }]);
});
- it("ignores a dwell that falls outside every clip's source window", () => {
+ it("ignores a click that falls outside every clip's source window", () => {
// The recording is long; the timeline keeps only its first 10s.
- const clips = [clip("clip_1", "a1", 0, 10, 0)];
- expect(
- buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(45000, 0.5, 0.5),
- assetId: "a1",
- clips,
- existingRegions: [],
- defaultDurationMs: 2000,
- }),
- ).toEqual([]);
+ const suggestions = buildAutoZoomSuggestionsForClips({
+ telemetryByAssetId: telemetryOf([
+ click(45000, 0.5, 0.5),
+ { timeMs: 60000, cx: 0.6, cy: 0.4 },
+ ]),
+ clips: [clip("clip_1", "a1", 0, 10, 0)],
+ existingRegions: [],
+ });
+ expect(suggestions).toEqual([]);
});
it("reserves an existing zoom on the clip it actually sits on, and only there", () => {
- // A zoom already covers the dwell on clip_1's ruler span. clip_1 yields; clip_2
- // replays the same source moment on a free stretch of ruler, so it still gets one.
- // Compared in source ms — the frame the caller used to hand down — that one region
- // suppressed the whole recording's worth of suggestions.
- const clips = [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)];
+ // A zoom already covers the click on clip_1's ruler span. clip_1 yields; clip_2 replays
+ // the same source moment on a free stretch of ruler, so it still gets one.
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(4000, 0.5, 0.5),
- assetId: "a1",
- clips,
+ telemetryByAssetId: telemetryOf([click(4000, 0.5, 0.5), { timeMs: 9500, cx: 0.6, cy: 0.4 }]),
+ clips: [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)],
existingRegions: [{ startMs: 3500, endMs: 4500 }],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 13000, end: 15000 }]);
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 13500, end: 15500 }]);
});
- it("only reads the clips of the asset the telemetry belongs to", () => {
- const clips = [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a2", 0, 10, 10)];
+ it("only reads the clips of the recordings it has telemetry for", () => {
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(4000, 0.5, 0.5),
- assetId: "a2",
- clips,
+ telemetryByAssetId: telemetryOf(
+ [click(4000, 0.5, 0.5), { timeMs: 9500, cx: 0.6, cy: 0.4 }],
+ "a2",
+ ),
+ clips: [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a2", 0, 10, 10)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 13000, end: 15000 }]);
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 13500, end: 15500 }]);
});
it("skips a clip whose duration has not been probed yet", () => {
- const clips = [clip("clip_1", "a1", 0, 0, 0)];
- expect(
- buildAutoZoomSuggestionsForClips({
- cursorTelemetry: dwell(4000, 0.5, 0.5),
- assetId: "a1",
- clips,
- existingRegions: [],
- defaultDurationMs: 2000,
- }),
- ).toEqual([]);
- });
-
- // Clicks ride the same per-clip projection dwells do (issue #699).
- it("gives a recorded click the same per-clip projection a dwell gets", () => {
- // A clip that starts 30s into the recording: a click at source 34s is ruler 4s.
- const clips = [clip("clip_1", "a1", 30, 40, 0)];
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(34000, 0.5, 0.5), { timeMs: 36000, cx: 0.6, cy: 0.4 }],
- assetId: "a1",
- clips,
+ telemetryByAssetId: telemetryOf([click(4000, 0.5, 0.5), { timeMs: 9500, cx: 0.6, cy: 0.4 }]),
+ clips: [clip("clip_1", "a1", 0, 0, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 3424, end: 5424 }]);
- expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 });
+ expect(suggestions).toEqual([]);
});
- it("gives a click to EVERY clip that replays it, one shifted span each", () => {
- const clips = [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)];
+ it("zooms a clip whose source window trims the telemetry down to a single click", () => {
+ // A clip covering 30..40s of the recording; the take's telemetry holds one move long
+ // before the window, one click inside it and one move long after. The per-clip filter
+ // hands the planner a SINGLE sample, still a zoom, correctly projected.
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(4000, 0.3, 0.7), { timeMs: 6000, cx: 0.4, cy: 0.6 }],
- assetId: "a1",
- clips,
+ telemetryByAssetId: telemetryOf([
+ { timeMs: 5000, cx: 0.1, cy: 0.9 },
+ click(34000, 0.5, 0.5),
+ { timeMs: 60000, cx: 0.1, cy: 0.9 },
+ ]),
+ clips: [clip("clip_1", "a1", 30, 40, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([
- { start: 3424, end: 5424 },
- { start: 13000, end: 15000 },
+ expect(suggestions.map((s) => [s.span, s.focus])).toEqual([
+ [
+ { start: 3500, end: 5500 },
+ { cx: 0.5, cy: 0.5 },
+ ],
]);
- for (const suggestion of suggestions) {
- expect(suggestion.focus.cx).toBeCloseTo(0.3, 5);
- expect(suggestion.focus.cy).toBeCloseTo(0.7, 5);
- }
});
- it("zooms a clip whose source window trims the telemetry down to a single click", () => {
- // A clip covering 30..40s of the recording; the take's telemetry holds one move
- // long before the window, one click inside it and one move long after. The per-clip
- // filter hands the detector a SINGLE sample — still a zoom, correctly projected
- // (source 34s is ruler 4s).
- const clips = [clip("clip_1", "a1", 30, 40, 0)];
+ it("does not hold a zoom across a cut: of two clips' clashing zooms, one stays", () => {
+ // Two recordings back to back. Each clip's zoom is fine alone, but together they would
+ // zoom out at the cut and straight back in, and a zoom does not merge across a cut.
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [
- { timeMs: 5000, cx: 0.1, cy: 0.9 },
- click(34000, 0.5, 0.5),
- { timeMs: 60000, cx: 0.1, cy: 0.9 },
- ],
- assetId: "a1",
- clips,
+ telemetryByAssetId: new Map([
+ ["a1", [click(7000, 0.5, 0.5), { timeMs: 9900, cx: 0.5, cy: 0.5 }]],
+ ["a2", [click(1000, 0.5, 0.5), { timeMs: 9900, cx: 0.5, cy: 0.5 }]],
+ ]),
+ clips: [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a2", 0, 10, 10)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 3424, end: 5424 }]);
- expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 });
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 6500, end: 8500 }]);
});
// The video opens and closes on the whole screen. At depth 3 a ramp lasts 923 ms, so a
@@ -646,42 +405,34 @@ describe("buildAutoZoomSuggestionsForClips", () => {
it("keeps the first seconds of the edit wide: an early click gets no zoom", () => {
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(1000, 0.3, 0.3), restOfTake],
- assetId: "a1",
+ telemetryByAssetId: telemetryOf([click(1000, 0.3, 0.3), restOfTake]),
clips: [clip("clip_1", "a1", 0, 10, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
expect(suggestions).toEqual([]);
});
it("holds a zoom back until the opening is over when it still covers its click", () => {
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(4000, 0.3, 0.3), restOfTake],
- assetId: "a1",
+ telemetryByAssetId: telemetryOf([click(3600, 0.3, 0.3), restOfTake]),
clips: [clip("clip_1", "a1", 0, 10, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 3424, end: 5424 }]);
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 3424, end: 5224 }]);
});
it("brings the last zoom back out a second before the end of the edit", () => {
// The recording goes on to 30 s, so this is the edit's end at work, not the stop click.
const zoomed = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(7500, 0.3, 0.3), restOfTake],
- assetId: "a1",
+ telemetryByAssetId: telemetryOf([click(7500, 0.3, 0.3), restOfTake]),
clips: [clip("clip_1", "a1", 0, 10, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(zoomed.map((s) => s.span)).toEqual([{ start: 6076, end: 8076 }]);
+ expect(zoomed.map((s) => s.span)).toEqual([{ start: 6276, end: 8076 }]);
const tooLate = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(8500, 0.3, 0.3), restOfTake],
- assetId: "a1",
+ telemetryByAssetId: telemetryOf([click(8500, 0.3, 0.3), restOfTake]),
clips: [clip("clip_1", "a1", 0, 10, 0)],
existingRegions: [],
- defaultDurationMs: 2000,
});
expect(tooLate).toEqual([]);
});
@@ -690,51 +441,48 @@ describe("buildAutoZoomSuggestionsForClips", () => {
// One recording laid down twice: the click at source 1 s is inside the opening
// on the first clip, and at ruler 11 s, clear of it, on the second.
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [click(1000, 0.3, 0.3), { timeMs: 9500, cx: 0.5, cy: 0.5 }],
- assetId: "a1",
+ telemetryByAssetId: telemetryOf([
+ click(1000, 0.3, 0.3),
+ { timeMs: 9500, cx: 0.5, cy: 0.5 },
+ ]),
clips: [clip("clip_1", "a1", 0, 10, 0), clip("clip_2", "a1", 0, 10, 10)],
existingRegions: [],
- defaultDurationMs: 2000,
});
- expect(suggestions.map((s) => s.span)).toEqual([{ start: 10000, end: 12000 }]);
+ expect(suggestions.map((s) => s.span)).toEqual([{ start: 10500, end: 12500 }]);
});
it("ignores the click that stopped the recording", () => {
// The take's last click, 100 ms before its telemetry ends: the HUD's Stop button.
// Another clip follows on the ruler, so the edit's own ending does not reach it.
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [
+ telemetryByAssetId: telemetryOf([
click(6000, 0.3, 0.3),
click(19800, 0.5, 0.92),
{ timeMs: 19900, cx: 0.5, cy: 0.92 },
- ],
- assetId: "a1",
+ ]),
clips: [clip("clip_1", "a1", 0, 20, 0), clip("clip_2", "a2", 0, 20, 20)],
existingRegions: [],
- defaultDurationMs: 2000,
});
expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.3, cy: 0.3 }]);
});
it("ignores the pointer resting on Stop before that click", () => {
// Hover-then-Stop: the pointer sits on the button from 18.8 s, clicks at 19.8 s,
- // and the telemetry ends at 19.9 s. That still run is a dwell, and it goes with its click.
+ // and the telemetry ends at 19.9 s. Neither the rest nor the click zooms.
const hover = Array.from({ length: 34 }, (_, i) => ({
timeMs: 18800 + i * 33,
cx: 0.5,
cy: 0.92,
}));
const suggestions = buildAutoZoomSuggestionsForClips({
- cursorTelemetry: [
+ telemetryByAssetId: telemetryOf([
click(6000, 0.3, 0.3),
...hover,
click(19800, 0.5, 0.92),
{ timeMs: 19900, cx: 0.5, cy: 0.92 },
- ],
- assetId: "a1",
+ ]),
clips: [clip("clip_1", "a1", 0, 20, 0), clip("clip_2", "a2", 0, 20, 20)],
existingRegions: [],
- defaultDurationMs: 2000,
});
expect(suggestions.map((s) => s.focus)).toEqual([{ cx: 0.3, cy: 0.3 }]);
});
diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts
index 97e3a2ba8..8df686f48 100644
--- a/src/lib/ai-edition/timeline/zoom-suggestions.ts
+++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts
@@ -1,22 +1,21 @@
-// Cursor-telemetry-driven auto-zoom suggestions. Pure, no DOM/IPC.
+// Cursor-telemetry-driven auto-zoom. Pure, no DOM/IPC.
//
-// Ported from main's `src/components/video-editor/timeline/zoomSuggestionUtils.ts`
-// (the legacy editor's "magic wand" auto-zoom) into the ai-edition timeline
-// module. This is NOT an AI feature — it's a deterministic detector over
-// recorded cursor movement: stretches where the cursor sits still become
-// zoom-in candidates, focused on the average cursor position during the dwell,
-// and so do recorded clicks, focused on the click itself (issue #699).
+// Ported from main's `src/components/video-editor/timeline/zoomSuggestionUtils.ts` (the legacy
+// editor's "magic wand"), since rewritten as a planner. This is NOT an AI feature: it is a
+// deterministic pass over the clicks recorded with a take, which thinks in shots the way an
+// editor would. The video opens on the whole screen. Clicks close in time and in space share one
+// zoom, which holds while the camera follows the cursor. A zoom never comes straight back after
+// leaving. The video ends on the whole screen.
+//
+// A still pointer is no reason to zoom: at rest it sits far from where the viewer looks, and
+// Screen Studio, Cap and Tella all zoom on clicks alone. A take recorded without clicks gets no
+// automatic zoom.
import type { CursorTelemetryPoint, ZoomFocus } from "@/components/video-editor/types";
import { zoomTransitionMs } from "@/lib/zoomMath/constants";
import type { AxcutClip } from "../schema";
-import { DEFAULT_ZOOM_DEPTH, ZOOM_DEPTH_SCALES } from "./zoom-scale";
+import { DEFAULT_ZOOM_DEPTH, ZOOM_DEPTH_SCALES, type ZoomDepth } from "./zoom-scale";
-export const MIN_DWELL_DURATION_MS = 450;
-export const MAX_DWELL_DURATION_MS = 2600;
-export const DWELL_MOVE_THRESHOLD = 0.02;
-/** Minimum spacing between two accepted suggestion centres. */
-export const SUGGESTION_SPACING_MS = 1800;
/** The video opens on the whole screen: no zoom starts moving in before this. */
export const OPENING_WIDE_MS = 2500;
/** And it ends on it: the last zoom is fully back out this long before the end. */
@@ -27,11 +26,33 @@ export const CLOSING_WIDE_MS = 1000;
* never shows either.
*/
export const STOP_CLICK_GUARD_MS = 1000;
+/** Clicks closer than this in time share one zoom, when they fit in it together. */
+export const SHOT_MERGE_GAP_MS = 3000;
+/** A zoom is fully in this long before its first click... */
+export const SHOT_LEAD_MS = 500;
+/** ...and holds this long after its last one, to show what the click did. */
+export const SHOT_TAIL_MS = 1500;
+/** The shortest time a zoom holds, fully in. */
+export const MIN_SHOT_HOLD_MS = 1800;
+/** The shortest wide shot between two zooms, ramps excluded. Less reads as a yo-yo. */
+export const MIN_WIDE_BETWEEN_MS = 1500;
+/** The margin kept around the clicks a zoom frames, as a fraction of the frame. */
+export const SHOT_FIT_MARGIN = 0.12;
+/** The depths a zoom may take, shallowest first: 1.25×, 1.5×, 1.8×. */
+const SHOT_DEPTHS: readonly ZoomDepth[] = [1, 2, 3];
-export interface ZoomDwellCandidate {
- centerTimeMs: number;
+export interface AutoZoomSuggestion {
+ span: { start: number; end: number };
focus: ZoomFocus;
- strength: number;
+ depth: ZoomDepth;
+}
+
+/** A zoom already on the timeline: the plan keeps clear of it and never moves it. */
+export interface ReservedZoom {
+ startMs: number;
+ endMs: number;
+ /** Its effective scale, which sets how long its ramps last. The default depth's if omitted. */
+ scale?: number;
}
function normalizeTelemetrySample(
@@ -60,70 +81,9 @@ export function normalizeCursorTelemetry(
.map((sample) => normalizeTelemetrySample(sample, totalMs));
}
-/**
- * `maxDwellDurationMs` exists for ONE caller and defaults to the magic wand's
- * own ceiling so that caller is untouched.
- *
- * ponytail: the ceiling REJECTS a run outright rather than splitting it, so a
- * cursor parked for eight seconds while the user reads the screen produces no
- * candidate at all. That is a defensible auto-zoom policy — a nine-second zoom
- * is not a zoom — and an indefensible reporting policy: the moments it drops are
- * precisely the ones a human would name first if asked "where did the pointer
- * sit?". The digest passes `Infinity`; nothing else does.
- */
-export function detectZoomDwellCandidates(
- samples: CursorTelemetryPoint[],
- maxDwellDurationMs: number = MAX_DWELL_DURATION_MS,
-): ZoomDwellCandidate[] {
- if (samples.length < 2) {
- return [];
- }
-
- const dwellCandidates: ZoomDwellCandidate[] = [];
- let runStart = 0;
-
- const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => {
- if (endIndexExclusive - startIndex < 2) {
- return;
- }
-
- const start = samples[startIndex];
- const end = samples[endIndexExclusive - 1];
- const runDuration = end.timeMs - start.timeMs;
- if (runDuration < MIN_DWELL_DURATION_MS || runDuration > maxDwellDurationMs) {
- return;
- }
-
- const runSamples = samples.slice(startIndex, endIndexExclusive);
- const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length;
- const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length;
-
- dwellCandidates.push({
- centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2),
- focus: { cx: avgCx, cy: avgCy },
- strength: runDuration,
- });
- };
-
- for (let index = 0; index < samples.length; index += 1) {
- if (samples[index].visible === false) {
- pushRunIfDwell(runStart, index);
- runStart = index + 1;
- continue;
- }
- if (index <= runStart) continue;
- const prev = samples[index - 1];
- const curr = samples[index];
- const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy);
-
- if (distance > DWELL_MOVE_THRESHOLD) {
- pushRunIfDwell(runStart, index);
- runStart = index;
- }
- }
- pushRunIfDwell(runStart, samples.length);
-
- return dwellCandidates;
+interface RecordedClick {
+ timeMs: number;
+ focus: ZoomFocus;
}
/** The recorded interactions that count as a click; `move` and `mouseup` do not. */
@@ -131,14 +91,11 @@ const CLICK_INTERACTION_TYPES: ReadonlySet<NonNullable<CursorTelemetryPoint["int
new Set(["click", "double-click", "right-click", "middle-click"]);
/**
- * Clicks are zoom candidates in their own right (issue #699): a click made while the
- * pointer is still moving forms no dwell, yet it is exactly the moment a viewer wants
- * magnified. The candidate is the click sample itself — its time, its position — and
- * `strength` is 0 because a click has no duration to rank by; that only ever breaks
- * ties between two clicks, where the stable sort keeps the earlier one.
+ * The clicks a viewer sees (issue #699), at the click's own time and position. A press the
+ * cursor hides during is a drag the app took over, a slider or a scrubbed field, not a click.
*/
-function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCandidate[] {
- const candidates: ZoomDwellCandidate[] = [];
+function detectClicks(samples: CursorTelemetryPoint[]): RecordedClick[] {
+ const clicks: RecordedClick[] = [];
// A press without its own mouse-up sample has no known release: PipeWire and older sidecars
// record none, and the Windows and macOS samplers fold a release into the press sample when both
// land in one tick. A hidden sample on the next sampler tick still identifies a slider press; a
@@ -153,10 +110,9 @@ function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCa
hiddenAtMs !== null &&
(released || hiddenAtMs - pendingClick.timeMs <= UNRELEASED_PRESS_GRACE_MS);
if (!hiddenDuringPress) {
- candidates.push({
- centerTimeMs: pendingClick.timeMs,
+ clicks.push({
+ timeMs: pendingClick.timeMs,
focus: { cx: pendingClick.cx, cy: pendingClick.cy },
- strength: 0,
});
}
pendingClick = null;
@@ -175,216 +131,292 @@ function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCa
}
}
finishPress(false);
- return candidates;
+ return clicks;
}
-export interface AutoZoomSuggestion {
- span: { start: number; end: number };
- focus: ZoomFocus;
+function visibleClicks(
+ samples: CursorTelemetryPoint[],
+ totalMs: number,
+ ignoreFromMs: number,
+): RecordedClick[] {
+ return detectClicks(normalizeCursorTelemetry(samples, totalMs)).filter(
+ (click) => click.timeMs < ignoreFromMs,
+ );
}
-/**
- * Build non-overlapping zoom suggestions from cursor telemetry: detect dwell moments
- * and recorded clicks, space by SUGGESTION_SPACING_MS, drop any overlapping an
- * existing region. Clicks are honoured FIRST, so only an ACCEPTED click folds the
- * dwells within suggestion spacing of it — the acceptance loop rejects those dwells
- * against the accepted click's centre — while a click that spacing or an existing
- * region rejects leaves its nearby dwells standing as the fallback. One recorded
- * click still yields ONE zoom anchored on the click's own time and position
- * (issue #699). Pure, shared by the magic-wand toggle and the on-load auto-suggest pass.
- *
- * `wideUntilMs` and `wideFromMs` keep the ends of the video on the whole screen: a zoom
- * starts moving in no earlier than the first and is back out by the second, ramps
- * included. A zoom those bounds push off its own moment is dropped rather than shown
- * after the fact. `ignoreClicksFromMs` drops the click that stopped the recording.
- * All three are on the telemetry's axis, and default to no bound.
- */
-export function buildAutoZoomSuggestions(options: {
- cursorTelemetry: CursorTelemetryPoint[];
- totalMs: number;
- existingRegions: { startMs: number; endMs: number }[];
- defaultDurationMs: number;
- wideUntilMs?: number;
- wideFromMs?: number;
- ignoreClicksFromMs?: number;
-}): AutoZoomSuggestion[] {
- const { cursorTelemetry, totalMs, existingRegions, defaultDurationMs } = options;
- if (totalMs <= 0 || cursorTelemetry.length === 0) {
- return [];
- }
-
- const defaultDuration = Math.min(defaultDurationMs, totalMs);
- if (defaultDuration <= 0) {
- return [];
- }
-
- // The ramps play outside the region, so the region itself keeps one ramp clear of each bound.
- const ramp = zoomTransitionMs(ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH]);
- const earliestStart = Math.max(0, Math.ceil((options.wideUntilMs ?? -Infinity) + ramp));
- const latestEnd = Math.min(totalMs, Math.floor((options.wideFromMs ?? Infinity) - ramp));
- if (latestEnd - earliestStart < defaultDuration) {
- return [];
+/** When the recording stopped: its telemetry runs until the take ends. */
+function lastSampleMs(samples: CursorTelemetryPoint[]): number {
+ let last = 0;
+ for (const sample of samples) {
+ if (Number.isFinite(sample.timeMs)) last = Math.max(last, sample.timeMs);
}
+ return last;
+}
- const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs);
- if (normalizedSamples.length === 0) {
- return [];
- }
+interface Box {
+ minX: number;
+ maxX: number;
+ minY: number;
+ maxY: number;
+}
- const ignoreClicksFromMs = options.ignoreClicksFromMs ?? Infinity;
- const allClicks = detectZoomClickCandidates(normalizedSamples);
- const clickCandidates = allClicks.filter(
- (candidate) => candidate.centerTimeMs < ignoreClicksFromMs,
- );
- const ignoredClicks = allClicks.filter(
- (candidate) => candidate.centerTimeMs >= ignoreClicksFromMs,
- );
- // The pointer usually rests on Stop before the click: that dwell goes with the click.
- const dwellCandidates = detectZoomDwellCandidates(normalizedSamples).filter(
- (dwell) =>
- !ignoredClicks.some(
- (click) => Math.abs(click.centerTimeMs - dwell.centerTimeMs) <= dwell.strength / 2,
- ),
- );
- if (dwellCandidates.length === 0 && clickCandidates.length === 0) {
- return [];
- }
+interface Shot {
+ start: number;
+ end: number;
+ depth: ZoomDepth;
+ /** Where its clicks fall, and how many: the weight when two shots compete. */
+ box: Box;
+ clicks: number;
+ /** Two shots merge only on one clip: a zoom does not hold across a cut. */
+ clip?: string;
+}
- // Clicks are honoured FIRST so that only an accepted click folds its nearby dwells:
- // the acceptance loop below rejects any dwell within SUGGESTION_SPACING_MS of an
- // accepted click's centre, while a click that spacing or an existing region rejects
- // leaves the dwells around it standing. Telemetry without click metadata runs the
- // unchanged dwell-only ranking. Within each kind the order is the old one — clicks
- // chronological, dwells by duration.
- const dwellByDuration = [...dwellCandidates].sort((a, b) => b.strength - a.strength);
- const sortedCandidates = [...clickCandidates, ...dwellByDuration];
+function boxAround(focus: ZoomFocus): Box {
+ return { minX: focus.cx, maxX: focus.cx, minY: focus.cy, maxY: focus.cy };
+}
- const reservedSpans = existingRegions
- .map((region) => ({ start: region.startMs, end: region.endMs }))
- .sort((a, b) => a.start - b.start);
+function union(a: Box, b: Box): Box {
+ return {
+ minX: Math.min(a.minX, b.minX),
+ maxX: Math.max(a.maxX, b.maxX),
+ minY: Math.min(a.minY, b.minY),
+ maxY: Math.max(a.maxY, b.maxY),
+ };
+}
- const acceptedCenters: number[] = [];
- const suggestions: AutoZoomSuggestion[] = [];
+/** The deepest of `SHOT_DEPTHS` that shows the whole box with its margin, or null. */
+function fitDepth(box: Box): ZoomDepth | null {
+ const size = Math.max(box.maxX - box.minX, box.maxY - box.minY) + 2 * SHOT_FIT_MARGIN;
+ let fit: ZoomDepth | null = null;
+ for (const depth of SHOT_DEPTHS) {
+ if (ZOOM_DEPTH_SCALES[depth] * size <= 1) fit = depth;
+ }
+ return fit;
+}
- for (const candidate of sortedCandidates) {
- const tooCloseToAccepted = acceptedCenters.some(
- (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS,
- );
- if (tooCloseToAccepted) {
- continue;
+/**
+ * One axis's clicks as timed shots. `wideUntilMs` and `wideFromMs` keep the ends of the video
+ * on the whole screen: a zoom starts moving in no earlier than the first and is back out by the
+ * second, ramps included. A shot the bounds leave too short a hold, or none of its clicks, is
+ * dropped: a zoom arriving after the fact shows nothing.
+ */
+function planShots(
+ clicks: RecordedClick[],
+ totalMs: number,
+ wideUntilMs: number,
+ wideFromMs: number,
+): Shot[] {
+ const groups: { times: number[]; box: Box }[] = [];
+ for (const click of clicks) {
+ const group = groups[groups.length - 1];
+ if (group && click.timeMs - group.times[group.times.length - 1] <= SHOT_MERGE_GAP_MS) {
+ const box = union(group.box, boxAround(click.focus));
+ if (fitDepth(box) !== null) {
+ group.times.push(click.timeMs);
+ group.box = box;
+ continue;
+ }
}
+ groups.push({ times: [click.timeMs], box: boxAround(click.focus) });
+ }
- const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2);
- const candidateStart = Math.max(
- earliestStart,
- Math.min(centeredStart, latestEnd - defaultDuration),
+ const shots: Shot[] = [];
+ for (const { times, box } of groups) {
+ // A group only grows while it fits, so it always has a depth.
+ const depth = fitDepth(box) ?? SHOT_DEPTHS[0];
+ const ramp = zoomTransitionMs(ZOOM_DEPTH_SCALES[depth]);
+ const earliest = Math.max(0, Math.ceil(wideUntilMs + ramp));
+ const latest = Math.min(totalMs, Math.floor(wideFromMs - ramp));
+ const first = times[0];
+ const last = times[times.length - 1];
+ // Near the end of the axis, the zoom arrives earlier rather than holding too briefly.
+ const start = Math.max(
+ Math.min(Math.round(first - SHOT_LEAD_MS), latest - MIN_SHOT_HOLD_MS),
+ earliest,
);
- const candidateEnd = candidateStart + defaultDuration;
- if (candidate.centerTimeMs < candidateStart || candidate.centerTimeMs > candidateEnd) {
- continue;
- }
- const hasOverlap = reservedSpans.some(
- (span) => candidateEnd > span.start && candidateStart < span.end,
+ const end = Math.min(
+ Math.max(Math.round(last + SHOT_TAIL_MS), start + MIN_SHOT_HOLD_MS),
+ latest,
);
- if (hasOverlap) {
+ if (end - start < MIN_SHOT_HOLD_MS) continue;
+ if (!times.some((time) => time >= start && time <= end)) continue;
+ shots.push({ start, end, depth, box, clicks: times.length });
+ }
+ return shots;
+}
+
+/**
+ * No yo-yo: between two zooms, a real wide shot or none at all. Two shots too close merge when
+ * they share a clip and fit in one zoom together; otherwise the one with fewer clicks goes, the
+ * later one on a tie. A zoom already on the timeline never moves: a shot too close to it goes.
+ *
+ * ponytail: the gap is measured in timeline ms, while the renderer plays ramps on the screen
+ * clock, so inside a speed region the real wide shot differs. Map through `screenTimeMs` if a
+ * take with speed regions ever shows a yo-yo.
+ */
+function resolveShots(shots: Shot[], reserved: ReservedZoom[]): Shot[] {
+ const plan = [...shots];
+ const fixed = reserved.map((zoom) => ({
+ start: zoom.startMs,
+ end: zoom.endMs,
+ scale: zoom.scale ?? ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH],
+ shot: null,
+ }));
+ for (;;) {
+ const spans = [
+ ...plan.map((shot) => ({
+ start: shot.start,
+ end: shot.end,
+ scale: ZOOM_DEPTH_SCALES[shot.depth],
+ shot,
+ })),
+ ...fixed,
+ ].sort((a, b) => a.start - b.start);
+ // Each span against the one before it that ends latest, ramp included: a long zoom
+ // reaches past its neighbours, and two placed zooms may even overlap each other.
+ const outOf = (span: (typeof spans)[number]) => span.end + zoomTransitionMs(span.scale);
+ let reach: (typeof spans)[number] | undefined;
+ let clash: [(typeof spans)[number], (typeof spans)[number]] | undefined;
+ for (const span of spans) {
+ if (reach && (reach.shot || span.shot)) {
+ const wide = span.start - outOf(reach) - zoomTransitionMs(span.scale);
+ if (wide < MIN_WIDE_BETWEEN_MS) {
+ clash = [reach, span];
+ break;
+ }
+ }
+ if (!reach || outOf(span) > outOf(reach)) reach = span;
+ }
+ if (!clash) return plan;
+
+ const a = clash[0].shot;
+ const b = clash[1].shot;
+ if (a && b) {
+ const box = union(a.box, b.box);
+ const depth = a.clip === b.clip ? fitDepth(box) : null;
+ if (depth !== null) {
+ const merged: Shot = {
+ start: a.start,
+ end: Math.max(a.end, b.end),
+ depth,
+ box,
+ clicks: a.clicks + b.clicks,
+ clip: a.clip,
+ };
+ plan.splice(plan.indexOf(a), 1, merged);
+ plan.splice(plan.indexOf(b), 1);
+ } else {
+ plan.splice(plan.indexOf(a.clicks >= b.clicks ? b : a), 1);
+ }
continue;
}
-
- reservedSpans.push({ start: candidateStart, end: candidateEnd });
- acceptedCenters.push(candidate.centerTimeMs);
- suggestions.push({
- span: { start: candidateStart, end: candidateEnd },
- focus: candidate.focus,
- });
+ const shot = a ?? b;
+ if (!shot) return plan; // unreachable: a clash always involves a shot
+ plan.splice(plan.indexOf(shot), 1);
}
+}
- return suggestions;
+function toSuggestion(shot: Shot): AutoZoomSuggestion {
+ return {
+ span: { start: shot.start, end: shot.end },
+ focus: { cx: (shot.box.minX + shot.box.maxX) / 2, cy: (shot.box.minY + shot.box.maxY) / 2 },
+ depth: shot.depth,
+ };
}
+const byStart = (a: Shot, b: Shot) => a.start - b.start;
+
/**
- * The same detector, run over a TIMELINE instead of over a bare media file — and the
- * only entry point a caller holding an `AxcutDocument` should use.
- *
- * Cursor telemetry is recorded against the ORIGINAL media file, so `timeMs` is the
- * asset's SOURCE time (the same axis `cursor-track.ts` maps through
- * `locateSourcePosition`, and the same one trims are stored in). Zoom regions are
- * authored in RAW TIMELINE ms — that is what `anchorRegionsWithDerivedMs` ventilates
- * across the clips. The two axes coincide for exactly one layout: a single clip,
- * starting at 0, covering the whole recording. Any other timeline made the detector's
- * output land wherever `[0, assetDuration]` happens to fall on the ruler, which is the
- * first clip's span — hence "auto-zoom only decorates the first clip". Two clips over
- * ONE recording make it plainer still: the second clip replays source time the first
- * already used, so no amount of arithmetic on a single asset-wide span can say which of
- * them a dwell belongs to. It belongs to BOTH, and gets one zoom on each.
+ * Plan the zooms of one recording laid on the timeline from 0, straight from its telemetry.
+ * `wideUntilMs` and `wideFromMs` are the video's opening and ending, `ignoreClicksFromMs` drops
+ * the stop click; all three are on the telemetry's axis and default to no bound. A caller
+ * holding an `AxcutDocument` wants `buildAutoZoomSuggestionsForClips`.
+ */
+export function buildAutoZoomSuggestions(options: {
+ cursorTelemetry: CursorTelemetryPoint[];
+ totalMs: number;
+ existingRegions: ReservedZoom[];
+ wideUntilMs?: number;
+ wideFromMs?: number;
+ ignoreClicksFromMs?: number;
+}): AutoZoomSuggestion[] {
+ const { cursorTelemetry, totalMs, existingRegions } = options;
+ if (totalMs <= 0 || cursorTelemetry.length === 0) return [];
+ const clicks = visibleClicks(cursorTelemetry, totalMs, options.ignoreClicksFromMs ?? Infinity);
+ const shots = planShots(
+ clicks,
+ totalMs,
+ options.wideUntilMs ?? -Infinity,
+ options.wideFromMs ?? Infinity,
+ );
+ return resolveShots(shots, existingRegions).sort(byStart).map(toSuggestion);
+}
+
+/**
+ * The same planner, run over a TIMELINE instead of over a bare media file — and the only entry
+ * point a caller holding an `AxcutDocument` should use.
*
- * So the projection is per clip, and it is a plain shift: a raw clip is identity between
- * its source time and its raw-virtual time (see timeline/timelineMap.ts), so a dwell or
- * click at source `t` on a clip covering `[sourceStartSec, sourceEndSec]` sits at
- * `timelineStartSec + (t - sourceStartSec)`. Each clip is handed only the samples inside
- * its own source window, so a dwell that a cut split across two clips is no longer one
- * dwell — which is right: the cursor did not sit still across the cut on the timeline the
- * user is watching. `existingRegions` is in RAW TIMELINE ms (what the store holds), so it
- * reserves the right stretch of ruler on every clip instead of only on the first.
+ * Cursor telemetry is recorded against the ORIGINAL media file, so `timeMs` is the asset's
+ * SOURCE time (the same axis `cursor-track.ts` maps through `locateSourcePosition`, and the same
+ * one trims are stored in). Zoom regions are authored in RAW TIMELINE ms — that is what
+ * `anchorRegionsWithDerivedMs` ventilates across the clips. The two axes coincide for exactly
+ * one layout: a single clip, starting at 0, covering the whole recording. Two clips over ONE
+ * recording make it plain: the second replays source time the first already used, so a click
+ * belongs to BOTH, and gets a zoom on each.
*
- * Clips of other assets are skipped, as are clips with no probed source window.
- * `buildAutoZoomSuggestions` is reused verbatim per clip — spacing, ranking and the
- * reserve rule keep their single definition.
+ * So each clip reads only the samples inside its own source window, shifted onto the ruler: a
+ * raw clip is identity between its source time and its raw-virtual time (see
+ * timeline/timelineMap.ts). Shots are planned per clip, since a zoom does not hold across a cut,
+ * then settled over the whole timeline at once, every recording included, against the zooms
+ * already on it (`existingRegions`, RAW TIMELINE ms). Clips with no telemetry or no probed
+ * source window get nothing.
*
* The wide opening and ending belong to the whole edit, not to each clip: a cut may land
- * straight on a zoom, but the video's first and last seconds show the whole screen. The
- * stop click belongs to the recording: its last second, read off the telemetry, which runs
- * until the take stops.
+ * straight on a zoom, but the video's first and last seconds show the whole screen. The stop
+ * click belongs to its recording: that recording's last second.
*/
export function buildAutoZoomSuggestionsForClips(options: {
- /** Samples in the asset's own SOURCE time. */
- cursorTelemetry: CursorTelemetryPoint[];
- assetId: string;
+ /** Each recording's samples, in its own SOURCE time, by asset id. */
+ telemetryByAssetId: ReadonlyMap<string, CursorTelemetryPoint[]>;
clips: AxcutClip[];
- /** Already-placed zoom spans, in RAW TIMELINE ms. */
- existingRegions: { startMs: number; endMs: number }[];
- defaultDurationMs: number;
+ existingRegions: ReservedZoom[];
}): AutoZoomSuggestion[] {
- const { cursorTelemetry, assetId, clips, existingRegions, defaultDurationMs } = options;
+ const { telemetryByAssetId, clips, existingRegions } = options;
+ if (clips.length === 0) return [];
const editStartMs = Math.min(...clips.map((clip) => clip.timelineStartSec)) * 1000;
const editEndMs = Math.max(...clips.map((clip) => clip.timelineEndSec)) * 1000;
- let recordingEndMs = 0;
- for (const sample of cursorTelemetry) {
- if (Number.isFinite(sample.timeMs)) recordingEndMs = Math.max(recordingEndMs, sample.timeMs);
- }
- const suggestions: AutoZoomSuggestion[] = [];
+ const shots: Shot[] = [];
for (const clip of clips) {
- if (clip.assetId !== assetId) continue;
+ const telemetry = telemetryByAssetId.get(clip.assetId);
const sourceEndSec = clip.sourceEndSec ?? clip.sourceStartSec;
const windowMs = (sourceEndSec - clip.sourceStartSec) * 1000;
- if (windowMs <= 0) continue;
+ if (!telemetry?.length || windowMs <= 0) continue;
const sourceOffsetMs = clip.sourceStartSec * 1000;
const timelineOffsetMs = clip.timelineStartSec * 1000;
- const clipTelemetry = cursorTelemetry
+ const clipTelemetry = telemetry
.filter(
(sample) => sample.timeMs >= sourceOffsetMs && sample.timeMs <= sourceOffsetMs + windowMs,
)
.map((sample) => ({ ...sample, timeMs: sample.timeMs - sourceOffsetMs }));
- const clipSuggestions = buildAutoZoomSuggestions({
- cursorTelemetry: clipTelemetry,
- totalMs: windowMs,
- existingRegions: existingRegions.map((region) => ({
- startMs: region.startMs - timelineOffsetMs,
- endMs: region.endMs - timelineOffsetMs,
- })),
- defaultDurationMs,
- wideUntilMs: editStartMs + OPENING_WIDE_MS - timelineOffsetMs,
- wideFromMs: editEndMs - CLOSING_WIDE_MS - timelineOffsetMs,
- ignoreClicksFromMs: recordingEndMs - STOP_CLICK_GUARD_MS - sourceOffsetMs,
- });
- suggestions.push(
- ...clipSuggestions.map((suggestion) => ({
- focus: suggestion.focus,
- span: {
- start: suggestion.span.start + timelineOffsetMs,
- end: suggestion.span.end + timelineOffsetMs,
- },
- })),
+ const clicks = visibleClicks(
+ clipTelemetry,
+ windowMs,
+ lastSampleMs(telemetry) - STOP_CLICK_GUARD_MS - sourceOffsetMs,
);
+ const clipShots = planShots(
+ clicks,
+ windowMs,
+ editStartMs + OPENING_WIDE_MS - timelineOffsetMs,
+ editEndMs - CLOSING_WIDE_MS - timelineOffsetMs,
+ );
+ for (const shot of clipShots) {
+ shots.push({
+ ...shot,
+ start: shot.start + timelineOffsetMs,
+ end: shot.end + timelineOffsetMs,
+ clip: clip.id,
+ });
+ }
}
- return suggestions;
+ return resolveShots(shots, existingRegions).sort(byStart).map(toSuggestion);
}
diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts
index 186613a85..03657887a 100644
--- a/src/lib/cliContracts.ts
+++ b/src/lib/cliContracts.ts
@@ -24,8 +24,8 @@ export interface CliExportRequest {
* into this box, mirroring the editor layout. Defaults to 1280x720.
*/
/**
- * Add automatic zoom regions derived from cursor-dwell telemetry (same
- * suggestion engine as the editor's magic wand) before rendering. Existing
+ * Add automatic zoom regions planned from the clicks in the cursor telemetry
+ * (same planner as the editor's magic wand) before rendering. Existing
* zoom regions are preserved; suggestions never overlap them.
*/
autoZoom: boolean;
diff --git a/technical-documentation/architecture/timeline-model.md b/technical-documentation/architecture/timeline-model.md
index e0529111e..84b31ddfa 100644
--- a/technical-documentation/architecture/timeline-model.md
+++ b/technical-documentation/architecture/timeline-model.md
@@ -153,20 +153,21 @@ those three now share is that **a named clip is believed over anything inferred
### Cursor telemetry is source time, and belongs to every clip that replays it
-Auto-zoom reads recorded cursor movement, which is captured against the ORIGINAL media
-file: `timeMs` is the asset's SOURCE time, the same axis `cursor-track.ts` maps through
-`locateSourcePosition`. Zoom regions are authored in RAW TIMELINE ms — that is what
-`anchorRegionsWithDerivedMs` ventilates across clips. The two axes coincide for exactly
+Auto-zoom reads the clicks in the recorded cursor telemetry, which is captured against the
+ORIGINAL media file: `timeMs` is the asset's SOURCE time, the same axis `cursor-track.ts`
+maps through `locateSourcePosition`. Zoom regions are authored in RAW TIMELINE ms — that is
+what `anchorRegionsWithDerivedMs` ventilates across clips. The two axes coincide for exactly
one layout: a single clip, starting at 0, covering the whole recording. Feeding the
-detector's source-time output straight to the store therefore dropped every suggestion
+planner's source-time output straight to the store therefore dropped every suggestion
wherever `[0, assetDuration]` happens to fall on the ruler, i.e. on the first clip —
"automatic zooms only decorate the first clip". `buildAutoZoomSuggestionsForClips`
([`zoom-suggestions.ts`](../../src/lib/ai-edition/timeline/zoom-suggestions.ts)) does the
-projection: per clip of the asset, a plain shift (a raw clip is identity between source and
-raw-virtual time), so a dwell replayed by two clips over one recording yields one zoom on
-each. Each clip sees only the samples inside its own source window, so a dwell a cut split
-across two clips is no longer one dwell — the cursor did not sit still across the cut on
-the timeline the user is watching.
+projection: per clip, a plain shift (a raw clip is identity between source and raw-virtual
+time), so a click replayed by two clips over one recording yields one zoom on each. Each
+clip plans its own zooms from the samples inside its own source window, so no zoom spans a
+cut. The plans are then settled over the whole timeline, every recording at once: two
+zooms must leave a wide shot between them, even across a cut, and the edit's first and
+last seconds stay wide.
### The same ambiguity in the transcript pane
diff --git a/workbench/l1/end-to-end.wb.ts b/workbench/l1/end-to-end.wb.ts
index 8a4c8c603..fcff0671e 100644
--- a/workbench/l1/end-to-end.wb.ts
+++ b/workbench/l1/end-to-end.wb.ts
@@ -552,8 +552,8 @@ describe("cursor telemetry on the wire", () => {
.filter((p: { kind?: string }) => p.kind === "click")
.map((p: { atSec: number }) => p.atSec),
).toEqual([3, 9.01, 17.03]);
- // The 6 s hold at 9 s is past MAX_DWELL_DURATION_MS — the magic wand's
- // detector would drop it. The track has no such ceiling: it is simply
+ // The 6 s hold at 9 s is longer than any the magic wand's old stillness
+ // detector kept (2.6 s). The track has no such ceiling: it is simply
// there, as points, for the model to read.
const held = track.points.filter((p: { atSec: number }) => p.atSec >= 6 && p.atSec <= 12);
expect(held.length).toBeGreaterThan(2);
diff --git a/workbench/lib/editorial.ts b/workbench/lib/editorial.ts
index 5f452067b..9f4f28150 100644
--- a/workbench/lib/editorial.ts
+++ b/workbench/lib/editorial.ts
@@ -407,8 +407,8 @@ export interface ZoomIssue {
}
/**
- * Shortest zoom that is not a flash: the app's own floor for a cursor dwell
- * worth zooming on (`zoom-suggestions.ts:11`, 450 ms). Below it the transition
+ * Shortest zoom that is not a flash: 450 ms, the floor the app's old stillness
+ * detector set for a dwell worth zooming on. Below it the transition
* has not finished before the region ends.
*/
export const MIN_ZOOM_SEC = 0.45;
diff --git a/workbench/lib/fixtures.ts b/workbench/lib/fixtures.ts
index af34e4c70..035217c45 100644
--- a/workbench/lib/fixtures.ts
+++ b/workbench/lib/fixtures.ts
@@ -422,8 +422,8 @@ export const DEMO_SPEECH_SPANS: Array<[number, number]> = [
];
/** Moments this take is about. A real pass would derive them from the cursor
- * telemetry digest (`timeline/zoom-suggestions.ts` already computes dwell
- * moments); these are hand-declared and say so. */
+ * telemetry (`timeline/zoom-suggestions.ts` plans zooms from its clicks); these
+ * are hand-declared and say so. */
export const DEMO_INTEREST_POINTS: InterestPoint[] = [
{ atSec: 14.2, label: "ouverture du panneau d'export", toleranceSec: 1.5 },
{ atSec: 38.9, label: "clic sur Rendre", toleranceSec: 1.5 },
@@ -484,11 +484,10 @@ export function withCameraTrack(options?: { projectId?: string }): AxcutDocument
* Synthetic pointer telemetry: 30 Hz movement with the cursor parked at declared
* moments, plus a click at the centre of each park.
*
- * ponytail: written as MOVEMENT and let through the real dwell detector, not
- * handed to the digest as pre-made moments. A fixture that fabricates the answer
- * measures nothing — the whole question is whether
- * `detectZoomDwellCandidates` finds the holds, including the long ones the magic
- * wand's ceiling throws away, and it can only be asked with samples.
+ * ponytail: written as MOVEMENT, not handed over as pre-made moments. A fixture
+ * that fabricates the answer measures nothing — the whole question is whether
+ * the reader finds the holds, including the long ones the magic wand's old
+ * stillness detector threw away, and it can only be asked with samples.
*/
export function cursorTelemetry(options: {
/** `{ atSec, holdSec, cx, cy }` — where the pointer stopped and for how long. */
@@ -504,8 +503,8 @@ export function cursorTelemetry(options: {
const endMs = options.durationSec * 1000;
const travelTo = (targetMs: number) => {
- // A visible move between parks: >DWELL_MOVE_THRESHOLD per step, so the
- // detector closes the previous run instead of merging the two.
+ // A visible move between parks, 5% of the frame per step, so two parks
+ // never read as one.
while (timeMs < targetMs) {
cursor = {
cx: Math.min(1, cursor.cx + 0.05) % 1,
diff --git a/workbench/scenarios/cursor-question.scn.ts b/workbench/scenarios/cursor-question.scn.ts
index 15b1e3103..7b02dedd8 100644
--- a/workbench/scenarios/cursor-question.scn.ts
+++ b/workbench/scenarios/cursor-question.scn.ts
@@ -52,8 +52,8 @@ const ASK =
"What cursor or pointer tracking data does this project contain? " +
"Answer from what you can actually see.";
-/** Three parks, one of them 6 s — past `MAX_DWELL_DURATION_MS`, so it exists in
- * the digest only because the digest lifts the magic wand's ceiling. */
+/** Three parks, one of them 6 s: longer than the 2.6 s the magic wand's old
+ * stillness detector kept, so only a reader of the raw track sees it. */
const DWELLS = [
{ atSec: 3, holdSec: 1.2, cx: 0.3, cy: 0.4 },
{ atSec: 9, holdSec: 6, cx: 0.72, cy: 0.55 },
diff --git a/workbench/scenarios/real-screencast.scn.ts b/workbench/scenarios/real-screencast.scn.ts
index 38eda0705..c15d1f8f5 100644
--- a/workbench/scenarios/real-screencast.scn.ts
+++ b/workbench/scenarios/real-screencast.scn.ts
@@ -30,7 +30,7 @@
// single uninterrupted block from 23.41 to 30.99.
// • the pointer is not still there either. It sweeps: cx 0.32 → 0.63 at a
// constant cy ≈ 0.50 between 24.1 and 29.2 s. A stillness detector — the
-// one the magic wand uses — is blind to it BY CONSTRUCTION, and the
+// one the magic wand used before clicks — is blind to it BY CONSTRUCTION, and the
// measurement in `cursor-track.ts` says so: 8 false positives out of 16
// dwells, and this zone fragmented into two one-second blips.
//