Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix: ignore hidden cursor telemetry in auto-zoom

NoahSeeger03915a2969cc
diff --git a/electron/media/cursorSidecar.test.ts b/electron/media/cursorSidecar.test.ts
index 1c0e46a4d..88c7ed0da 100644
--- a/electron/media/cursorSidecar.test.ts
+++ b/electron/media/cursorSidecar.test.ts
@@ -149,7 +149,9 @@ describe("readCursorTelemetryFile", () => {
const result = await readCursorTelemetryFile(video, {});
expect(result.success).toBe(true);
- expect(result.samples).toEqual([{ timeMs: 10, cx: 0.5, cy: 0.5, interactionType: "click" }]);
+ expect(result.samples).toEqual([
+ { timeMs: 10, cx: 0.5, cy: 0.5, visible: true, interactionType: "click" },
+ ]);
// This projection feeds the ai-edition auto-zoom detector, whose click
// candidates ARE the recorded interactions (issue #699). It used to strip
// the field — locked by a test, even — and the detector never saw a click
@@ -158,6 +160,18 @@ describe("readCursorTelemetryFile", () => {
// the extra field.
});
+ it("keeps hidden cursor state for auto-zoom", async () => {
+ const video = path.join(dir, "hidden.mp4");
+ await writeSidecar(video, {
+ samples: [{ timeMs: 10, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }],
+ });
+
+ const result = await readCursorTelemetryFile(video, {});
+
+ expect(result.success).toBe(true);
+ expect(result.samples[0].visible).toBe(false);
+ });
+
it("keeps every click kind, not only the plain left click", async () => {
const video = path.join(dir, "kinds.mp4");
await writeSidecar(video, {
diff --git a/electron/media/cursorSidecar.ts b/electron/media/cursorSidecar.ts
index a47f63f5b..0a588e0ae 100644
--- a/electron/media/cursorSidecar.ts
+++ b/electron/media/cursorSidecar.ts
@@ -242,6 +242,7 @@ export async function readCursorTelemetryFile(
timeMs: sample.timeMs,
cx: sample.cx,
cy: sample.cy,
+ visible: sample.visible,
interactionType: sample.interactionType,
})),
};
diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts
index 312c9a70d..dda4abbf7 100644
--- a/src/components/video-editor/types.ts
+++ b/src/components/video-editor/types.ts
@@ -263,6 +263,7 @@ export interface CursorTelemetryPoint {
timeMs: number;
cx: number;
cy: number;
+ visible?: boolean;
interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup";
cursorType?:
| "arrow"
diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
index f36b1cc32..accf82f68 100644
--- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
+++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts
@@ -52,6 +52,105 @@ describe("detectZoomDwellCandidates", () => {
});
describe("buildAutoZoomSuggestions", () => {
+ it("ignores a click made while the cursor is hidden", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [{ ...click(2000, 0.5, 0.5), visible: false }],
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toEqual([]);
+ });
+
+ it("ignores a visible mouse-down when the cursor hides during that drag", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [
+ { ...click(1000, 0.5, 0.5), visible: true },
+ { timeMs: 1016, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
+ { timeMs: 1600, cx: 0.7, cy: 0.5, visible: false, interactionType: "mouseup" },
+ ],
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toEqual([]);
+ });
+
+ it("keeps a click released before the cursor later hides", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [
+ { ...click(1000, 0.5, 0.5), visible: true },
+ { timeMs: 1050, cx: 0.5, cy: 0.5, visible: true, interactionType: "mouseup" },
+ { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" },
+ ],
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toHaveLength(1);
+ expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 });
+ });
+
+ it("keeps a later visible click after a hidden drag", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [
+ { ...click(1000, 0.5, 0.5), visible: true },
+ { timeMs: 1100, cx: 0.6, cy: 0.5, visible: false, interactionType: "move" },
+ { timeMs: 1600, cx: 0.7, cy: 0.5, visible: true, interactionType: "mouseup" },
+ { ...click(4000, 0.2, 0.8), visible: true },
+ ],
+ totalMs: 6000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toHaveLength(1);
+ expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 });
+ });
+
+ it("does not join visible samples across a hidden interval into a dwell", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [
+ { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true },
+ { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true },
+ { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false },
+ { timeMs: 1800, cx: 0.5, cy: 0.5, visible: true },
+ { timeMs: 2000, cx: 0.5, cy: 0.5, visible: true },
+ ],
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toEqual([]);
+ });
+
+ it("ignores a dwell recorded while the cursor is hidden", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: dwell(2000, 0.5, 0.5).map((sample) => ({
+ ...sample,
+ visible: false,
+ })),
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toEqual([]);
+ });
+
+ it("keeps a visible dwell beside hidden cursor samples", () => {
+ const suggestions = buildAutoZoomSuggestions({
+ cursorTelemetry: [
+ ...dwell(1000, 0.2, 0.8),
+ { timeMs: 2000, cx: 0.2, cy: 0.8, visible: false },
+ ],
+ totalMs: 5000,
+ existingRegions: [],
+ defaultDurationMs: 2000,
+ });
+ expect(suggestions).toHaveLength(1);
+ expect(suggestions[0].focus.cx).toBeCloseTo(0.2);
+ expect(suggestions[0].focus.cy).toBeCloseTo(0.8);
+ });
+
it("returns a centered span around each accepted dwell", () => {
const telemetry = dwell(2000, 0.5, 0.5);
const suggestions = buildAutoZoomSuggestions({
diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts
index b82872267..145b892cd 100644
--- a/src/lib/ai-edition/timeline/zoom-suggestions.ts
+++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts
@@ -30,6 +30,7 @@ function normalizeTelemetrySample(
timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)),
cx: Math.max(0, Math.min(sample.cx, 1)),
cy: Math.max(0, Math.min(sample.cy, 1)),
+ visible: sample.visible,
interactionType: sample.interactionType,
};
}
@@ -92,7 +93,13 @@ export function detectZoomDwellCandidates(
});
};
- for (let index = 1; index < samples.length; index += 1) {
+ for (let index = 0; index < samples.length; index += 1) {
+ if (samples[index].visible === false) {
+ pushRunIfDwell(runStart, index);
+ runStart = index + 1;
+ continue;
+ }
+ if (index <= runStart) continue;
const prev = samples[index - 1];
const curr = samples[index];
const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy);
@@ -119,16 +126,32 @@ const CLICK_INTERACTION_TYPES: ReadonlySet<NonNullable<CursorTelemetryPoint["int
* ties between two clicks, where the stable sort keeps the earlier one.
*/
function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCandidate[] {
- return samples
- .filter(
- (sample) =>
- sample.interactionType !== undefined && CLICK_INTERACTION_TYPES.has(sample.interactionType),
- )
- .map((sample) => ({
- centerTimeMs: sample.timeMs,
- focus: { cx: sample.cx, cy: sample.cy },
- strength: 0,
- }));
+ const candidates: ZoomDwellCandidate[] = [];
+ let pendingClick: CursorTelemetryPoint | null = null;
+ let hiddenDuringPress = false;
+ const finishPress = () => {
+ if (pendingClick && !hiddenDuringPress) {
+ candidates.push({
+ centerTimeMs: pendingClick.timeMs,
+ focus: { cx: pendingClick.cx, cy: pendingClick.cy },
+ strength: 0,
+ });
+ }
+ pendingClick = null;
+ hiddenDuringPress = false;
+ };
+
+ for (const sample of samples) {
+ if (sample.interactionType && CLICK_INTERACTION_TYPES.has(sample.interactionType)) {
+ finishPress();
+ pendingClick = sample.visible === false ? null : sample;
+ } else {
+ if (sample.visible === false) hiddenDuringPress = true;
+ if (sample.interactionType === "mouseup") finishPress();
+ }
+ }
+ finishPress();
+ return candidates;
}
export interface AutoZoomSuggestion {
diff --git a/src/native/contracts.ts b/src/native/contracts.ts
index a2115e7f6..10164ad18 100644
--- a/src/native/contracts.ts
+++ b/src/native/contracts.ts
@@ -27,6 +27,7 @@ export interface CursorTelemetryPoint {
timeMs: number;
cx: number;
cy: number;
+ visible?: boolean;
}
export interface CursorRecordingSample extends CursorTelemetryPoint {