Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

timeline: suggest zoom tracks from click clusters instead of dwell heuristics

webadderall691309869cf9
diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts
new file mode 100644
index 00000000..6994025d
--- /dev/null
+++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts
@@ -0,0 +1,152 @@
+import { describe, expect, it } from "vitest";
+import {
+ CLICK_CLUSTER_MERGE_GAP_MS,
+ CLICK_CLUSTER_PAD_MS,
+ buildInteractionZoomSuggestions,
+} from "./zoomSuggestionUtils";
+import type { CursorTelemetryPoint } from "../types";
+
+function makeClick(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint {
+ return { timeMs, cx, cy, interactionType: "click" };
+}
+
+/** Wraps a list of click samples with surrounding move events so the telemetry
+ * passes the `normalizedSamples.length < 2` guard. */
+function withMoves(
+ clicks: CursorTelemetryPoint[],
+ totalMs: number,
+): CursorTelemetryPoint[] {
+ return [
+ { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move" },
+ ...clicks,
+ { timeMs: totalMs, cx: 0.5, cy: 0.5, interactionType: "move" },
+ ];
+}
+
+const TOTAL_MS = 30_000;
+
+describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
+ it("creates one zoom track for a single isolated click with 500ms padding", () => {
+ const telemetry = withMoves([makeClick(5_000)], TOTAL_MS);
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: telemetry,
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("ok");
+ expect(result.suggestions).toHaveLength(1);
+
+ const [s] = result.suggestions;
+ expect(s.start).toBe(5_000 - CLICK_CLUSTER_PAD_MS);
+ expect(s.end).toBe(5_000 + CLICK_CLUSTER_PAD_MS);
+ });
+
+ it("merges two clicks within 2500ms into one zoom track", () => {
+ const telemetry = withMoves(
+ [makeClick(4_000), makeClick(4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1)],
+ TOTAL_MS,
+ );
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: telemetry,
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("ok");
+ expect(result.suggestions).toHaveLength(1);
+
+ const [s] = result.suggestions;
+ const lastClickMs = 4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1;
+ expect(s.start).toBe(4_000 - CLICK_CLUSTER_PAD_MS);
+ expect(s.end).toBe(lastClickMs + CLICK_CLUSTER_PAD_MS);
+ });
+
+ it("splits two clicks more than 2500ms apart into separate zoom tracks", () => {
+ const click1 = 3_000;
+ const click2 = 3_000 + CLICK_CLUSTER_MERGE_GAP_MS + 1; // just outside the merge gap
+
+ const telemetry = withMoves([makeClick(click1), makeClick(click2)], TOTAL_MS);
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: telemetry,
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("ok");
+ expect(result.suggestions).toHaveLength(2);
+
+ const [a, b] = result.suggestions;
+ expect(a.start).toBe(click1 - CLICK_CLUSTER_PAD_MS);
+ expect(a.end).toBe(click1 + CLICK_CLUSTER_PAD_MS);
+ expect(b.start).toBe(click2 - CLICK_CLUSTER_PAD_MS);
+ expect(b.end).toBe(click2 + CLICK_CLUSTER_PAD_MS);
+ });
+
+ it("chains multiple clicks: 3 in a row within 2500ms each become one track", () => {
+ // click at 0, 2000, 4000 — each gap is 2000ms < 2500ms
+ const telemetry = withMoves([makeClick(0), makeClick(2_000), makeClick(4_000)], TOTAL_MS);
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: telemetry,
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("ok");
+ expect(result.suggestions).toHaveLength(1);
+
+ const [s] = result.suggestions;
+ expect(s.start).toBe(0); // clamped to 0 (would be -500)
+ expect(s.end).toBe(4_000 + CLICK_CLUSTER_PAD_MS);
+ });
+
+ it("returns no-interactions when there are no click telemetry points", () => {
+ // Move events only — no clicks
+ const telemetry: CursorTelemetryPoint[] = [
+ { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move" },
+ { timeMs: 1_000, cx: 0.5, cy: 0.5, interactionType: "move" },
+ { timeMs: 2_000, cx: 0.6, cy: 0.6, interactionType: "move" },
+ { timeMs: TOTAL_MS, cx: 0.6, cy: 0.6, interactionType: "move" },
+ ];
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: telemetry,
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("no-interactions");
+ expect(result.suggestions).toHaveLength(0);
+ });
+
+ it("skips clusters that overlap reserved spans", () => {
+ const click = 5_000;
+
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: withMoves([makeClick(click)], TOTAL_MS),
+ totalMs: TOTAL_MS,
+ defaultDurationMs: 3_000,
+ reservedSpans: [{ start: 4_000, end: 6_000 }], // overlaps the cluster window
+ });
+
+ expect(result.status).toBe("no-slots");
+ expect(result.suggestions).toHaveLength(0);
+ });
+
+ it("clamps start to 0 and end to totalMs at video boundaries", () => {
+ const result = buildInteractionZoomSuggestions({
+ cursorTelemetry: withMoves([makeClick(200)], 1_000),
+ totalMs: 1_000,
+ defaultDurationMs: 3_000,
+ });
+
+ expect(result.status).toBe("ok");
+ const [s] = result.suggestions;
+ expect(s.start).toBeGreaterThanOrEqual(0);
+ expect(s.end).toBeLessThanOrEqual(1_000);
+ });
+});
diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts
index 2df2db85..10de1548 100644
--- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts
+++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts
@@ -38,8 +38,10 @@ export interface InteractionZoomSuggestionResult {
suggestions: SuggestedZoomRegion[];
}
-const DEFAULT_SUGGESTION_SPACING_MS = 1800;
-const DEFAULT_MERGE_NEARBY_GAP_MS = 1500;
+/** Max gap between consecutive clicks before they are split into separate zoom clusters. */
+export const CLICK_CLUSTER_MERGE_GAP_MS = 2500;
+/** Padding added before the first click and after the last click in a cluster. */
+export const CLICK_CLUSTER_PAD_MS = 500;
function normalizeTelemetrySample(
sample: CursorTelemetryPoint,
@@ -254,6 +256,73 @@ export function detectInteractionCandidates(
return [...explicitInteractionCandidates, ...dwellCandidates, ...doubleClickCandidates];
}
+/**
+ * Groups a sorted list of click timestamps into clusters where consecutive
+ * clicks are no more than `mergeGapMs` apart. Returns an array of
+ * `{ firstMs, lastMs, focus }` objects, one per cluster. The focus is taken
+ * from the click with the highest interaction strength, falling back to the
+ * centroid of all clicks in the cluster.
+ */
+function buildClickClusters(
+ clicks: CursorInteractionCandidate[],
+ mergeGapMs: number,
+): Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> {
+ if (clicks.length === 0) {
+ return [];
+ }
+
+ const sorted = [...clicks].sort((a, b) => a.centerTimeMs - b.centerTimeMs);
+ const clusters: Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> = [];
+
+ let clusterStart = sorted[0].centerTimeMs;
+ let clusterEnd = sorted[0].centerTimeMs;
+ let bestStrength = sorted[0].strength;
+ let bestFocus = sorted[0].focus;
+ let sumCx = sorted[0].focus.cx;
+ let sumCy = sorted[0].focus.cy;
+ let count = 1;
+
+ for (let i = 1; i < sorted.length; i++) {
+ const click = sorted[i];
+ const gap = click.centerTimeMs - clusterEnd;
+
+ if (gap <= mergeGapMs) {
+ // Extend current cluster
+ clusterEnd = Math.max(clusterEnd, click.centerTimeMs);
+ if (click.strength > bestStrength) {
+ bestStrength = click.strength;
+ bestFocus = click.focus;
+ }
+ sumCx += click.focus.cx;
+ sumCy += click.focus.cy;
+ count += 1;
+ } else {
+ // Flush current cluster and start a new one
+ clusters.push({
+ firstMs: clusterStart,
+ lastMs: clusterEnd,
+ focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
+ });
+ clusterStart = click.centerTimeMs;
+ clusterEnd = click.centerTimeMs;
+ bestStrength = click.strength;
+ bestFocus = click.focus;
+ sumCx = click.focus.cx;
+ sumCy = click.focus.cy;
+ count = 1;
+ }
+ }
+
+ // Flush last cluster
+ clusters.push({
+ firstMs: clusterStart,
+ lastMs: clusterEnd,
+ focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
+ });
+
+ return clusters;
+}
+
export function buildInteractionZoomSuggestions(params: {
cursorTelemetry: CursorTelemetryPoint[];
totalMs: number;
@@ -261,18 +330,17 @@ export function buildInteractionZoomSuggestions(params: {
reservedSpans?: Array<{ start: number; end: number }>;
spacingMs?: number;
mergeGapMs?: number;
+ padMs?: number;
}): InteractionZoomSuggestionResult {
const {
cursorTelemetry,
totalMs,
- defaultDurationMs,
reservedSpans = [],
- spacingMs = DEFAULT_SUGGESTION_SPACING_MS,
- mergeGapMs = DEFAULT_MERGE_NEARBY_GAP_MS,
+ mergeGapMs = CLICK_CLUSTER_MERGE_GAP_MS,
+ padMs = CLICK_CLUSTER_PAD_MS,
} = params;
- const defaultDuration = Math.min(defaultDurationMs, totalMs);
- if (defaultDuration <= 0) {
+ if (totalMs <= 0) {
return { status: "no-slots", suggestions: [] };
}
@@ -281,62 +349,53 @@ export function buildInteractionZoomSuggestions(params: {
return { status: "no-telemetry", suggestions: [] };
}
- const interactionCandidates = detectInteractionCandidates(normalizedSamples);
- if (interactionCandidates.length === 0) {
+ // Only use explicit click events (uiohook telemetry) – ignore dwell heuristics
+ const clickCandidates = detectInteractionCandidates(normalizedSamples).filter(
+ (c) => c.kind === "click-like" || c.kind === "double-click-like" || c.kind === "dropdown-open" || c.kind === "text-field-click" || c.kind === "text-selection",
+ );
+
+ if (clickCandidates.length === 0) {
return { status: "no-interactions", suggestions: [] };
}
- const sortedCandidates = [...interactionCandidates].sort((a, b) => b.strength - a.strength);
- const acceptedCenters: number[] = [];
- const accepted: SuggestedZoomRegion[] = [];
+ // Group nearby clicks into clusters, then derive zoom windows from those clusters
+ const clusters = buildClickClusters(clickCandidates, mergeGapMs);
+
const reserved = [...reservedSpans].sort((a, b) => a.start - b.start);
+ const suggestions: SuggestedZoomRegion[] = [];
- sortedCandidates.forEach((candidate) => {
- const tooCloseToAccepted = acceptedCenters.some(
- (center) => Math.abs(center - candidate.centerTimeMs) < spacingMs,
- );
+ for (const cluster of clusters) {
+ const regionStart = Math.max(0, cluster.firstMs - padMs);
+ const regionEnd = Math.min(totalMs, cluster.lastMs + padMs);
- if (tooCloseToAccepted) {
- return;
+ if (regionEnd <= regionStart) {
+ continue;
}
- const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2);
- const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration));
- const candidateEnd = candidateStart + defaultDuration;
const hasOverlap = reserved.some(
- (span) => candidateEnd > span.start && candidateStart < span.end,
+ (span) => regionEnd > span.start && regionStart < span.end,
);
if (hasOverlap) {
- return;
- }
-
- reserved.push({ start: candidateStart, end: candidateEnd });
- acceptedCenters.push(candidate.centerTimeMs);
- accepted.push({
- start: candidateStart,
- end: candidateEnd,
- focus: candidate.focus,
- });
- });
-
- const sortedAccepted = [...accepted].sort((a, b) => a.start - b.start);
- const merged: SuggestedZoomRegion[] = [];
- for (const region of sortedAccepted) {
- const previous = merged[merged.length - 1];
- if (previous && region.start - previous.end <= mergeGapMs) {
- previous.end = Math.max(previous.end, region.end);
continue;
}
- merged.push({ ...region });
+ reserved.push({ start: regionStart, end: regionEnd });
+ suggestions.push({
+ start: regionStart,
+ end: regionEnd,
+ focus: cluster.focus,
+ });
}
- if (merged.length === 0) {
+ if (suggestions.length === 0) {
return { status: "no-slots", suggestions: [] };
}
- return { status: "ok", suggestions: merged };
+ // Sort chronologically
+ suggestions.sort((a, b) => a.start - b.start);
+
+ return { status: "ok", suggestions };
}
/**