Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(agent): preserve recorded cursor visibility
Satwik Sharma5c4dcbc45f50
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.tsindex 96bdc852c..10a3af4c4 100644--- a/electron/ai-edition/agent-tools.test.ts+++ b/electron/ai-edition/agent-tools.test.ts@@ -1686,6 +1686,29 @@ describe("getCursorTrack", () => {expect(result.resultJson).not.toContain('virtualSec":');});+ it("reports a hidden click and the return to visibility through the tool payload", () => {+ const samples = [+ { timeMs: 0, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },+ { timeMs: 50, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" },+ { timeMs: 100, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },+ { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },+ ];+ const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", {+ cursorTelemetry: { load: { status: "ok", assetId: "asset_1", samples } },+ });+ const payload = JSON.parse(result.resultJson);++ expect(result.ok).toBe(true);+ expect(payload.points).toContainEqual({+ atSec: 0.05,+ cx: 0.5,+ cy: 0.5,+ kind: "click",+ visible: false,+ });+ expect(payload.points).toContainEqual({ atSec: 0.1, cx: 0.5, cy: 0.5 });+ });+it("says 'no-sidecar' when the asset was checked and has none", () => {const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", {cursorTelemetry: { load: { status: "no-sidecar", assetId: "asset_1" } },diff --git a/src/lib/ai-edition/timeline/cursor-track.test.ts b/src/lib/ai-edition/timeline/cursor-track.test.tsindex b773e099b..941d10d65 100644--- a/src/lib/ai-edition/timeline/cursor-track.test.ts+++ b/src/lib/ai-edition/timeline/cursor-track.test.ts@@ -76,6 +76,43 @@ describe("buildCursorTrack", () => {expect(click?.atSec).toBeCloseTo(13 * 0.05, 2);});+ it("keeps both visibility changes even when a stationary cursor is sampled below their rate", () => {+ const samples = Array.from({ length: 8 }, (_, i) => ({+ timeMs: i * 50,+ cx: 0.5,+ cy: 0.5,+ assetId: "arrow",+ interactionType: "move",+ visible: i === 3 ? false : undefined,+ }));+ const track = build(samples, 0.1);++ expect(track.points.find((point) => point.atSec === 0.15)).toMatchObject({ visible: false });+ expect(track.points.find((point) => point.atSec === 0.2)).toBeDefined();+ expect(track.points.find((point) => point.atSec === 0.2)).not.toHaveProperty("visible");+ });++ it("marks a recorded click as hidden instead of presenting it as visible to the agent", () => {+ const samples = [+ { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move", visible: true },+ { timeMs: 500, cx: 0.5, cy: 0.5, interactionType: "click", visible: false },+ { timeMs: 1000, cx: 0.5, cy: 0.5, interactionType: "move", visible: true },+ ];+ expect(build(samples).points.find((point) => point.kind === "click")).toMatchObject({+ atSec: 0.5,+ visible: false,+ });+ });++ it("treats legacy samples without visibility as visible, omitting the default field", () => {+ const legacy = sweep(40);+ const explicit = legacy.map((sample) => ({ ...sample, visible: true }));+ const legacyTrack = build(legacy);++ expect(build(explicit).points).toEqual(legacyTrack.points);+ expect(legacyTrack.points.every((point) => !("visible" in point))).toBe(true);+ });+it("drops to a coarser rate rather than blowing the ceiling, and says so", () => {// 40 minutes at 20 Hz: 5 Hz would be 12 000 points.const track = buildCursorTrack({diff --git a/src/lib/ai-edition/timeline/cursor-track.ts b/src/lib/ai-edition/timeline/cursor-track.tsindex b82db3648..6a700f383 100644--- a/src/lib/ai-edition/timeline/cursor-track.ts+++ b/src/lib/ai-edition/timeline/cursor-track.ts@@ -1,5 +1,5 @@// The recorded cursor track the agent reads: where the pointer was, when, and-// what shape it had. Pure — no fs, no IPC. The caller supplies the samples.+// what shape it had, and when it was visible. Pure — no fs, no IPC. The caller supplies the samples.//// ponytail: this is an OBSERVATION, not an interpretation, and the distinction// is the whole point of the module. Its predecessor handed the model a list of@@ -28,6 +28,8 @@ export interface CursorTrackSample {timeMs: number;cx: number;cy: number;+ /** Absent in older sidecars, where the pointer is treated as visible. */+ visible?: boolean;/** The cursor BITMAP's id, not a media asset: the sidecar stores one entry per* distinct pointer image (arrow, hand, text caret, resize…). A change means the* pointer shape changed, which is why these points are never dropped. */@@ -44,6 +46,8 @@ export interface CursorTrackPoint {virtualSec?: number | null;cx: number;cy: number;+ /** Present only while the recorded pointer was hidden. Absent means visible. */+ visible?: false;/** Small stable index per distinct pointer shape within THIS track. Absent when* the recording carries no shape information. */shape?: number;@@ -68,7 +72,7 @@ export interface CursorTrack {truncated: boolean;/** Present ONLY when the ceiling did not hold. `maxPoints` budgets the rate and* the gap floor; the MANDATORY points are exempt and stack on top — the first- * and last sample, a pointer-shape change, a non-move event, the ends of a run+ * and last sample, a pointer-shape or visibility change, a non-move event, the ends of a run* longer than the max gap — so a capture rich in them lands above the ceiling.* Absent means the budget held. It is a separate field from `truncated` on* purpose: that one says "you are seeing less than you asked for", this one@@ -240,8 +244,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {// saving — 356 grid points → 170. A parked pointer collapses for the same reason: the// chord through a stationary run is a point, so every in-between lies on it.//- // Three things outrank the tolerance and are never dropped, because no interpolation- // puts them back: a pointer-shape change, a non-move event, and the ends of a run+ // Four things outrank the tolerance and are never dropped, because no interpolation+ // puts them back: a pointer-shape change, a visibility change, a non-move event, and the ends of a run// longer than `maxGapMs` (a parked cursor must read as "still here", never as missing// data). `minIntervalMs` then caps the rate so a thrashing pointer cannot spend the// whole budget in one second.@@ -252,16 +256,19 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {mandatory.add(ordered.length - 1);}let lastShape: string | null | undefined = ordered[0]?.assetId;+ let lastVisible = ordered[0]?.visible !== false;let lastMandatoryMs = ordered[0]?.timeMs ?? 0;for (let i = 0; i < ordered.length; i += 1) {const s = ordered[i];const shapeChanged = s.assetId !== lastShape && shapeIndex.size > 1;+ const visibilityChanged = (s.visible !== false) !== lastVisible;const notAMove = typeof s.interactionType === "string" && s.interactionType !== "move";const stale = s.timeMs - lastMandatoryMs >= maxGapMs;- if (shapeChanged || notAMove || stale) {+ if (shapeChanged || visibilityChanged || notAMove || stale) {mandatory.add(i);lastMandatoryMs = s.timeMs;lastShape = s.assetId;+ lastVisible = s.visible !== false;}}@@ -309,6 +316,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {cx: round3(s.cx),cy: round3(s.cy),};+ if (s.visible === false) point.visible = false;if (shifted) point.virtualSec = position ? round2(position.virtualTimeSec) : null;const shape = typeof s.assetId === "string" ? shapeIndex.get(s.assetId) : undefined;if (shape !== undefined && shapeIndex.size > 1) point.shape = shape;@@ -330,7 +338,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {const overBudget =points.length > maxPoints? `${points.length} points for a ceiling of ${maxPoints}: the mandatory points are ` +- `never dropped — the first and last sample, pointer-shape changes, non-move ` ++ `never dropped — the first and last sample, pointer-shape and visibility changes, non-move ` +`events and the ends of a parked run — and this recording has enough of them ` +`to land above the budget.`: undefined;@@ -352,7 +360,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {"the points. A null virtualSec means no clip carries that moment; trimmed:true means a " +"trim cuts it out of " +"playback, so a zoom there would never be seen. `shape` is an index into the pointer " +- "bitmaps this recording used: equal values are the same pointer, a change is a change.",+ "bitmaps this recording used: equal values are the same pointer, a change is a change. " ++ "visible:false marks a hidden pointer; absent means visible, including older recordings.",points,};}diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.mdindex 180402381..5136f0ae2 100644--- a/technical-documentation/architecture/ai-agent.md+++ b/technical-documentation/architecture/ai-agent.md@@ -47,7 +47,7 @@ The model never free-writes the project document. It can only call the fixed set|---|---|---|| `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, and `cutByJunction` when a clip junction does (the compositor renders each clip on its own, so a move running past its clip's edge is lost too), so the export jumps there (#1028). A side cut by both is blamed on the cut nearest the pill, where the jump is. Chained zooms (at most 1.5 s apart on screen, no trim or clip junction between them) report the pan instead: `panToZoomId` / `panUntilSec` on the first, `panFromZoomId` on the second, in place of its zoom-out and zoom-in (`chainedPans`). | Nothing. || `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. |-| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. |+| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. `visible:false` marks a hidden pointer; absent means visible, including older sidecars. Visibility changes survive downsampling even when the pointer stays still. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. || `addTrim` | Adds one source-time cut inside a clip. Like the zoom writes and `setTrim`, it reports `cutTransitions` (zoom, side, seconds lost, `by` trim or junction) when the edit newly cuts a zoom's move in or out; a warning, never a refusal. | `timeline.trimRanges`. || `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. || `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. |