Skip to content

Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.

Add Copilot describer: reconstruct intent + steps from a recording

adilei8c73408bd8b7
diff --git a/common/analysis.ts b/common/analysis.ts
new file mode 100644
index 0000000..b4e46ae
--- /dev/null
+++ b/common/analysis.ts
@@ -0,0 +1,113 @@
+import { z } from "zod";
+
+/**
+ * The structured result of the Copilot describer: a high-level **intent** plus an
+ * ordered list of **actions/steps**, produced by the multi-turn agent and revised
+ * from the user's natural-language feedback. This is the stable contract shared by
+ * the agent's `submit_analysis` tool, the IPC surface, and on-disk persistence
+ * (`analysis.json`). Kept separate from the deterministic `SessionBundle` (which is
+ * the agent's *input*); this is the agent's *output*.
+ */
+
+export const Confidence = z.enum(["high", "medium", "low"]);
+export type Confidence = z.infer<typeof Confidence>;
+
+/** One reconstructed action the user took, in order. */
+export const AnalysisStepSchema = z.object({
+ /** Stable id assigned by the agent (e.g. "s1"); used to target feedback. */
+ id: z.string(),
+ /** Short imperative label, e.g. "Search Google for 'interesting articles'". */
+ title: z.string(),
+ /** 1–3 sentences of what happened and why it matters. */
+ detail: z.string(),
+ /** Wall-clock epoch span (ms) this step covers, when known. */
+ startMs: z.number().optional(),
+ endMs: z.number().optional(),
+ /** Apps involved (e.g. ["Microsoft Edge"]). */
+ apps: z.array(z.string()).default([]),
+ /** Free-text evidence refs the agent leaned on (events, urls, frame files). */
+ evidence: z.array(z.string()).default([]),
+ confidence: Confidence.default("medium"),
+});
+export type AnalysisStep = z.infer<typeof AnalysisStepSchema>;
+
+/**
+ * The payload the agent submits via the `submit_analysis` tool. The engine wraps
+ * this into a full {@link Analysis} (adding sessionId/revision/timestamps/feedback).
+ */
+export const AnalysisSubmissionSchema = z.object({
+ /** Short 2–5 word label for lists/menus, e.g. "Research habit articles". */
+ title: z.string().default(""),
+ /** The user's overall goal, e.g. "Submit an expense report". */
+ intent: z.string(),
+ intentConfidence: Confidence.default("medium"),
+ /** Why the agent believes this is the intent (evidence-grounded). */
+ intentRationale: z.string().default(""),
+ steps: z.array(AnalysisStepSchema).default([]),
+});
+export type AnalysisSubmission = z.infer<typeof AnalysisSubmissionSchema>;
+
+/** One round of user feedback that was fed back into the session. */
+export const FeedbackEntrySchema = z.object({
+ /** The revision number this feedback produced. */
+ revision: z.number().int(),
+ at: z.number(),
+ /** Natural-language feedback on the overall intent, if any. */
+ overall: z.string().optional(),
+ /** Per-step natural-language notes. */
+ steps: z
+ .array(z.object({ stepId: z.string(), note: z.string() }))
+ .default([]),
+});
+export type FeedbackEntry = z.infer<typeof FeedbackEntrySchema>;
+
+/** The full, persisted analysis for a session. */
+export const AnalysisSchema = z.object({
+ version: z.literal(1),
+ sessionId: z.string(),
+ /** Bumped on every (re)analysis; revision 1 is the first pass. */
+ revision: z.number().int(),
+ createdAt: z.number(),
+ /** Short 2–5 word label for lists/menus. Empty on analyses saved before titles existed. */
+ title: z.string().default(""),
+ intent: z.string(),
+ intentConfidence: Confidence,
+ intentRationale: z.string(),
+ steps: z.array(AnalysisStepSchema),
+ /** Chronological log of feedback rounds applied so far. */
+ feedbackLog: z.array(FeedbackEntrySchema).default([]),
+ /** User has accepted this analysis as correct — the artifact a Skill is built from. */
+ approved: z.boolean().default(false),
+ /** When the user approved it (epoch ms). */
+ approvedAt: z.number().optional(),
+});
+export type Analysis = z.infer<typeof AnalysisSchema>;
+
+/** Feedback the UI sends for a re-analysis round. */
+export interface AnalysisFeedback {
+ /** NL feedback on the overall intent. */
+ overall?: string;
+ /** Per-step NL notes (only steps the user commented on). */
+ steps: { stepId: string; note: string }[];
+}
+
+/** Build a full Analysis from an agent submission + engine-managed fields. */
+export function toAnalysis(
+ sessionId: string,
+ revision: number,
+ submission: AnalysisSubmission,
+ feedbackLog: FeedbackEntry[],
+): Analysis {
+ return AnalysisSchema.parse({
+ version: 1,
+ sessionId,
+ revision,
+ createdAt: Date.now(),
+ title: submission.title,
+ intent: submission.intent,
+ intentConfidence: submission.intentConfidence,
+ intentRationale: submission.intentRationale,
+ steps: submission.steps,
+ feedbackLog,
+ });
+}
diff --git a/common/ipc.ts b/common/ipc.ts
index 30b31d7..b075d39 100644
--- a/common/ipc.ts
+++ b/common/ipc.ts
@@ -1,11 +1,69 @@
+import type { Analysis, AnalysisFeedback, Confidence } from "./analysis";
import type { CaptureConfig, CaptureLevel } from "./config";
import type { RecorderState } from "./types";
+/** The last completed session — the one that can be analyzed. */
+export interface LastSession {
+ id: string;
+ /** True once post-processing (bundle/description/frames) has finished. */
+ processed: boolean;
+}
+
+/** A saved recording as shown in the sessions library. */
+export interface SessionSummary {
+ id: string;
+ startedAt: number | null;
+ stoppedAt: number | null;
+ durationMs: number | null;
+ /** True once post-processing produced a bundle. */
+ processed: boolean;
+ hasVideo: boolean;
+ /** Present once the describer has produced an analysis for this session. */
+ analysis: {
+ revision: number;
+ title: string;
+ intent: string;
+ intentConfidence: Confidence;
+ stepCount: number;
+ approved: boolean;
+ } | null;
+}
+
export interface RecorderStatus {
state: RecorderState;
sessionId: string | null;
startedAt: number | null;
eventCount: number;
+ /** Set after a recording stops; drives the "Analyze" affordance. */
+ lastSession: LastSession | null;
+}
+
+/** Streamed to the renderer while the describer agent works. */
+export interface AnalyzeProgress {
+ sessionId: string;
+ phase: "start" | "working" | "drafting" | "done" | "error";
+ message: string;
+}
+
+/** Result of an analyze / feedback round. */
+export interface AnalyzeResult {
+ ok: boolean;
+ analysis?: Analysis;
+ error?: string;
+}
+
+/** Feedback payload sent from the renderer for a re-analysis round. */
+export interface AnalysisFeedbackInput extends AnalysisFeedback {
+ sessionId: string;
+}
+
+/** A direct text edit to the intent/title, applied without re-running the agent. */
+export interface AnalysisEditInput {
+ sessionId: string;
+ /** New short label; empty string clears it (list falls back to the intent). */
+ title?: string;
+ /** New one-sentence goal; blank/whitespace is ignored (intent can't be emptied). */
+ intent?: string;
}
export interface StartResult {
@@ -63,6 +121,16 @@ export const IPC = {
getCapture: "capture:get",
setLevel: "capture:set-level",
setConfig: "capture:set-config",
+ analyze: "analyze:start",
+ analyzeFeedback: "analyze:feedback",
+ getAnalysis: "analyze:get",
+ approveAnalysis: "analyze:approve",
+ updateAnalysis: "analyze:update",
+ cancelAnalysis: "analyze:cancel",
+ analyzeProgress: "analyze:progress",
+ listSessions: "sessions:list",
+ openLibrary: "ui:open-library",
+ closeLibrary: "ui:close-library",
} as const;
/** Shape exposed on `window.skillRecorder` by the preload bridge. */
@@ -76,4 +144,23 @@ export interface SkillRecorderApi {
setLevel(level: Exclude<CaptureLevel, "custom">): Promise<CaptureState>;
setConfig(config: CaptureConfig): Promise<CaptureState>;
onStatusChanged(cb: (status: RecorderStatus) => void): () => void;
+ /** Run the Copilot describer on a session (defaults to the last completed one). */
+ analyze(sessionId?: string): Promise<AnalyzeResult>;
+ /** Send NL feedback and re-analyze in the same multi-turn session. */
+ analyzeFeedback(input: AnalysisFeedbackInput): Promise<AnalyzeResult>;
+ /** Load the persisted analysis for a session, if any. */
+ getAnalysis(sessionId: string): Promise<Analysis | null>;
+ /** Mark an analysis as accepted/correct — the artifact a Skill is built from. */
+ approveAnalysis(sessionId: string): Promise<AnalyzeResult>;
+ /** Edit the title/intent text directly (no re-analysis). */
+ updateAnalysis(input: AnalysisEditInput): Promise<AnalyzeResult>;
+ /** Abort an in-flight analysis. */
+ cancelAnalysis(sessionId: string): Promise<{ ok: boolean }>;
+ onAnalyzeProgress(cb: (progress: AnalyzeProgress) => void): () => void;
+ /** All saved recordings, newest first, for the sessions library. */
+ listSessions(): Promise<SessionSummary[]>;
+ /** Open (and focus) the Sessions library window, docked to the recorder. */
+ openLibrary(): Promise<void>;
+ /** Close the Sessions library window from within it. */
+ closeLibrary(): Promise<void>;
}
diff --git a/electron/describer/describer.ts b/electron/describer/describer.ts
new file mode 100644
index 0000000..4267379
--- /dev/null
+++ b/electron/describer/describer.ts
@@ -0,0 +1,363 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import { approveAll, CopilotClient, type CopilotSession } from "@github/copilot-sdk";
+
+import {
+ AnalysisSchema,
+ toAnalysis,
+ type Analysis,
+ type AnalysisFeedback,
+ type AnalysisSubmission,
+} from "../../common/analysis";
+import type { AnalyzeProgress } from "../../common/ipc";
+import type { SessionMeta } from "../../common/types";
+import { FrameExtractor } from "../frames/extractor";
+import { createLogger } from "../logger";
+import { sessionsRoot } from "../recorder/session-store";
+import { DESCRIBER_INSTRUCTIONS } from "./instructions";
+import { createDescriberTools } from "./tools";
+
+const log = createLogger("Describer");
+
+/** How long a single agent turn may run before we give up (multi-tool loop). */
+const TURN_TIMEOUT_MS = 180_000;
+
+const KICKOFF_PROMPT =
+ "Reconstruct what the user did in this recording. Start with get_timeline, then read events " +
+ "where anything is unclear, and look at frames only where the events are ambiguous. When " +
+ "confident, call submit_analysis with the overall intent and the ordered list of steps.";
+
+const NUDGE_PROMPT =
+ "Please call submit_analysis now with your best analysis of the overall intent and the ordered steps.";
+
+const msg = (err: unknown) => (err instanceof Error ? err.message : String(err));
+
+interface VideoMeta {
+ file: string;
+ startEpoch: number;
+ durationMs: number;
+}
+
+/** A live, resumable analysis session for one recording. */
+interface LiveSession {
+ sessionId: string;
+ sessionDir: string;
+ copilot: CopilotSession;
+ revision: number;
+ feedbackLog: Analysis["feedbackLog"];
+ /** Mutable capture slot the `submit_analysis` tool writes into. */
+ holder: { submission: AnalysisSubmission | undefined };
+}
+
+function readJson<T>(file: string): T | null {
+ if (!existsSync(file)) return null;
+ try {
+ return JSON.parse(readFileSync(file, "utf8")) as T;
+ } catch {
+ return null;
+ }
+}
+
+/** Load a persisted analysis from a session dir (validated), or null. */
+export function loadPersistedAnalysis(sessionId: string): Analysis | null {
+ const file = path.join(sessionsRoot(), sessionId, "analysis.json");
+ const raw = readJson<unknown>(file);
+ if (!raw) return null;
+ const parsed = AnalysisSchema.safeParse(raw);
+ return parsed.success ? parsed.data : null;
+}
+
+/**
+ * Drives the multi-turn GitHub Copilot CLI agent that turns a recording's captured
+ * signals into an intent + ordered steps, and revises them from NL feedback. Owns
+ * a single shared {@link CopilotClient}; keeps one live session per recording so the
+ * feedback loop stays in the same conversation. Streams progress out via a callback.
+ * The deterministic baseline `description.md` remains the fallback when Copilot is
+ * unavailable (callers surface the thrown error).
+ */
+export class Describer {
+ private client: CopilotClient | null = null;
+ private clientStart: Promise<CopilotClient> | null = null;
+ private model: string | undefined;
+ private readonly live = new Map<string, LiveSession>();
+ private readonly active = new Set<string>();
+
+ constructor(private readonly emitProgress: (p: AnalyzeProgress) => void) {}
+
+ /** First pass: reconstruct the session from scratch. */
+ async analyze(sessionId: string): Promise<Analysis> {
+ if (this.active.has(sessionId)) throw new Error("An analysis is already running for this session.");
+ this.active.add(sessionId);
+ try {
+ this.emit(sessionId, "start", "Starting analysis…");
+ await this.disposeLive(sessionId); // fresh conversation each explicit analyze
+ const live = await this.createLive(sessionId);
+ return await this.runTurn(live, KICKOFF_PROMPT);
+ } finally {
+ this.active.delete(sessionId);
+ }
+ }
+
+ /** Later pass: fold in NL feedback and revise holistically. */
+ async feedback(sessionId: string, fb: AnalysisFeedback): Promise<Analysis> {
+ if (this.active.has(sessionId)) throw new Error("An analysis is already running for this session.");
+ this.active.add(sessionId);
+ try {
+ this.emit(sessionId, "start", "Re-analyzing with your feedback…");
+ const live = this.live.get(sessionId) ?? (await this.createLive(sessionId));
+ const prior = loadPersistedAnalysis(sessionId);
+ live.feedbackLog = [
+ ...live.feedbackLog,
+ { revision: live.revision + 1, at: Date.now(), overall: fb.overall, steps: fb.steps },
+ ];
+ return await this.runTurn(live, renderFeedbackPrompt(fb, prior));
+ } finally {
+ this.active.delete(sessionId);
+ }
+ }
+
+ /** Accept an analysis as correct. Persists `approved` so a Skill can be built from it. */
+ async approve(sessionId: string): Promise<Analysis> {
+ const prior = loadPersistedAnalysis(sessionId);
+ if (!prior) throw new Error("There is no analysis to approve yet.");
+ const approved: Analysis = { ...prior, approved: true, approvedAt: Date.now() };
+ const dir = path.join(sessionsRoot(), sessionId);
+ this.persist(dir, approved, this.live.get(sessionId)?.copilot.sessionId);
+ return approved;
+ }
+
+ /**
+ * Apply a direct text edit to the title and/or intent — a user correction, NOT a
+ * re-analysis. Steps and evidence are untouched; the agent is not invoked. Blank
+ * intent is ignored (a session always keeps a goal); the title may be cleared.
+ */
+ async edit(sessionId: string, patch: { title?: string; intent?: string }): Promise<Analysis> {
+ const prior = loadPersistedAnalysis(sessionId);
+ if (!prior) throw new Error("There is no analysis to edit yet.");
+ const next: Analysis = { ...prior };
+ if (patch.title !== undefined) next.title = patch.title.trim();
+ if (patch.intent !== undefined && patch.intent.trim()) next.intent = patch.intent.trim();
+ const dir = path.join(sessionsRoot(), sessionId);
+ this.persist(dir, next, this.live.get(sessionId)?.copilot.sessionId);
+ return next;
+ }
+
+ /** Abort an in-flight run for a session. */
+ async cancel(sessionId: string): Promise<void> {
+ const live = this.live.get(sessionId);
+ if (live) await live.copilot.abort().catch(() => undefined);
+ }
+
+ /** Tear down the client + all live sessions (called on app quit). */
+ async dispose(): Promise<void> {
+ for (const [id] of this.live) await this.disposeLive(id);
+ if (this.client) await this.client.stop().catch(() => undefined);
+ this.client = null;
+ this.clientStart = null;
+ }
+
+ // --- internals -----------------------------------------------------------
+
+ private emit(sessionId: string, phase: AnalyzeProgress["phase"], message: string): void {
+ this.emitProgress({ sessionId, phase, message });
+ }
+
+ private async ensureClient(): Promise<CopilotClient> {
+ if (this.client) return this.client;
+ if (this.clientStart) return this.clientStart;
+ this.clientStart = (async () => {
+ const client = new CopilotClient();
+ await client.start();
+ const auth = await client.getAuthStatus();
+ if (!auth.isAuthenticated) {
+ await client.stop().catch(() => undefined);
+ throw new Error(
+ "GitHub Copilot CLI is not signed in. Open a terminal, run `copilot`, sign in, then try again.",
+ );
+ }
+ this.model = await this.pickVisionModel(client);
+ log.info("Copilot ready", auth.login ? `as ${auth.login}` : "", this.model ? `· model ${this.model}` : "");
+ this.client = client;
+ return client;
+ })();
+ try {
+ return await this.clientStart;
+ } catch (err) {
+ this.clientStart = null;
+ throw err;
+ }
+ }
+
+ /** Prefer a vision-capable, enabled model (frames need vision). */
+ private async pickVisionModel(client: CopilotClient): Promise<string | undefined> {
+ const override = process.env.SKILL_RECORDER_MODEL;
+ if (override) return override;
+ try {
+ const models = await client.listModels();
+ const vision = models.filter(
+ (m) => m.capabilities?.supports?.vision && m.policy?.state !== "disabled",
+ );
+ return vision[0]?.id;
+ } catch (err) {
+ log.warn("listModels failed; using CLI default model:", msg(err));
+ return undefined;
+ }
+ }
+
+ private async createLive(sessionId: string): Promise<LiveSession> {
+ const sessionDir = path.join(sessionsRoot(), sessionId);
+ const meta = readJson<SessionMeta>(path.join(sessionDir, "session.json"));
+ if (!meta) throw new Error(`Session ${sessionId} not found or has no session.json.`);
+
+ const extractor = buildExtractor(sessionDir);
+ const holder: LiveSession["holder"] = { submission: undefined };
+
+ const tools = createDescriberTools({
+ sessionDir,
+ startedAt: meta.startedAt,
+ extractor,
+ onProgress: (m) => this.emit(sessionId, "working", m),
+ onSubmit: (s) => {
+ holder.submission = s;
+ },
+ });
+
+ const client = await this.ensureClient();
+ const config = {
+ systemMessage: { mode: "append" as const, content: DESCRIBER_INSTRUCTIONS },
+ tools,
+ onPermissionRequest: approveAll,
+ workingDirectory: sessionDir,
+ enableHostGitOperations: false,
+ infiniteSessions: { enabled: false },
+ ...(this.model ? { model: this.model } : {}),
+ };
+ // Restrict to our custom tools where supported; fall back if the runtime
+ // rejects an allowlist of custom names.
+ let copilot: CopilotSession;
+ try {
+ copilot = await client.createSession({ ...config, availableTools: tools.map((t) => t.name) });
+ } catch (err) {
+ log.warn("createSession with tool allowlist failed; retrying without it:", msg(err));
+ copilot = await client.createSession(config);
+ }
+
+ const prior = loadPersistedAnalysis(sessionId);
+ const live: LiveSession = {
+ sessionId,
+ sessionDir,
+ copilot,
+ revision: prior?.revision ?? 0,
+ feedbackLog: prior?.feedbackLog ?? [],
+ holder,
+ };
+ this.live.set(sessionId, live);
+ return live;
+ }
+
+ private async disposeLive(sessionId: string): Promise<void> {
+ const live = this.live.get(sessionId);
+ if (!live) return;
+ this.live.delete(sessionId);
+ await live.copilot.disconnect().catch(() => undefined);
+ }
+
+ private async runTurn(live: LiveSession, prompt: string): Promise<Analysis> {
+ live.holder.submission = undefined;
+ this.emit(live.sessionId, "working", "Thinking…");
+ try {
+ await live.copilot.sendAndWait(prompt, TURN_TIMEOUT_MS);
+ } catch (err) {
+ throw new Error(`Analysis run failed: ${msg(err)}`);
+ }
+ if (!live.holder.submission) {
+ this.emit(live.sessionId, "working", "Asking the agent to finalize its analysis…");
+ await live.copilot.sendAndWait(NUDGE_PROMPT, TURN_TIMEOUT_MS).catch(() => undefined);
+ }
+ const submission = live.holder.submission;
+ if (!submission) throw new Error("The agent finished without submitting an analysis.");
+
+ live.revision += 1;
+ this.emit(live.sessionId, "drafting", "Finalizing analysis…");
+ const analysis = toAnalysis(live.sessionId, live.revision, submission, [...live.feedbackLog]);
+ this.persist(live.sessionDir, analysis, live.copilot.sessionId);
+ this.emit(live.sessionId, "done", `Analysis ready (revision ${analysis.revision}).`);
+ return analysis;
+ }
+
+ private persist(sessionDir: string, analysis: Analysis, copilotSessionId?: string): void {
+ try {
+ writeFileSync(path.join(sessionDir, "analysis.json"), JSON.stringify(analysis, null, 2));
+ writeFileSync(path.join(sessionDir, "analysis.md"), renderAnalysisMarkdown(analysis));
+ if (copilotSessionId) {
+ writeFileSync(
+ path.join(sessionDir, "describer.json"),
+ JSON.stringify({ copilotSessionId }, null, 2),
+ );
+ }
+ } catch (err) {
+ log.warn("failed to persist analysis:", msg(err));
+ }
+ }
+}
+
+function buildExtractor(sessionDir: string): FrameExtractor | null {
+ const video = readJson<VideoMeta>(path.join(sessionDir, "video.json"));
+ if (!video) return null;
+ const videoPath = path.join(sessionDir, video.file);
+ if (!existsSync(videoPath)) return null;
+ return new FrameExtractor({
+ videoPath,
+ framesDir: path.join(sessionDir, "frames"),
+ anchorEpochMs: video.startEpoch,
+ durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,
+ });
+}
+
+function renderFeedbackPrompt(fb: AnalysisFeedback, prior: Analysis | null): string {
+ const lines: string[] = [
+ "The user reviewed your analysis and left feedback. Revise the ENTIRE analysis accordingly and",
+ "call submit_analysis again. Re-examine signals (get_events / get_frames) where the feedback",
+ "points to a gap or error. Keep step ids stable for steps that don't change.",
+ "",
+ ];
+ if (prior) {
+ lines.push("Your current analysis:");
+ lines.push(`- intent (${prior.intentConfidence}): ${prior.intent}`);
+ for (const s of prior.steps) lines.push(`- ${s.id}: ${s.title} — ${s.detail}`);
+ lines.push("");
+ }
+ lines.push(`Overall-intent feedback: ${fb.overall?.trim() ? fb.overall.trim() : "(none)"}`);
+ if (fb.steps.length) {
+ lines.push("Per-step feedback:");
+ for (const s of fb.steps) {
+ const title = prior?.steps.find((p) => p.id === s.stepId)?.title ?? "";
+ lines.push(`- ${s.stepId}${title ? ` (${title})` : ""}: ${s.note}`);
+ }
+ }
+ return lines.join("\n");
+}
+
+/** Human-readable rendering of an analysis (persisted alongside analysis.json). */
+export function renderAnalysisMarkdown(a: Analysis): string {
+ const out: string[] = [];
+ out.push(`# ${a.title?.trim() || "Session analysis"}`);
+ out.push("");
+ out.push(`**Intent** (${a.intentConfidence}): ${a.intent}`);
+ if (a.intentRationale) out.push(`\n${a.intentRationale}`);
+ out.push("");
+ out.push(`## Steps`);
+ a.steps.forEach((s, i) => {
+ const span =
+ s.startMs != null ? ` _(${(s.startMs / 1000).toFixed(1)}s${s.endMs != null ? `–${(s.endMs / 1000).toFixed(1)}s` : ""})_` : "";
+ out.push(`\n### ${i + 1}. ${s.title}${span}`);
+ out.push(s.detail);
+ if (s.apps.length) out.push(`\n- apps: ${s.apps.join(", ")}`);
+ if (s.evidence.length) out.push(`- evidence: ${s.evidence.join("; ")}`);
+ out.push(`- confidence: ${s.confidence}`);
+ });
+ out.push("");
+ out.push(`_Revision ${a.revision} · generated ${new Date(a.createdAt).toISOString()}_`);
+ return out.join("\n") + "\n";
+}
diff --git a/electron/describer/instructions.ts b/electron/describer/instructions.ts
new file mode 100644
index 0000000..25a948e
--- /dev/null
+++ b/electron/describer/instructions.ts
@@ -0,0 +1,103 @@
+/**
+ * The describer **brief** — the agent's system message (appended to the SDK
+ * foundation). It tells the Copilot CLI agent its job, the tools it has, the
+ * method to follow, and the exact structured output it must produce. This is the
+ * "skill" in the loose sense: a human-editable instruction document, NOT a
+ * packaged Copilot CLI skill directory and NOT a `.github/extensions` extension.
+ *
+ * Time model exposed to the agent: **`atMs` = milliseconds since the recording
+ * started** (0 = the moment the user hit Start). Every tool speaks this single
+ * clock; the mapping to video offsets is hidden behind the frame tools.
+ */
+export const DESCRIBER_INSTRUCTIONS = `
+# Role: Session Describer
+
+You reconstruct what a user did during a short screen-recording session and
+produce (1) their **overall intent** and (2) an **ordered list of the concrete
+actions** they took. Your output becomes the raw material for building an
+AI-agent "skill", so be accurate, specific, and grounded in the captured signals.
+
+## What was captured
+The recorder harvested cheap, high-signal OS events as the PRIMARY source:
+- **app switches** (which application was focused),
+- **window titles**,
+- **browser URLs** (the pages visited),
+- **clipboard changes** (copied text),
+- **terminal commands** (when the shell hook was active).
+
+A low-frame-rate **screen video** may also exist. It is OPPORTUNISTIC enrichment
+— you pull frames only where the events are ambiguous. Do NOT assume you must
+look at video; most steps are fully explained by events alone.
+
+All times are **\`atMs\` = milliseconds since the recording started**.
+
+## Your tools
+- **get_timeline** — the segmented timeline: ordered steps (app / urls / titles /
+ commands / clipboard counts / markers) with their \`atMs\` start + duration. Start here.
+- **get_events({ types?, fromMs?, toMs? })** — the raw event stream (with clipboard
+ text, full titles, full URLs, commands). Use to inspect a specific window closely.
+- **list_frames** — index of screen frames already available (file + \`atMs\` + why kept).
+ Empty/absent means no video was recorded.
+- **get_frames({ fromMs, toMs, fps?, crop?, reason? })** — sample and **view** screen
+ frames within a time window. Returns the images inline so you can actually see the
+ screen. Optional \`crop\` ({x,y,w,h}) zooms a region. This is your "look closer"
+ primitive — use it ONLY where events leave real ambiguity.
+- **submit_analysis({ title, intent, intentConfidence, intentRationale, steps })** — your
+ REQUIRED final action. Call it exactly once when confident. See the schema below.
+
+## Method
+1. **Read the timeline** (get_timeline) to get the shape of the session.
+2. **Form a hypothesis** about the overall intent from apps + urls + commands.
+3. **Read events** (get_events) around anything unclear — clipboard text, exact
+ URLs, the sequence of title changes.
+4. **Look at frames ONLY where events are silent or ambiguous** (get_frames): e.g. a
+ step with a visual change but no explaining event, a clipboard copy whose purpose
+ is unclear, or a terminal step with no captured command. Budget ~5 frames for a
+ ~30–60s session. Cost should scale with ambiguity, not video length.
+5. **Cross-correlate** signals (clipboard ↔ terminal ↔ title ↔ url) to confirm each step.
+6. **Call submit_analysis** with the intent and ordered steps.
+
+## Noise to ignore
+- **The Skill Recorder app itself** (this Electron recorder, app name "Skill Recorder").
+ Activating/focusing it is how the user reaches the Start and Stop buttons — it is NOT
+ part of their task. In particular, the FIRST step (focusing Skill Recorder to press Start,
+ usually at \`atMs\` ≈ 0) and the LAST step (returning to Skill Recorder to press Stop) are
+ recorder bracketing, not user actions — do NOT emit them as steps. Also drop any mid-session
+ flick back to Skill Recorder just to add a marker. The real task starts with the first
+ non-recorder app the user works in.
+- \`UserNotificationCenter\` / OS permission dialogs (e.g. "requesting to record the
+ screen") are the recorder's own consent prompts — NOT user actions. Skip them.
+- URL tracking params (\`gclid\`, \`gad_source\`, \`utm_*\`) and ad-redirect hops carry no
+ intent — treat two URLs that differ only in these as the same page.
+- Momentary app focus flickers (sub-second activations with no follow-up) are usually
+ not real steps.
+
+## Output schema (submit_analysis)
+- **title**: a SHORT 2–5 word label for the task, in Title Case with no trailing period,
+ e.g. "Research Habit Articles", "Extract Invoice Data", "Compare Pricing Plans". This is
+ the session's name in lists, so make it scannable — name the task, not the apps used.
+- **intent**: one sentence naming the user's goal, e.g. "Research and compare
+ articles on building better habits" or "Submit an expense report".
+- **intentConfidence**: "high" | "medium" | "low".
+- **intentRationale**: 1–2 sentences citing the evidence for the intent.
+- **steps[]**: ordered; each is:
+ - **id**: stable short id you assign, "s1", "s2", …
+ - **title**: short imperative label, e.g. "Search Google for 'atomic habits'".
+ - **detail**: 1–3 sentences of what happened and why it matters.
+ - **startMs / endMs**: the step's \`atMs\` span when known.
+ - **apps[]**: apps involved (e.g. ["Microsoft Edge"]).
+ - **evidence[]**: brief refs you relied on — event types, a URL, a frame file, a
+ copied string. Keep them short.
+ - **confidence**: "high" | "medium" | "low".
+
+## Handling feedback
+Later turns may deliver the user's natural-language feedback on your analysis
+(corrections to the intent, notes like "this step is irrelevant", "you missed a
+step", "not accurate"). When you receive feedback:
+- Treat it as authoritative. Re-examine the relevant signals (fetch more events or
+ frames if needed).
+- Produce a fully revised analysis and call **submit_analysis** again with the
+ improved intent + steps. Keep step ids stable where a step is unchanged.
+
+Always finish a turn by calling submit_analysis. Do not reply with prose instead.
+`.trim();
diff --git a/electron/describer/tools.ts b/electron/describer/tools.ts
new file mode 100644
index 0000000..d7399a1
--- /dev/null
+++ b/electron/describer/tools.ts
@@ -0,0 +1,302 @@
+import { existsSync, readFileSync } from "node:fs";
+import path from "node:path";
+
+import type { Tool } from "@github/copilot-sdk";
+
+import { AnalysisSubmissionSchema, type AnalysisSubmission } from "../../common/analysis";
+import { MEANINGFUL_EVENT_TYPES } from "../../common/correlation";
+import type { SessionBundle } from "../../common/bundle";
+import type { RecEvent } from "../../common/types";
+import { readEvents } from "../frames/correlate";
+import type { FrameExtractor } from "../frames/extractor";
+import { createLogger } from "../logger";
+
+const log = createLogger("Describer/tools");
+
+/** Everything a session's tools are bound to. */
+export interface ToolContext {
+ sessionDir: string;
+ /** `session.json` startedAt (epoch ms) — origin of the agent's `atMs` clock. */
+ startedAt: number;
+ /** Frame service, or null when the session has no video. */
+ extractor: FrameExtractor | null;
+ /** Streamed to the UI as the agent works. */
+ onProgress?: (message: string) => void;
+ /** Called when the agent submits a (validated) analysis. */
+ onSubmit: (submission: AnalysisSubmission) => void;
+}
+
+const MAX_IMAGES_PER_CALL = 6;
+const MAX_STR = 2000;
+
+const truncate = (v: unknown): unknown =>
+ typeof v === "string" && v.length > MAX_STR ? v.slice(0, MAX_STR) + "…[truncated]" : v;
+
+function readBundle(sessionDir: string): SessionBundle | null {
+ const p = path.join(sessionDir, "bundle.json");
+ if (!existsSync(p)) return null;
+ try {
+ return JSON.parse(readFileSync(p, "utf8")) as SessionBundle;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Build the custom in-process tools exposed to one Copilot session. Handlers run
+ * in the Electron main process (dispatched over JSON-RPC by the SDK), read the
+ * session's captured signals, and — for frames — return the JPEGs **inline** so
+ * the model can see the screen without any `view`/`bash` tool. `submit_analysis`
+ * captures guaranteed-structured output. The whole surface is sandboxed to the
+ * one session dir.
+ */
+export function createDescriberTools(ctx: ToolContext): Tool[] {
+ const { sessionDir, startedAt, extractor, onSubmit } = ctx;
+ const progress = (m: string) => ctx.onProgress?.(m);
+ const rel = (epoch: number) => Math.round(epoch - startedAt);
+ const abs = (atMs: number) => startedAt + atMs;
+ const sec = (atMs: number) => (atMs / 1000).toFixed(1);
+
+ const getTimeline: Tool = {
+ name: "get_timeline",
+ description:
+ "Return the segmented session timeline: ordered steps with their start time (atMs, ms since recording start), duration, app, urls, titles, commands, clipboard counts and markers. Call this first.",
+ parameters: { type: "object", properties: {}, additionalProperties: false },
+ handler: () => {
+ progress("Reading the timeline…");
+ const bundle = readBundle(sessionDir);
+ if (!bundle) {
+ return "No timeline is available yet (bundle.json missing). Use get_events to read the raw event stream instead.";
+ }
+ const view = {
+ durationMs: bundle.session.durationMs,
+ platform: bundle.session.platform,
+ stats: bundle.stats,
+ steps: bundle.steps.map((s) => ({
+ index: s.index,
+ atMs: rel(s.startMs),
+ durationMs: s.durationMs,
+ boundary: s.boundary,
+ app: s.app,
+ titles: s.titles,
+ hosts: s.hosts,
+ urls: s.urls,
+ commands: s.commands,
+ clipboardCount: s.clipboardCount,
+ markers: s.markers,
+ frameCount: s.frames.length,
+ summary: s.summary,
+ })),
+ };
+ return JSON.stringify(view, null, 2);
+ },
+ };
+
+ const getEvents: Tool = {
+ name: "get_events",
+ description:
+ "Return the raw event stream (with clipboard text, full titles, full URLs, commands). Optionally filter by `types` (e.g. [\"browser.url\",\"clipboard.change\"]) and/or a time window `fromMs`/`toMs` (atMs, ms since recording start). Defaults to the meaningful events across the whole session.",
+ parameters: {
+ type: "object",
+ properties: {
+ types: { type: "array", items: { type: "string" }, description: "Event types to include." },
+ fromMs: { type: "number", description: "Window start (atMs)." },
+ toMs: { type: "number", description: "Window end (atMs)." },
+ },
+ additionalProperties: false,
+ },
+ handler: (raw) => {
+ const args = (raw ?? {}) as { types?: string[]; fromMs?: number; toMs?: number };
+ const wanted = args.types && args.types.length ? new Set(args.types) : null;
+ const from = typeof args.fromMs === "number" ? args.fromMs : -Infinity;
+ const to = typeof args.toMs === "number" ? args.toMs : Infinity;
+ progress(
+ `Reading events ${args.fromMs != null ? sec(args.fromMs) + "s" : "start"}–${
+ args.toMs != null ? sec(args.toMs) + "s" : "end"
+ }…`,
+ );
+ let events: RecEvent[];
+ try {
+ events = readEvents(path.join(sessionDir, "events.jsonl"));
+ } catch (err) {
+ return `Could not read events: ${err instanceof Error ? err.message : String(err)}`;
+ }
+ const rows = events
+ .filter((e) => (wanted ? wanted.has(e.type) : MEANINGFUL_EVENT_TYPES.has(e.type)))
+ .map((e) => ({ e, atMs: rel(e.epoch) }))
+ .filter(({ atMs }) => atMs >= from && atMs <= to)
+ .map(({ e, atMs }) => {
+ const payload: Record<string, unknown> = {};
+ for (const [k, v] of Object.entries(e.payload)) payload[k] = truncate(v);
+ return { seq: e.seq, atMs, type: e.type, source: e.source, ...payload };
+ });
+ return JSON.stringify({ count: rows.length, events: rows }, null, 2);
+ },
+ };
+
+ const listFrames: Tool = {
+ name: "list_frames",
+ description:
+ "List the screen frames already available for this session (file, atMs, source, reason). Returns an empty list when no video was recorded. Use get_frames to actually view frames.",
+ parameters: { type: "object", properties: {}, additionalProperties: false },
+ handler: () => {
+ progress("Listing available frames…");
+ if (!extractor) return JSON.stringify({ hasVideo: false, frames: [] });
+ const frames = extractor.manifest.map((f) => ({
+ file: f.file,
+ atMs: rel(f.tMs),
+ source: f.source,
+ reason: f.reason,
+ }));
+ return JSON.stringify({ hasVideo: true, frames }, null, 2);
+ },
+ };
+
+ const getFrames: Tool = {
+ name: "get_frames",
+ description:
+ "Sample and VIEW screen frames within a time window [fromMs, toMs] (atMs, ms since recording start). Densely re-samples the window if needed (optional `fps`, default 6) and returns the frames inline as images so you can see the screen. Optional `crop` {x,y,w,h} zooms a region. Use only where events are ambiguous.",
+ parameters: {
+ type: "object",
+ properties: {
+ fromMs: { type: "number", description: "Window start (atMs)." },
+ toMs: { type: "number", description: "Window end (atMs)." },
+ fps: { type: "number", description: "Sampling density (frames/sec). Default 6." },
+ crop: {
+ type: "object",
+ properties: {
+ x: { type: "number" },
+ y: { type: "number" },
+ w: { type: "number" },
+ h: { type: "number" },
+ },
+ required: ["x", "y", "w", "h"],
+ additionalProperties: false,
+ },
+ reason: { type: "string", description: "Why you need a closer look." },
+ },
+ required: ["fromMs", "toMs"],
+ additionalProperties: false,
+ },
+ handler: async (raw) => {
+ const args = (raw ?? {}) as {
+ fromMs?: number;
+ toMs?: number;
+ fps?: number;
+ crop?: { x: number; y: number; w: number; h: number };
+ reason?: string;
+ };
+ if (!extractor) {
+ return { textResultForLlm: "No video was recorded for this session; no frames are available.", resultType: "failure" };
+ }
+ const fromMs = typeof args.fromMs === "number" ? args.fromMs : 0;
+ const toMs = typeof args.toMs === "number" ? Math.max(fromMs, args.toMs) : fromMs + 3000;
+ progress(`Looking at ${sec(fromMs)}–${sec(toMs)}s…`);
+ try {
+ await extractor.extractWindow({
+ startMs: abs(fromMs),
+ endMs: abs(toMs),
+ fps: args.fps,
+ crop: args.crop,
+ reason: args.reason ?? "describer:get_frames",
+ });
+ } catch (err) {
+ log.warn("get_frames extract failed:", err instanceof Error ? err.message : err);
+ }
+
+ const inWindow = extractor.manifest.filter((f) => {
+ const atMs = rel(f.tMs);
+ return atMs >= fromMs && atMs <= toMs;
+ });
+ if (inWindow.length === 0) {
+ return { textResultForLlm: `No frames could be extracted for ${sec(fromMs)}–${sec(toMs)}s.`, resultType: "success" };
+ }
+ // Evenly sample which frames to attach as images (bounded cost).
+ const picks: typeof inWindow = [];
+ if (inWindow.length <= MAX_IMAGES_PER_CALL) picks.push(...inWindow);
+ else {
+ const stride = (inWindow.length - 1) / (MAX_IMAGES_PER_CALL - 1);
+ for (let i = 0; i < MAX_IMAGES_PER_CALL; i++) picks.push(inWindow[Math.round(i * stride)]);
+ }
+
+ const binaryResultsForLlm: { data: string; mimeType: string; type: "image"; description?: string }[] = [];
+ for (const f of picks) {
+ const file = path.join(sessionDir, "frames", f.file);
+ if (!existsSync(file)) continue;
+ try {
+ binaryResultsForLlm.push({
+ data: readFileSync(file).toString("base64"),
+ mimeType: "image/jpeg",
+ type: "image",
+ description: `${sec(rel(f.tMs))}s (${f.source}) ${f.file}`,
+ });
+ } catch (err) {
+ log.warn("frame read failed:", err instanceof Error ? err.message : err);
+ }
+ }
+ const listing = inWindow.map((f) => `- ${sec(rel(f.tMs))}s ${f.file} (${f.source})`).join("\n");
+ progress(`Attached ${binaryResultsForLlm.length} frame image(s) from ${sec(fromMs)}–${sec(toMs)}s.`);
+ return {
+ textResultForLlm:
+ `Frames in ${sec(fromMs)}–${sec(toMs)}s (${inWindow.length} total, ${binaryResultsForLlm.length} shown as images):\n${listing}`,
+ binaryResultsForLlm,
+ resultType: "success",
+ };
+ },
+ };
+
+ const submitAnalysis: Tool = {
+ name: "submit_analysis",
+ description:
+ "Submit your final structured analysis: the overall intent and the ordered list of steps. Call this exactly once when confident (and again on later turns after incorporating feedback).",
+ parameters: {
+ type: "object",
+ properties: {
+ title: {
+ type: "string",
+ description:
+ 'A short 2–5 word label for this task in Title Case, no trailing period, e.g. "Research habit articles" or "Extract invoice data". Used as the session name in lists.',
+ },
+ intent: { type: "string", description: "One sentence naming the user's overall goal." },
+ intentConfidence: { type: "string", enum: ["high", "medium", "low"] },
+ intentRationale: { type: "string", description: "1–2 sentences of evidence for the intent." },
+ steps: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ id: { type: "string", description: "Stable short id, e.g. \"s1\"." },
+ title: { type: "string" },
+ detail: { type: "string" },
+ startMs: { type: "number", description: "Step start (atMs)." },
+ endMs: { type: "number", description: "Step end (atMs)." },
+ apps: { type: "array", items: { type: "string" } },
+ evidence: { type: "array", items: { type: "string" } },
+ confidence: { type: "string", enum: ["high", "medium", "low"] },
+ },
+ required: ["id", "title", "detail"],
+ additionalProperties: false,
+ },
+ },
+ },
+ required: ["title", "intent", "steps"],
+ additionalProperties: false,
+ },
+ handler: (raw) => {
+ const parsed = AnalysisSubmissionSchema.safeParse(raw);
+ if (!parsed.success) {
+ return {
+ textResultForLlm:
+ "submit_analysis rejected — the payload did not match the schema. Fix these and call again:\n" +
+ parsed.error.issues.map((i) => `- ${i.path.join(".") || "(root)"}: ${i.message}`).join("\n"),
+ resultType: "failure",
+ };
+ }
+ progress("Received analysis draft.");
+ onSubmit(parsed.data);
+ return "Analysis recorded. You may stop now.";
+ },
+ };
+
+ return [getTimeline, getEvents, listFrames, getFrames, submitAnalysis];
+}
diff --git a/electron/ipc.ts b/electron/ipc.ts
index 1a60c6f..af75f29 100644
--- a/electron/ipc.ts
+++ b/electron/ipc.ts
@@ -1,14 +1,24 @@
import { ipcMain } from "electron";
+import path from "node:path";
import { type CaptureConfig, type CaptureLevel, levelForConfig } from "../common/config";
-import type { CaptureState } from "../common/ipc";
+import type { AnalysisEditInput, AnalysisFeedbackInput, AnalyzeResult, CaptureState } from "../common/ipc";
import { IPC } from "../common/ipc";
+import { Describer, loadPersistedAnalysis } from "./describer/describer";
import { runDoctor } from "./doctor";
+import { createLogger } from "./logger";
import type { RecorderController } from "./recorder/controller";
+import { listSessions } from "./sessions";
import type { SettingsStore } from "./settings";
-/** Wire the renderer-facing invoke channels to the recorder, doctor + settings. */
-export function registerIpc(recorder: RecorderController, settings: SettingsStore): void {
+const log = createLogger("IPC");
+
+/** Wire the renderer-facing invoke channels to the recorder, describer, doctor + settings. */
+export function registerIpc(
+ recorder: RecorderController,
+ settings: SettingsStore,
+ describer: Describer,
+): void {
const captureState = (): CaptureState => {
const config = settings.resolve();
return { level: levelForConfig(config), config };
@@ -28,4 +38,80 @@ export function registerIpc(recorder: RecorderController, settings: SettingsStor
settings.setConfig(config);
return captureState();
});
+
+ const resolveSessionId = (sessionId?: string): string | null => {
+ if (sessionId) return sessionId;
+ const dir = recorder.lastSessionDir();
+ return dir ? path.basename(dir) : null;
+ };
+
+ ipcMain.handle(IPC.analyze, async (_event, sessionId?: string): Promise<AnalyzeResult> => {
+ const id = resolveSessionId(sessionId);
+ if (!id) return { ok: false, error: "No completed session to analyze yet." };
+ try {
+ const analysis = await describer.analyze(id);
+ return { ok: true, analysis };
+ } catch (err) {
+ const error = err instanceof Error ? err.message : String(err);
+ log.warn("analyze failed:", error);
+ return { ok: false, error };
+ }
+ });
+
+ ipcMain.handle(
+ IPC.analyzeFeedback,
+ async (_event, input: AnalysisFeedbackInput): Promise<AnalyzeResult> => {
+ if (!input?.sessionId) return { ok: false, error: "Missing sessionId." };
+ try {
+ const analysis = await describer.feedback(input.sessionId, {
+ overall: input.overall,
+ steps: input.steps ?? [],
+ });
+ return { ok: true, analysis };
+ } catch (err) {
+ const error = err instanceof Error ? err.message : String(err);
+ log.warn("feedback failed:", error);
+ return { ok: false, error };
+ }
+ },
+ );
+
+ ipcMain.handle(IPC.getAnalysis, (_event, sessionId: string) =>
+ sessionId ? loadPersistedAnalysis(sessionId) : null,
+ );
+
+ ipcMain.handle(IPC.approveAnalysis, async (_event, sessionId: string): Promise<AnalyzeResult> => {
+ if (!sessionId) return { ok: false, error: "Missing sessionId." };
+ try {
+ const analysis = await describer.approve(sessionId);
+ return { ok: true, analysis };
+ } catch (err) {
+ const error = err instanceof Error ? err.message : String(err);
+ log.warn("approve failed:", error);
+ return { ok: false, error };
+ }
+ });
+
+ ipcMain.handle(IPC.updateAnalysis, async (_event, input: AnalysisEditInput): Promise<AnalyzeResult> => {
+ if (!input?.sessionId) return { ok: false, error: "Missing sessionId." };
+ try {
+ const analysis = await describer.edit(input.sessionId, {
+ title: input.title,
+ intent: input.intent,
+ });
+ return { ok: true, analysis };
+ } catch (err) {
+ const error = err instanceof Error ? err.message : String(err);
+ log.warn("update failed:", error);
+ return { ok: false, error };
+ }
+ });
+
+ ipcMain.handle(IPC.cancelAnalysis, async (_event, sessionId: string) => {
+ if (sessionId) await describer.cancel(sessionId);
+ return { ok: true };
+ });
+
+ ipcMain.handle(IPC.listSessions, () => listSessions());
}
+
diff --git a/electron/preload.cjs b/electron/preload.cjs
index 5c5a798..b2a0674 100644
--- a/electron/preload.cjs
+++ b/electron/preload.cjs
@@ -12,6 +12,16 @@ const IPC = {
getCapture: "capture:get",
setLevel: "capture:set-level",
setConfig: "capture:set-config",
+ analyze: "analyze:start",
+ analyzeFeedback: "analyze:feedback",
+ getAnalysis: "analyze:get",
+ approveAnalysis: "analyze:approve",
+ updateAnalysis: "analyze:update",
+ cancelAnalysis: "analyze:cancel",
+ analyzeProgress: "analyze:progress",
+ listSessions: "sessions:list",
+ openLibrary: "ui:open-library",
+ closeLibrary: "ui:close-library",
};
contextBridge.exposeInMainWorld("skillRecorder", {
@@ -28,4 +38,18 @@ contextBridge.exposeInMainWorld("skillRecorder", {
ipcRenderer.on(IPC.statusChanged, listener);
return () => ipcRenderer.removeListener(IPC.statusChanged, listener);
},
+ analyze: (sessionId) => ipcRenderer.invoke(IPC.analyze, sessionId),
+ analyzeFeedback: (input) => ipcRenderer.invoke(IPC.analyzeFeedback, input),
+ getAnalysis: (sessionId) => ipcRenderer.invoke(IPC.getAnalysis, sessionId),
+ approveAnalysis: (sessionId) => ipcRenderer.invoke(IPC.approveAnalysis, sessionId),
+ updateAnalysis: (input) => ipcRenderer.invoke(IPC.updateAnalysis, input),
+ cancelAnalysis: (sessionId) => ipcRenderer.invoke(IPC.cancelAnalysis, sessionId),
+ onAnalyzeProgress: (cb) => {
+ const listener = (_event, progress) => cb(progress);
+ ipcRenderer.on(IPC.analyzeProgress, listener);
+ return () => ipcRenderer.removeListener(IPC.analyzeProgress, listener);
+ },
+ listSessions: () => ipcRenderer.invoke(IPC.listSessions),
+ openLibrary: () => ipcRenderer.invoke(IPC.openLibrary),
+ closeLibrary: () => ipcRenderer.invoke(IPC.closeLibrary),
});
diff --git a/electron/sessions.ts b/electron/sessions.ts
new file mode 100644
index 0000000..6866295
--- /dev/null
+++ b/electron/sessions.ts
@@ -0,0 +1,64 @@
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import path from "node:path";
+
+import type { SessionSummary } from "../common/ipc";
+import type { SessionMeta } from "../common/types";
+import { loadPersistedAnalysis } from "./describer/describer";
+import { createLogger } from "./logger";
+import { sessionsRoot } from "./recorder/session-store";
+
+const log = createLogger("Sessions");
+
+/** Enumerate saved recordings for the sessions library, newest first. */
+export function listSessions(): SessionSummary[] {
+ const root = sessionsRoot();
+ if (!existsSync(root)) return [];
+
+ const out: SessionSummary[] = [];
+ let names: string[];
+ try {
+ names = readdirSync(root);
+ } catch (err) {
+ log.warn("failed to read sessions dir:", err instanceof Error ? err.message : err);
+ return [];
+ }
+
+ for (const name of names) {
+ const dir = path.join(root, name);
+ try {
+ if (!statSync(dir).isDirectory()) continue;
+ } catch {
+ continue;
+ }
+ let meta: SessionMeta | null = null;
+ try {
+ meta = JSON.parse(readFileSync(path.join(dir, "session.json"), "utf8")) as SessionMeta;
+ } catch {
+ continue; // not a valid session dir
+ }
+ if (!meta?.id) continue;
+
+ const analysis = loadPersistedAnalysis(name);
+ out.push({
+ id: meta.id,
+ startedAt: meta.startedAt ?? null,
+ stoppedAt: meta.stoppedAt ?? null,
+ durationMs: meta.startedAt && meta.stoppedAt ? meta.stoppedAt - meta.startedAt : null,
+ processed: existsSync(path.join(dir, "bundle.json")),
+ hasVideo: existsSync(path.join(dir, "video.json")),
+ analysis: analysis
+ ? {
+ revision: analysis.revision,
+ title: analysis.title ?? "",
+ intent: analysis.intent,
+ intentConfidence: analysis.intentConfidence,
+ stepCount: analysis.steps.length,
+ approved: analysis.approved ?? false,
+ }
+ : null,
+ });
+ }
+
+ out.sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));
+ return out;
+}