Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.
TypeScript agent-skillsai-agentsautomationcopilotcopilot-clicopilot-coworkcopilot-studioelectronmicrosoft-scoutscreen-recording
Add Copilot describer: reconstruct intent + steps from a recording
adilei8c73408bd8b7
diff --git a/common/analysis.ts b/common/analysis.tsnew file mode 100644index 0000000..b4e46ae--- /dev/null+++ b/common/analysis.ts@@ -0,0 +1,113 @@+import { z } from "zod";++/**+ * The structured result of the Copilot describer: a high-level **intent** plus an+ * ordered list of **actions/steps**, produced by the multi-turn agent and revised+ * from the user's natural-language feedback. This is the stable contract shared by+ * the agent's `submit_analysis` tool, the IPC surface, and on-disk persistence+ * (`analysis.json`). Kept separate from the deterministic `SessionBundle` (which is+ * the agent's *input*); this is the agent's *output*.+ */++export const Confidence = z.enum(["high", "medium", "low"]);+export type Confidence = z.infer<typeof Confidence>;++/** One reconstructed action the user took, in order. */+export const AnalysisStepSchema = z.object({+ /** Stable id assigned by the agent (e.g. "s1"); used to target feedback. */+ id: z.string(),+ /** Short imperative label, e.g. "Search Google for 'interesting articles'". */+ title: z.string(),+ /** 1–3 sentences of what happened and why it matters. */+ detail: z.string(),+ /** Wall-clock epoch span (ms) this step covers, when known. */+ startMs: z.number().optional(),+ endMs: z.number().optional(),+ /** Apps involved (e.g. ["Microsoft Edge"]). */+ apps: z.array(z.string()).default([]),+ /** Free-text evidence refs the agent leaned on (events, urls, frame files). */+ evidence: z.array(z.string()).default([]),+ confidence: Confidence.default("medium"),+});+export type AnalysisStep = z.infer<typeof AnalysisStepSchema>;++/**+ * The payload the agent submits via the `submit_analysis` tool. The engine wraps+ * this into a full {@link Analysis} (adding sessionId/revision/timestamps/feedback).+ */+export const AnalysisSubmissionSchema = z.object({+ /** Short 2–5 word label for lists/menus, e.g. "Research habit articles". */+ title: z.string().default(""),+ /** The user's overall goal, e.g. "Submit an expense report". */+ intent: z.string(),+ intentConfidence: Confidence.default("medium"),+ /** Why the agent believes this is the intent (evidence-grounded). */+ intentRationale: z.string().default(""),+ steps: z.array(AnalysisStepSchema).default([]),+});+export type AnalysisSubmission = z.infer<typeof AnalysisSubmissionSchema>;++/** One round of user feedback that was fed back into the session. */+export const FeedbackEntrySchema = z.object({+ /** The revision number this feedback produced. */+ revision: z.number().int(),+ at: z.number(),+ /** Natural-language feedback on the overall intent, if any. */+ overall: z.string().optional(),+ /** Per-step natural-language notes. */+ steps: z+ .array(z.object({ stepId: z.string(), note: z.string() }))+ .default([]),+});+export type FeedbackEntry = z.infer<typeof FeedbackEntrySchema>;++/** The full, persisted analysis for a session. */+export const AnalysisSchema = z.object({+ version: z.literal(1),+ sessionId: z.string(),+ /** Bumped on every (re)analysis; revision 1 is the first pass. */+ revision: z.number().int(),+ createdAt: z.number(),+ /** Short 2–5 word label for lists/menus. Empty on analyses saved before titles existed. */+ title: z.string().default(""),+ intent: z.string(),+ intentConfidence: Confidence,+ intentRationale: z.string(),+ steps: z.array(AnalysisStepSchema),+ /** Chronological log of feedback rounds applied so far. */+ feedbackLog: z.array(FeedbackEntrySchema).default([]),+ /** User has accepted this analysis as correct — the artifact a Skill is built from. */+ approved: z.boolean().default(false),+ /** When the user approved it (epoch ms). */+ approvedAt: z.number().optional(),+});+export type Analysis = z.infer<typeof AnalysisSchema>;++/** Feedback the UI sends for a re-analysis round. */+export interface AnalysisFeedback {+ /** NL feedback on the overall intent. */+ overall?: string;+ /** Per-step NL notes (only steps the user commented on). */+ steps: { stepId: string; note: string }[];+}++/** Build a full Analysis from an agent submission + engine-managed fields. */+export function toAnalysis(+ sessionId: string,+ revision: number,+ submission: AnalysisSubmission,+ feedbackLog: FeedbackEntry[],+): Analysis {+ return AnalysisSchema.parse({+ version: 1,+ sessionId,+ revision,+ createdAt: Date.now(),+ title: submission.title,+ intent: submission.intent,+ intentConfidence: submission.intentConfidence,+ intentRationale: submission.intentRationale,+ steps: submission.steps,+ feedbackLog,+ });+}diff --git a/common/ipc.ts b/common/ipc.tsindex 30b31d7..b075d39 100644--- a/common/ipc.ts+++ b/common/ipc.ts@@ -1,11 +1,69 @@+import type { Analysis, AnalysisFeedback, Confidence } from "./analysis";import type { CaptureConfig, CaptureLevel } from "./config";import type { RecorderState } from "./types";+/** The last completed session — the one that can be analyzed. */+export interface LastSession {+ id: string;+ /** True once post-processing (bundle/description/frames) has finished. */+ processed: boolean;+}++/** A saved recording as shown in the sessions library. */+export interface SessionSummary {+ id: string;+ startedAt: number | null;+ stoppedAt: number | null;+ durationMs: number | null;+ /** True once post-processing produced a bundle. */+ processed: boolean;+ hasVideo: boolean;+ /** Present once the describer has produced an analysis for this session. */+ analysis: {+ revision: number;+ title: string;+ intent: string;+ intentConfidence: Confidence;+ stepCount: number;+ approved: boolean;+ } | null;+}+export interface RecorderStatus {state: RecorderState;sessionId: string | null;startedAt: number | null;eventCount: number;+ /** Set after a recording stops; drives the "Analyze" affordance. */+ lastSession: LastSession | null;+}++/** Streamed to the renderer while the describer agent works. */+export interface AnalyzeProgress {+ sessionId: string;+ phase: "start" | "working" | "drafting" | "done" | "error";+ message: string;+}++/** Result of an analyze / feedback round. */+export interface AnalyzeResult {+ ok: boolean;+ analysis?: Analysis;+ error?: string;+}++/** Feedback payload sent from the renderer for a re-analysis round. */+export interface AnalysisFeedbackInput extends AnalysisFeedback {+ sessionId: string;+}++/** A direct text edit to the intent/title, applied without re-running the agent. */+export interface AnalysisEditInput {+ sessionId: string;+ /** New short label; empty string clears it (list falls back to the intent). */+ title?: string;+ /** New one-sentence goal; blank/whitespace is ignored (intent can't be emptied). */+ intent?: string;}export interface StartResult {@@ -63,6 +121,16 @@ export const IPC = {getCapture: "capture:get",setLevel: "capture:set-level",setConfig: "capture:set-config",+ analyze: "analyze:start",+ analyzeFeedback: "analyze:feedback",+ getAnalysis: "analyze:get",+ approveAnalysis: "analyze:approve",+ updateAnalysis: "analyze:update",+ cancelAnalysis: "analyze:cancel",+ analyzeProgress: "analyze:progress",+ listSessions: "sessions:list",+ openLibrary: "ui:open-library",+ closeLibrary: "ui:close-library",} as const;/** Shape exposed on `window.skillRecorder` by the preload bridge. */@@ -76,4 +144,23 @@ export interface SkillRecorderApi {setLevel(level: Exclude<CaptureLevel, "custom">): Promise<CaptureState>;setConfig(config: CaptureConfig): Promise<CaptureState>;onStatusChanged(cb: (status: RecorderStatus) => void): () => void;+ /** Run the Copilot describer on a session (defaults to the last completed one). */+ analyze(sessionId?: string): Promise<AnalyzeResult>;+ /** Send NL feedback and re-analyze in the same multi-turn session. */+ analyzeFeedback(input: AnalysisFeedbackInput): Promise<AnalyzeResult>;+ /** Load the persisted analysis for a session, if any. */+ getAnalysis(sessionId: string): Promise<Analysis | null>;+ /** Mark an analysis as accepted/correct — the artifact a Skill is built from. */+ approveAnalysis(sessionId: string): Promise<AnalyzeResult>;+ /** Edit the title/intent text directly (no re-analysis). */+ updateAnalysis(input: AnalysisEditInput): Promise<AnalyzeResult>;+ /** Abort an in-flight analysis. */+ cancelAnalysis(sessionId: string): Promise<{ ok: boolean }>;+ onAnalyzeProgress(cb: (progress: AnalyzeProgress) => void): () => void;+ /** All saved recordings, newest first, for the sessions library. */+ listSessions(): Promise<SessionSummary[]>;+ /** Open (and focus) the Sessions library window, docked to the recorder. */+ openLibrary(): Promise<void>;+ /** Close the Sessions library window from within it. */+ closeLibrary(): Promise<void>;}diff --git a/electron/describer/describer.ts b/electron/describer/describer.tsnew file mode 100644index 0000000..4267379--- /dev/null+++ b/electron/describer/describer.ts@@ -0,0 +1,363 @@+import { existsSync, readFileSync, writeFileSync } from "node:fs";+import path from "node:path";++import { approveAll, CopilotClient, type CopilotSession } from "@github/copilot-sdk";++import {+ AnalysisSchema,+ toAnalysis,+ type Analysis,+ type AnalysisFeedback,+ type AnalysisSubmission,+} from "../../common/analysis";+import type { AnalyzeProgress } from "../../common/ipc";+import type { SessionMeta } from "../../common/types";+import { FrameExtractor } from "../frames/extractor";+import { createLogger } from "../logger";+import { sessionsRoot } from "../recorder/session-store";+import { DESCRIBER_INSTRUCTIONS } from "./instructions";+import { createDescriberTools } from "./tools";++const log = createLogger("Describer");++/** How long a single agent turn may run before we give up (multi-tool loop). */+const TURN_TIMEOUT_MS = 180_000;++const KICKOFF_PROMPT =+ "Reconstruct what the user did in this recording. Start with get_timeline, then read events " ++ "where anything is unclear, and look at frames only where the events are ambiguous. When " ++ "confident, call submit_analysis with the overall intent and the ordered list of steps.";++const NUDGE_PROMPT =+ "Please call submit_analysis now with your best analysis of the overall intent and the ordered steps.";++const msg = (err: unknown) => (err instanceof Error ? err.message : String(err));++interface VideoMeta {+ file: string;+ startEpoch: number;+ durationMs: number;+}++/** A live, resumable analysis session for one recording. */+interface LiveSession {+ sessionId: string;+ sessionDir: string;+ copilot: CopilotSession;+ revision: number;+ feedbackLog: Analysis["feedbackLog"];+ /** Mutable capture slot the `submit_analysis` tool writes into. */+ holder: { submission: AnalysisSubmission | undefined };+}++function readJson<T>(file: string): T | null {+ if (!existsSync(file)) return null;+ try {+ return JSON.parse(readFileSync(file, "utf8")) as T;+ } catch {+ return null;+ }+}++/** Load a persisted analysis from a session dir (validated), or null. */+export function loadPersistedAnalysis(sessionId: string): Analysis | null {+ const file = path.join(sessionsRoot(), sessionId, "analysis.json");+ const raw = readJson<unknown>(file);+ if (!raw) return null;+ const parsed = AnalysisSchema.safeParse(raw);+ return parsed.success ? parsed.data : null;+}++/**+ * Drives the multi-turn GitHub Copilot CLI agent that turns a recording's captured+ * signals into an intent + ordered steps, and revises them from NL feedback. Owns+ * a single shared {@link CopilotClient}; keeps one live session per recording so the+ * feedback loop stays in the same conversation. Streams progress out via a callback.+ * The deterministic baseline `description.md` remains the fallback when Copilot is+ * unavailable (callers surface the thrown error).+ */+export class Describer {+ private client: CopilotClient | null = null;+ private clientStart: Promise<CopilotClient> | null = null;+ private model: string | undefined;+ private readonly live = new Map<string, LiveSession>();+ private readonly active = new Set<string>();++ constructor(private readonly emitProgress: (p: AnalyzeProgress) => void) {}++ /** First pass: reconstruct the session from scratch. */+ async analyze(sessionId: string): Promise<Analysis> {+ if (this.active.has(sessionId)) throw new Error("An analysis is already running for this session.");+ this.active.add(sessionId);+ try {+ this.emit(sessionId, "start", "Starting analysis…");+ await this.disposeLive(sessionId); // fresh conversation each explicit analyze+ const live = await this.createLive(sessionId);+ return await this.runTurn(live, KICKOFF_PROMPT);+ } finally {+ this.active.delete(sessionId);+ }+ }++ /** Later pass: fold in NL feedback and revise holistically. */+ async feedback(sessionId: string, fb: AnalysisFeedback): Promise<Analysis> {+ if (this.active.has(sessionId)) throw new Error("An analysis is already running for this session.");+ this.active.add(sessionId);+ try {+ this.emit(sessionId, "start", "Re-analyzing with your feedback…");+ const live = this.live.get(sessionId) ?? (await this.createLive(sessionId));+ const prior = loadPersistedAnalysis(sessionId);+ live.feedbackLog = [+ ...live.feedbackLog,+ { revision: live.revision + 1, at: Date.now(), overall: fb.overall, steps: fb.steps },+ ];+ return await this.runTurn(live, renderFeedbackPrompt(fb, prior));+ } finally {+ this.active.delete(sessionId);+ }+ }++ /** Accept an analysis as correct. Persists `approved` so a Skill can be built from it. */+ async approve(sessionId: string): Promise<Analysis> {+ const prior = loadPersistedAnalysis(sessionId);+ if (!prior) throw new Error("There is no analysis to approve yet.");+ const approved: Analysis = { ...prior, approved: true, approvedAt: Date.now() };+ const dir = path.join(sessionsRoot(), sessionId);+ this.persist(dir, approved, this.live.get(sessionId)?.copilot.sessionId);+ return approved;+ }++ /**+ * Apply a direct text edit to the title and/or intent — a user correction, NOT a+ * re-analysis. Steps and evidence are untouched; the agent is not invoked. Blank+ * intent is ignored (a session always keeps a goal); the title may be cleared.+ */+ async edit(sessionId: string, patch: { title?: string; intent?: string }): Promise<Analysis> {+ const prior = loadPersistedAnalysis(sessionId);+ if (!prior) throw new Error("There is no analysis to edit yet.");+ const next: Analysis = { ...prior };+ if (patch.title !== undefined) next.title = patch.title.trim();+ if (patch.intent !== undefined && patch.intent.trim()) next.intent = patch.intent.trim();+ const dir = path.join(sessionsRoot(), sessionId);+ this.persist(dir, next, this.live.get(sessionId)?.copilot.sessionId);+ return next;+ }++ /** Abort an in-flight run for a session. */+ async cancel(sessionId: string): Promise<void> {+ const live = this.live.get(sessionId);+ if (live) await live.copilot.abort().catch(() => undefined);+ }++ /** Tear down the client + all live sessions (called on app quit). */+ async dispose(): Promise<void> {+ for (const [id] of this.live) await this.disposeLive(id);+ if (this.client) await this.client.stop().catch(() => undefined);+ this.client = null;+ this.clientStart = null;+ }++ // --- internals -----------------------------------------------------------++ private emit(sessionId: string, phase: AnalyzeProgress["phase"], message: string): void {+ this.emitProgress({ sessionId, phase, message });+ }++ private async ensureClient(): Promise<CopilotClient> {+ if (this.client) return this.client;+ if (this.clientStart) return this.clientStart;+ this.clientStart = (async () => {+ const client = new CopilotClient();+ await client.start();+ const auth = await client.getAuthStatus();+ if (!auth.isAuthenticated) {+ await client.stop().catch(() => undefined);+ throw new Error(+ "GitHub Copilot CLI is not signed in. Open a terminal, run `copilot`, sign in, then try again.",+ );+ }+ this.model = await this.pickVisionModel(client);+ log.info("Copilot ready", auth.login ? `as ${auth.login}` : "", this.model ? `· model ${this.model}` : "");+ this.client = client;+ return client;+ })();+ try {+ return await this.clientStart;+ } catch (err) {+ this.clientStart = null;+ throw err;+ }+ }++ /** Prefer a vision-capable, enabled model (frames need vision). */+ private async pickVisionModel(client: CopilotClient): Promise<string | undefined> {+ const override = process.env.SKILL_RECORDER_MODEL;+ if (override) return override;+ try {+ const models = await client.listModels();+ const vision = models.filter(+ (m) => m.capabilities?.supports?.vision && m.policy?.state !== "disabled",+ );+ return vision[0]?.id;+ } catch (err) {+ log.warn("listModels failed; using CLI default model:", msg(err));+ return undefined;+ }+ }++ private async createLive(sessionId: string): Promise<LiveSession> {+ const sessionDir = path.join(sessionsRoot(), sessionId);+ const meta = readJson<SessionMeta>(path.join(sessionDir, "session.json"));+ if (!meta) throw new Error(`Session ${sessionId} not found or has no session.json.`);++ const extractor = buildExtractor(sessionDir);+ const holder: LiveSession["holder"] = { submission: undefined };++ const tools = createDescriberTools({+ sessionDir,+ startedAt: meta.startedAt,+ extractor,+ onProgress: (m) => this.emit(sessionId, "working", m),+ onSubmit: (s) => {+ holder.submission = s;+ },+ });++ const client = await this.ensureClient();+ const config = {+ systemMessage: { mode: "append" as const, content: DESCRIBER_INSTRUCTIONS },+ tools,+ onPermissionRequest: approveAll,+ workingDirectory: sessionDir,+ enableHostGitOperations: false,+ infiniteSessions: { enabled: false },+ ...(this.model ? { model: this.model } : {}),+ };+ // Restrict to our custom tools where supported; fall back if the runtime+ // rejects an allowlist of custom names.+ let copilot: CopilotSession;+ try {+ copilot = await client.createSession({ ...config, availableTools: tools.map((t) => t.name) });+ } catch (err) {+ log.warn("createSession with tool allowlist failed; retrying without it:", msg(err));+ copilot = await client.createSession(config);+ }++ const prior = loadPersistedAnalysis(sessionId);+ const live: LiveSession = {+ sessionId,+ sessionDir,+ copilot,+ revision: prior?.revision ?? 0,+ feedbackLog: prior?.feedbackLog ?? [],+ holder,+ };+ this.live.set(sessionId, live);+ return live;+ }++ private async disposeLive(sessionId: string): Promise<void> {+ const live = this.live.get(sessionId);+ if (!live) return;+ this.live.delete(sessionId);+ await live.copilot.disconnect().catch(() => undefined);+ }++ private async runTurn(live: LiveSession, prompt: string): Promise<Analysis> {+ live.holder.submission = undefined;+ this.emit(live.sessionId, "working", "Thinking…");+ try {+ await live.copilot.sendAndWait(prompt, TURN_TIMEOUT_MS);+ } catch (err) {+ throw new Error(`Analysis run failed: ${msg(err)}`);+ }+ if (!live.holder.submission) {+ this.emit(live.sessionId, "working", "Asking the agent to finalize its analysis…");+ await live.copilot.sendAndWait(NUDGE_PROMPT, TURN_TIMEOUT_MS).catch(() => undefined);+ }+ const submission = live.holder.submission;+ if (!submission) throw new Error("The agent finished without submitting an analysis.");++ live.revision += 1;+ this.emit(live.sessionId, "drafting", "Finalizing analysis…");+ const analysis = toAnalysis(live.sessionId, live.revision, submission, [...live.feedbackLog]);+ this.persist(live.sessionDir, analysis, live.copilot.sessionId);+ this.emit(live.sessionId, "done", `Analysis ready (revision ${analysis.revision}).`);+ return analysis;+ }++ private persist(sessionDir: string, analysis: Analysis, copilotSessionId?: string): void {+ try {+ writeFileSync(path.join(sessionDir, "analysis.json"), JSON.stringify(analysis, null, 2));+ writeFileSync(path.join(sessionDir, "analysis.md"), renderAnalysisMarkdown(analysis));+ if (copilotSessionId) {+ writeFileSync(+ path.join(sessionDir, "describer.json"),+ JSON.stringify({ copilotSessionId }, null, 2),+ );+ }+ } catch (err) {+ log.warn("failed to persist analysis:", msg(err));+ }+ }+}++function buildExtractor(sessionDir: string): FrameExtractor | null {+ const video = readJson<VideoMeta>(path.join(sessionDir, "video.json"));+ if (!video) return null;+ const videoPath = path.join(sessionDir, video.file);+ if (!existsSync(videoPath)) return null;+ return new FrameExtractor({+ videoPath,+ framesDir: path.join(sessionDir, "frames"),+ anchorEpochMs: video.startEpoch,+ durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,+ });+}++function renderFeedbackPrompt(fb: AnalysisFeedback, prior: Analysis | null): string {+ const lines: string[] = [+ "The user reviewed your analysis and left feedback. Revise the ENTIRE analysis accordingly and",+ "call submit_analysis again. Re-examine signals (get_events / get_frames) where the feedback",+ "points to a gap or error. Keep step ids stable for steps that don't change.",+ "",+ ];+ if (prior) {+ lines.push("Your current analysis:");+ lines.push(`- intent (${prior.intentConfidence}): ${prior.intent}`);+ for (const s of prior.steps) lines.push(`- ${s.id}: ${s.title} — ${s.detail}`);+ lines.push("");+ }+ lines.push(`Overall-intent feedback: ${fb.overall?.trim() ? fb.overall.trim() : "(none)"}`);+ if (fb.steps.length) {+ lines.push("Per-step feedback:");+ for (const s of fb.steps) {+ const title = prior?.steps.find((p) => p.id === s.stepId)?.title ?? "";+ lines.push(`- ${s.stepId}${title ? ` (${title})` : ""}: ${s.note}`);+ }+ }+ return lines.join("\n");+}++/** Human-readable rendering of an analysis (persisted alongside analysis.json). */+export function renderAnalysisMarkdown(a: Analysis): string {+ const out: string[] = [];+ out.push(`# ${a.title?.trim() || "Session analysis"}`);+ out.push("");+ out.push(`**Intent** (${a.intentConfidence}): ${a.intent}`);+ if (a.intentRationale) out.push(`\n${a.intentRationale}`);+ out.push("");+ out.push(`## Steps`);+ a.steps.forEach((s, i) => {+ const span =+ s.startMs != null ? ` _(${(s.startMs / 1000).toFixed(1)}s${s.endMs != null ? `–${(s.endMs / 1000).toFixed(1)}s` : ""})_` : "";+ out.push(`\n### ${i + 1}. ${s.title}${span}`);+ out.push(s.detail);+ if (s.apps.length) out.push(`\n- apps: ${s.apps.join(", ")}`);+ if (s.evidence.length) out.push(`- evidence: ${s.evidence.join("; ")}`);+ out.push(`- confidence: ${s.confidence}`);+ });+ out.push("");+ out.push(`_Revision ${a.revision} · generated ${new Date(a.createdAt).toISOString()}_`);+ return out.join("\n") + "\n";+}diff --git a/electron/describer/instructions.ts b/electron/describer/instructions.tsnew file mode 100644index 0000000..25a948e--- /dev/null+++ b/electron/describer/instructions.ts@@ -0,0 +1,103 @@+/**+ * The describer **brief** — the agent's system message (appended to the SDK+ * foundation). It tells the Copilot CLI agent its job, the tools it has, the+ * method to follow, and the exact structured output it must produce. This is the+ * "skill" in the loose sense: a human-editable instruction document, NOT a+ * packaged Copilot CLI skill directory and NOT a `.github/extensions` extension.+ *+ * Time model exposed to the agent: **`atMs` = milliseconds since the recording+ * started** (0 = the moment the user hit Start). Every tool speaks this single+ * clock; the mapping to video offsets is hidden behind the frame tools.+ */+export const DESCRIBER_INSTRUCTIONS = `+# Role: Session Describer++You reconstruct what a user did during a short screen-recording session and+produce (1) their **overall intent** and (2) an **ordered list of the concrete+actions** they took. Your output becomes the raw material for building an+AI-agent "skill", so be accurate, specific, and grounded in the captured signals.++## What was captured+The recorder harvested cheap, high-signal OS events as the PRIMARY source:+- **app switches** (which application was focused),+- **window titles**,+- **browser URLs** (the pages visited),+- **clipboard changes** (copied text),+- **terminal commands** (when the shell hook was active).++A low-frame-rate **screen video** may also exist. It is OPPORTUNISTIC enrichment+— you pull frames only where the events are ambiguous. Do NOT assume you must+look at video; most steps are fully explained by events alone.++All times are **\`atMs\` = milliseconds since the recording started**.++## Your tools+- **get_timeline** — the segmented timeline: ordered steps (app / urls / titles /+ commands / clipboard counts / markers) with their \`atMs\` start + duration. Start here.+- **get_events({ types?, fromMs?, toMs? })** — the raw event stream (with clipboard+ text, full titles, full URLs, commands). Use to inspect a specific window closely.+- **list_frames** — index of screen frames already available (file + \`atMs\` + why kept).+ Empty/absent means no video was recorded.+- **get_frames({ fromMs, toMs, fps?, crop?, reason? })** — sample and **view** screen+ frames within a time window. Returns the images inline so you can actually see the+ screen. Optional \`crop\` ({x,y,w,h}) zooms a region. This is your "look closer"+ primitive — use it ONLY where events leave real ambiguity.+- **submit_analysis({ title, intent, intentConfidence, intentRationale, steps })** — your+ REQUIRED final action. Call it exactly once when confident. See the schema below.++## Method+1. **Read the timeline** (get_timeline) to get the shape of the session.+2. **Form a hypothesis** about the overall intent from apps + urls + commands.+3. **Read events** (get_events) around anything unclear — clipboard text, exact+ URLs, the sequence of title changes.+4. **Look at frames ONLY where events are silent or ambiguous** (get_frames): e.g. a+ step with a visual change but no explaining event, a clipboard copy whose purpose+ is unclear, or a terminal step with no captured command. Budget ~5 frames for a+ ~30–60s session. Cost should scale with ambiguity, not video length.+5. **Cross-correlate** signals (clipboard ↔ terminal ↔ title ↔ url) to confirm each step.+6. **Call submit_analysis** with the intent and ordered steps.++## Noise to ignore+- **The Skill Recorder app itself** (this Electron recorder, app name "Skill Recorder").+ Activating/focusing it is how the user reaches the Start and Stop buttons — it is NOT+ part of their task. In particular, the FIRST step (focusing Skill Recorder to press Start,+ usually at \`atMs\` ≈ 0) and the LAST step (returning to Skill Recorder to press Stop) are+ recorder bracketing, not user actions — do NOT emit them as steps. Also drop any mid-session+ flick back to Skill Recorder just to add a marker. The real task starts with the first+ non-recorder app the user works in.+- \`UserNotificationCenter\` / OS permission dialogs (e.g. "requesting to record the+ screen") are the recorder's own consent prompts — NOT user actions. Skip them.+- URL tracking params (\`gclid\`, \`gad_source\`, \`utm_*\`) and ad-redirect hops carry no+ intent — treat two URLs that differ only in these as the same page.+- Momentary app focus flickers (sub-second activations with no follow-up) are usually+ not real steps.++## Output schema (submit_analysis)+- **title**: a SHORT 2–5 word label for the task, in Title Case with no trailing period,+ e.g. "Research Habit Articles", "Extract Invoice Data", "Compare Pricing Plans". This is+ the session's name in lists, so make it scannable — name the task, not the apps used.+- **intent**: one sentence naming the user's goal, e.g. "Research and compare+ articles on building better habits" or "Submit an expense report".+- **intentConfidence**: "high" | "medium" | "low".+- **intentRationale**: 1–2 sentences citing the evidence for the intent.+- **steps[]**: ordered; each is:+ - **id**: stable short id you assign, "s1", "s2", …+ - **title**: short imperative label, e.g. "Search Google for 'atomic habits'".+ - **detail**: 1–3 sentences of what happened and why it matters.+ - **startMs / endMs**: the step's \`atMs\` span when known.+ - **apps[]**: apps involved (e.g. ["Microsoft Edge"]).+ - **evidence[]**: brief refs you relied on — event types, a URL, a frame file, a+ copied string. Keep them short.+ - **confidence**: "high" | "medium" | "low".++## Handling feedback+Later turns may deliver the user's natural-language feedback on your analysis+(corrections to the intent, notes like "this step is irrelevant", "you missed a+step", "not accurate"). When you receive feedback:+- Treat it as authoritative. Re-examine the relevant signals (fetch more events or+ frames if needed).+- Produce a fully revised analysis and call **submit_analysis** again with the+ improved intent + steps. Keep step ids stable where a step is unchanged.++Always finish a turn by calling submit_analysis. Do not reply with prose instead.+`.trim();diff --git a/electron/describer/tools.ts b/electron/describer/tools.tsnew file mode 100644index 0000000..d7399a1--- /dev/null+++ b/electron/describer/tools.ts@@ -0,0 +1,302 @@+import { existsSync, readFileSync } from "node:fs";+import path from "node:path";++import type { Tool } from "@github/copilot-sdk";++import { AnalysisSubmissionSchema, type AnalysisSubmission } from "../../common/analysis";+import { MEANINGFUL_EVENT_TYPES } from "../../common/correlation";+import type { SessionBundle } from "../../common/bundle";+import type { RecEvent } from "../../common/types";+import { readEvents } from "../frames/correlate";+import type { FrameExtractor } from "../frames/extractor";+import { createLogger } from "../logger";++const log = createLogger("Describer/tools");++/** Everything a session's tools are bound to. */+export interface ToolContext {+ sessionDir: string;+ /** `session.json` startedAt (epoch ms) — origin of the agent's `atMs` clock. */+ startedAt: number;+ /** Frame service, or null when the session has no video. */+ extractor: FrameExtractor | null;+ /** Streamed to the UI as the agent works. */+ onProgress?: (message: string) => void;+ /** Called when the agent submits a (validated) analysis. */+ onSubmit: (submission: AnalysisSubmission) => void;+}++const MAX_IMAGES_PER_CALL = 6;+const MAX_STR = 2000;++const truncate = (v: unknown): unknown =>+ typeof v === "string" && v.length > MAX_STR ? v.slice(0, MAX_STR) + "…[truncated]" : v;++function readBundle(sessionDir: string): SessionBundle | null {+ const p = path.join(sessionDir, "bundle.json");+ if (!existsSync(p)) return null;+ try {+ return JSON.parse(readFileSync(p, "utf8")) as SessionBundle;+ } catch {+ return null;+ }+}++/**+ * Build the custom in-process tools exposed to one Copilot session. Handlers run+ * in the Electron main process (dispatched over JSON-RPC by the SDK), read the+ * session's captured signals, and — for frames — return the JPEGs **inline** so+ * the model can see the screen without any `view`/`bash` tool. `submit_analysis`+ * captures guaranteed-structured output. The whole surface is sandboxed to the+ * one session dir.+ */+export function createDescriberTools(ctx: ToolContext): Tool[] {+ const { sessionDir, startedAt, extractor, onSubmit } = ctx;+ const progress = (m: string) => ctx.onProgress?.(m);+ const rel = (epoch: number) => Math.round(epoch - startedAt);+ const abs = (atMs: number) => startedAt + atMs;+ const sec = (atMs: number) => (atMs / 1000).toFixed(1);++ const getTimeline: Tool = {+ name: "get_timeline",+ description:+ "Return the segmented session timeline: ordered steps with their start time (atMs, ms since recording start), duration, app, urls, titles, commands, clipboard counts and markers. Call this first.",+ parameters: { type: "object", properties: {}, additionalProperties: false },+ handler: () => {+ progress("Reading the timeline…");+ const bundle = readBundle(sessionDir);+ if (!bundle) {+ return "No timeline is available yet (bundle.json missing). Use get_events to read the raw event stream instead.";+ }+ const view = {+ durationMs: bundle.session.durationMs,+ platform: bundle.session.platform,+ stats: bundle.stats,+ steps: bundle.steps.map((s) => ({+ index: s.index,+ atMs: rel(s.startMs),+ durationMs: s.durationMs,+ boundary: s.boundary,+ app: s.app,+ titles: s.titles,+ hosts: s.hosts,+ urls: s.urls,+ commands: s.commands,+ clipboardCount: s.clipboardCount,+ markers: s.markers,+ frameCount: s.frames.length,+ summary: s.summary,+ })),+ };+ return JSON.stringify(view, null, 2);+ },+ };++ const getEvents: Tool = {+ name: "get_events",+ description:+ "Return the raw event stream (with clipboard text, full titles, full URLs, commands). Optionally filter by `types` (e.g. [\"browser.url\",\"clipboard.change\"]) and/or a time window `fromMs`/`toMs` (atMs, ms since recording start). Defaults to the meaningful events across the whole session.",+ parameters: {+ type: "object",+ properties: {+ types: { type: "array", items: { type: "string" }, description: "Event types to include." },+ fromMs: { type: "number", description: "Window start (atMs)." },+ toMs: { type: "number", description: "Window end (atMs)." },+ },+ additionalProperties: false,+ },+ handler: (raw) => {+ const args = (raw ?? {}) as { types?: string[]; fromMs?: number; toMs?: number };+ const wanted = args.types && args.types.length ? new Set(args.types) : null;+ const from = typeof args.fromMs === "number" ? args.fromMs : -Infinity;+ const to = typeof args.toMs === "number" ? args.toMs : Infinity;+ progress(+ `Reading events ${args.fromMs != null ? sec(args.fromMs) + "s" : "start"}–${+ args.toMs != null ? sec(args.toMs) + "s" : "end"+ }…`,+ );+ let events: RecEvent[];+ try {+ events = readEvents(path.join(sessionDir, "events.jsonl"));+ } catch (err) {+ return `Could not read events: ${err instanceof Error ? err.message : String(err)}`;+ }+ const rows = events+ .filter((e) => (wanted ? wanted.has(e.type) : MEANINGFUL_EVENT_TYPES.has(e.type)))+ .map((e) => ({ e, atMs: rel(e.epoch) }))+ .filter(({ atMs }) => atMs >= from && atMs <= to)+ .map(({ e, atMs }) => {+ const payload: Record<string, unknown> = {};+ for (const [k, v] of Object.entries(e.payload)) payload[k] = truncate(v);+ return { seq: e.seq, atMs, type: e.type, source: e.source, ...payload };+ });+ return JSON.stringify({ count: rows.length, events: rows }, null, 2);+ },+ };++ const listFrames: Tool = {+ name: "list_frames",+ description:+ "List the screen frames already available for this session (file, atMs, source, reason). Returns an empty list when no video was recorded. Use get_frames to actually view frames.",+ parameters: { type: "object", properties: {}, additionalProperties: false },+ handler: () => {+ progress("Listing available frames…");+ if (!extractor) return JSON.stringify({ hasVideo: false, frames: [] });+ const frames = extractor.manifest.map((f) => ({+ file: f.file,+ atMs: rel(f.tMs),+ source: f.source,+ reason: f.reason,+ }));+ return JSON.stringify({ hasVideo: true, frames }, null, 2);+ },+ };++ const getFrames: Tool = {+ name: "get_frames",+ description:+ "Sample and VIEW screen frames within a time window [fromMs, toMs] (atMs, ms since recording start). Densely re-samples the window if needed (optional `fps`, default 6) and returns the frames inline as images so you can see the screen. Optional `crop` {x,y,w,h} zooms a region. Use only where events are ambiguous.",+ parameters: {+ type: "object",+ properties: {+ fromMs: { type: "number", description: "Window start (atMs)." },+ toMs: { type: "number", description: "Window end (atMs)." },+ fps: { type: "number", description: "Sampling density (frames/sec). Default 6." },+ crop: {+ type: "object",+ properties: {+ x: { type: "number" },+ y: { type: "number" },+ w: { type: "number" },+ h: { type: "number" },+ },+ required: ["x", "y", "w", "h"],+ additionalProperties: false,+ },+ reason: { type: "string", description: "Why you need a closer look." },+ },+ required: ["fromMs", "toMs"],+ additionalProperties: false,+ },+ handler: async (raw) => {+ const args = (raw ?? {}) as {+ fromMs?: number;+ toMs?: number;+ fps?: number;+ crop?: { x: number; y: number; w: number; h: number };+ reason?: string;+ };+ if (!extractor) {+ return { textResultForLlm: "No video was recorded for this session; no frames are available.", resultType: "failure" };+ }+ const fromMs = typeof args.fromMs === "number" ? args.fromMs : 0;+ const toMs = typeof args.toMs === "number" ? Math.max(fromMs, args.toMs) : fromMs + 3000;+ progress(`Looking at ${sec(fromMs)}–${sec(toMs)}s…`);+ try {+ await extractor.extractWindow({+ startMs: abs(fromMs),+ endMs: abs(toMs),+ fps: args.fps,+ crop: args.crop,+ reason: args.reason ?? "describer:get_frames",+ });+ } catch (err) {+ log.warn("get_frames extract failed:", err instanceof Error ? err.message : err);+ }++ const inWindow = extractor.manifest.filter((f) => {+ const atMs = rel(f.tMs);+ return atMs >= fromMs && atMs <= toMs;+ });+ if (inWindow.length === 0) {+ return { textResultForLlm: `No frames could be extracted for ${sec(fromMs)}–${sec(toMs)}s.`, resultType: "success" };+ }+ // Evenly sample which frames to attach as images (bounded cost).+ const picks: typeof inWindow = [];+ if (inWindow.length <= MAX_IMAGES_PER_CALL) picks.push(...inWindow);+ else {+ const stride = (inWindow.length - 1) / (MAX_IMAGES_PER_CALL - 1);+ for (let i = 0; i < MAX_IMAGES_PER_CALL; i++) picks.push(inWindow[Math.round(i * stride)]);+ }++ const binaryResultsForLlm: { data: string; mimeType: string; type: "image"; description?: string }[] = [];+ for (const f of picks) {+ const file = path.join(sessionDir, "frames", f.file);+ if (!existsSync(file)) continue;+ try {+ binaryResultsForLlm.push({+ data: readFileSync(file).toString("base64"),+ mimeType: "image/jpeg",+ type: "image",+ description: `${sec(rel(f.tMs))}s (${f.source}) ${f.file}`,+ });+ } catch (err) {+ log.warn("frame read failed:", err instanceof Error ? err.message : err);+ }+ }+ const listing = inWindow.map((f) => `- ${sec(rel(f.tMs))}s ${f.file} (${f.source})`).join("\n");+ progress(`Attached ${binaryResultsForLlm.length} frame image(s) from ${sec(fromMs)}–${sec(toMs)}s.`);+ return {+ textResultForLlm:+ `Frames in ${sec(fromMs)}–${sec(toMs)}s (${inWindow.length} total, ${binaryResultsForLlm.length} shown as images):\n${listing}`,+ binaryResultsForLlm,+ resultType: "success",+ };+ },+ };++ const submitAnalysis: Tool = {+ name: "submit_analysis",+ description:+ "Submit your final structured analysis: the overall intent and the ordered list of steps. Call this exactly once when confident (and again on later turns after incorporating feedback).",+ parameters: {+ type: "object",+ properties: {+ title: {+ type: "string",+ description:+ 'A short 2–5 word label for this task in Title Case, no trailing period, e.g. "Research habit articles" or "Extract invoice data". Used as the session name in lists.',+ },+ intent: { type: "string", description: "One sentence naming the user's overall goal." },+ intentConfidence: { type: "string", enum: ["high", "medium", "low"] },+ intentRationale: { type: "string", description: "1–2 sentences of evidence for the intent." },+ steps: {+ type: "array",+ items: {+ type: "object",+ properties: {+ id: { type: "string", description: "Stable short id, e.g. \"s1\"." },+ title: { type: "string" },+ detail: { type: "string" },+ startMs: { type: "number", description: "Step start (atMs)." },+ endMs: { type: "number", description: "Step end (atMs)." },+ apps: { type: "array", items: { type: "string" } },+ evidence: { type: "array", items: { type: "string" } },+ confidence: { type: "string", enum: ["high", "medium", "low"] },+ },+ required: ["id", "title", "detail"],+ additionalProperties: false,+ },+ },+ },+ required: ["title", "intent", "steps"],+ additionalProperties: false,+ },+ handler: (raw) => {+ const parsed = AnalysisSubmissionSchema.safeParse(raw);+ if (!parsed.success) {+ return {+ textResultForLlm:+ "submit_analysis rejected — the payload did not match the schema. Fix these and call again:\n" ++ parsed.error.issues.map((i) => `- ${i.path.join(".") || "(root)"}: ${i.message}`).join("\n"),+ resultType: "failure",+ };+ }+ progress("Received analysis draft.");+ onSubmit(parsed.data);+ return "Analysis recorded. You may stop now.";+ },+ };++ return [getTimeline, getEvents, listFrames, getFrames, submitAnalysis];+}diff --git a/electron/ipc.ts b/electron/ipc.tsindex 1a60c6f..af75f29 100644--- a/electron/ipc.ts+++ b/electron/ipc.ts@@ -1,14 +1,24 @@import { ipcMain } from "electron";+import path from "node:path";import { type CaptureConfig, type CaptureLevel, levelForConfig } from "../common/config";-import type { CaptureState } from "../common/ipc";+import type { AnalysisEditInput, AnalysisFeedbackInput, AnalyzeResult, CaptureState } from "../common/ipc";import { IPC } from "../common/ipc";+import { Describer, loadPersistedAnalysis } from "./describer/describer";import { runDoctor } from "./doctor";+import { createLogger } from "./logger";import type { RecorderController } from "./recorder/controller";+import { listSessions } from "./sessions";import type { SettingsStore } from "./settings";-/** Wire the renderer-facing invoke channels to the recorder, doctor + settings. */-export function registerIpc(recorder: RecorderController, settings: SettingsStore): void {+const log = createLogger("IPC");++/** Wire the renderer-facing invoke channels to the recorder, describer, doctor + settings. */+export function registerIpc(+ recorder: RecorderController,+ settings: SettingsStore,+ describer: Describer,+): void {const captureState = (): CaptureState => {const config = settings.resolve();return { level: levelForConfig(config), config };@@ -28,4 +38,80 @@ export function registerIpc(recorder: RecorderController, settings: SettingsStorsettings.setConfig(config);return captureState();});++ const resolveSessionId = (sessionId?: string): string | null => {+ if (sessionId) return sessionId;+ const dir = recorder.lastSessionDir();+ return dir ? path.basename(dir) : null;+ };++ ipcMain.handle(IPC.analyze, async (_event, sessionId?: string): Promise<AnalyzeResult> => {+ const id = resolveSessionId(sessionId);+ if (!id) return { ok: false, error: "No completed session to analyze yet." };+ try {+ const analysis = await describer.analyze(id);+ return { ok: true, analysis };+ } catch (err) {+ const error = err instanceof Error ? err.message : String(err);+ log.warn("analyze failed:", error);+ return { ok: false, error };+ }+ });++ ipcMain.handle(+ IPC.analyzeFeedback,+ async (_event, input: AnalysisFeedbackInput): Promise<AnalyzeResult> => {+ if (!input?.sessionId) return { ok: false, error: "Missing sessionId." };+ try {+ const analysis = await describer.feedback(input.sessionId, {+ overall: input.overall,+ steps: input.steps ?? [],+ });+ return { ok: true, analysis };+ } catch (err) {+ const error = err instanceof Error ? err.message : String(err);+ log.warn("feedback failed:", error);+ return { ok: false, error };+ }+ },+ );++ ipcMain.handle(IPC.getAnalysis, (_event, sessionId: string) =>+ sessionId ? loadPersistedAnalysis(sessionId) : null,+ );++ ipcMain.handle(IPC.approveAnalysis, async (_event, sessionId: string): Promise<AnalyzeResult> => {+ if (!sessionId) return { ok: false, error: "Missing sessionId." };+ try {+ const analysis = await describer.approve(sessionId);+ return { ok: true, analysis };+ } catch (err) {+ const error = err instanceof Error ? err.message : String(err);+ log.warn("approve failed:", error);+ return { ok: false, error };+ }+ });++ ipcMain.handle(IPC.updateAnalysis, async (_event, input: AnalysisEditInput): Promise<AnalyzeResult> => {+ if (!input?.sessionId) return { ok: false, error: "Missing sessionId." };+ try {+ const analysis = await describer.edit(input.sessionId, {+ title: input.title,+ intent: input.intent,+ });+ return { ok: true, analysis };+ } catch (err) {+ const error = err instanceof Error ? err.message : String(err);+ log.warn("update failed:", error);+ return { ok: false, error };+ }+ });++ ipcMain.handle(IPC.cancelAnalysis, async (_event, sessionId: string) => {+ if (sessionId) await describer.cancel(sessionId);+ return { ok: true };+ });++ ipcMain.handle(IPC.listSessions, () => listSessions());}+diff --git a/electron/preload.cjs b/electron/preload.cjsindex 5c5a798..b2a0674 100644--- a/electron/preload.cjs+++ b/electron/preload.cjs@@ -12,6 +12,16 @@ const IPC = {getCapture: "capture:get",setLevel: "capture:set-level",setConfig: "capture:set-config",+ analyze: "analyze:start",+ analyzeFeedback: "analyze:feedback",+ getAnalysis: "analyze:get",+ approveAnalysis: "analyze:approve",+ updateAnalysis: "analyze:update",+ cancelAnalysis: "analyze:cancel",+ analyzeProgress: "analyze:progress",+ listSessions: "sessions:list",+ openLibrary: "ui:open-library",+ closeLibrary: "ui:close-library",};contextBridge.exposeInMainWorld("skillRecorder", {@@ -28,4 +38,18 @@ contextBridge.exposeInMainWorld("skillRecorder", {ipcRenderer.on(IPC.statusChanged, listener);return () => ipcRenderer.removeListener(IPC.statusChanged, listener);},+ analyze: (sessionId) => ipcRenderer.invoke(IPC.analyze, sessionId),+ analyzeFeedback: (input) => ipcRenderer.invoke(IPC.analyzeFeedback, input),+ getAnalysis: (sessionId) => ipcRenderer.invoke(IPC.getAnalysis, sessionId),+ approveAnalysis: (sessionId) => ipcRenderer.invoke(IPC.approveAnalysis, sessionId),+ updateAnalysis: (input) => ipcRenderer.invoke(IPC.updateAnalysis, input),+ cancelAnalysis: (sessionId) => ipcRenderer.invoke(IPC.cancelAnalysis, sessionId),+ onAnalyzeProgress: (cb) => {+ const listener = (_event, progress) => cb(progress);+ ipcRenderer.on(IPC.analyzeProgress, listener);+ return () => ipcRenderer.removeListener(IPC.analyzeProgress, listener);+ },+ listSessions: () => ipcRenderer.invoke(IPC.listSessions),+ openLibrary: () => ipcRenderer.invoke(IPC.openLibrary),+ closeLibrary: () => ipcRenderer.invoke(IPC.closeLibrary),});diff --git a/electron/sessions.ts b/electron/sessions.tsnew file mode 100644index 0000000..6866295--- /dev/null+++ b/electron/sessions.ts@@ -0,0 +1,64 @@+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";+import path from "node:path";++import type { SessionSummary } from "../common/ipc";+import type { SessionMeta } from "../common/types";+import { loadPersistedAnalysis } from "./describer/describer";+import { createLogger } from "./logger";+import { sessionsRoot } from "./recorder/session-store";++const log = createLogger("Sessions");++/** Enumerate saved recordings for the sessions library, newest first. */+export function listSessions(): SessionSummary[] {+ const root = sessionsRoot();+ if (!existsSync(root)) return [];++ const out: SessionSummary[] = [];+ let names: string[];+ try {+ names = readdirSync(root);+ } catch (err) {+ log.warn("failed to read sessions dir:", err instanceof Error ? err.message : err);+ return [];+ }++ for (const name of names) {+ const dir = path.join(root, name);+ try {+ if (!statSync(dir).isDirectory()) continue;+ } catch {+ continue;+ }+ let meta: SessionMeta | null = null;+ try {+ meta = JSON.parse(readFileSync(path.join(dir, "session.json"), "utf8")) as SessionMeta;+ } catch {+ continue; // not a valid session dir+ }+ if (!meta?.id) continue;++ const analysis = loadPersistedAnalysis(name);+ out.push({+ id: meta.id,+ startedAt: meta.startedAt ?? null,+ stoppedAt: meta.stoppedAt ?? null,+ durationMs: meta.startedAt && meta.stoppedAt ? meta.stoppedAt - meta.startedAt : null,+ processed: existsSync(path.join(dir, "bundle.json")),+ hasVideo: existsSync(path.join(dir, "video.json")),+ analysis: analysis+ ? {+ revision: analysis.revision,+ title: analysis.title ?? "",+ intent: analysis.intent,+ intentConfidence: analysis.intentConfidence,+ stepCount: analysis.steps.length,+ approved: analysis.approved ?? false,+ }+ : null,+ });+ }++ out.sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));+ return out;+}