Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.
TypeScript agent-skillsai-agentsautomationcopilotcopilot-clicopilot-coworkcopilot-studioelectronmicrosoft-scoutscreen-recording
Fix narration dropping real speech via midpoint silence test (#30)
adileifb977cd06b9a
diff --git a/electron/narration/transcribe.test.ts b/electron/narration/transcribe.test.tsnew file mode 100644index 0000000..b8c7b19--- /dev/null+++ b/electron/narration/transcribe.test.ts@@ -0,0 +1,44 @@+import assert from "node:assert/strict";+import test from "node:test";++import type { SilenceSpan } from "./audio-analysis";+import { isMostlySilent } from "./transcribe";++test("keeps a long real-speech chunk that is only ~44% silent", () => {+ // Regression: the old midpoint test dropped this 16.18s chunk purely because its+ // center (8.09s) landed inside the 7.14-9.44s mid-sentence pause. Natural pauses+ // add up to ~44% of the span, but the majority is speech, so it must be kept.+ const silences: SilenceSpan[] = [+ { start: 1.0, end: 2.4 },+ { start: 7.14, end: 9.44 },+ { start: 11.0, end: 12.5 },+ { start: 14.0, end: 16.0 },+ ];+ assert.equal(isMostlySilent(0, 16.18, silences), false);+});++test("drops a chunk whose span is essentially all silence", () => {+ // A silence span that fully covers (and overruns) the chunk: clipped coverage+ // is 100%, so it's a hallucination over silence and gets dropped.+ assert.equal(isMostlySilent(10, 20, [{ start: 9, end: 21 }]), true);+});++test("uses 0.85 as the drop threshold", () => {+ assert.equal(isMostlySilent(0, 10, [{ start: 0, end: 9 }]), true); // 0.90 -> drop+ assert.equal(isMostlySilent(0, 10, [{ start: 0, end: 8 }]), false); // 0.80 -> keep+});++test("degenerates to a point-in-silence test for a zero-length span", () => {+ assert.equal(isMostlySilent(5, 5, [{ start: 4, end: 6 }]), true);+ assert.equal(isMostlySilent(5, 5, [{ start: 10, end: 12 }]), false);+});++test("treats a negative span defensively via the start point", () => {+ assert.equal(isMostlySilent(8, 5, [{ start: 7, end: 9 }]), true);+ assert.equal(isMostlySilent(8, 5, [{ start: 1, end: 2 }]), false);+});++test("keeps a chunk with no overlapping silence", () => {+ assert.equal(isMostlySilent(0, 10, [{ start: 20, end: 25 }, { start: 30, end: 35 }]), false);+ assert.equal(isMostlySilent(0, 10, []), false);+});diff --git a/electron/narration/transcribe.ts b/electron/narration/transcribe.tsindex b5c9bda..5948412 100644--- a/electron/narration/transcribe.ts+++ b/electron/narration/transcribe.ts@@ -62,10 +62,30 @@ export async function transcribeNarration(return { model: narrationModelId(), segments };}-/** A chunk is dropped when its midpoint sits inside a detected silence span. */-function isMostlySilent(startSec: number, endSec: number, silences: SilenceSpan[]): boolean {- const mid = (startSec + endSec) / 2;- return silences.some((s) => mid >= s.start && mid <= s.end);+/**+ * A chunk is dropped only when almost its entire span is silence, i.e. it is a+ * Whisper hallucination over near-silent audio (see `BOILERPLATE`/`isMeaningful`).+ *+ * We measure the fraction of `[startSec, endSec]` covered by detected silence and+ * treat the chunk as silent at >= 0.85 coverage. The earlier midpoint test dropped+ * long real-speech chunks whenever a natural mid-sentence pause happened to land+ * near the temporal center — a single 16s chunk of clear speech could vanish. A+ * coverage test keeps those chunks while still catching all-silence hallucinations.+ *+ * `detectSilence` returns non-overlapping ascending spans, so summing the clipped+ * overlaps never double-counts. A zero/negative span can't have a fraction, so it+ * falls back to a point-in-silence test.+ */+export function isMostlySilent(startSec: number, endSec: number, silences: SilenceSpan[]): boolean {+ const span = endSec - startSec;+ if (span <= 0) return silences.some((s) => startSec >= s.start && startSec <= s.end);+ let covered = 0;+ for (const s of silences) {+ const lo = Math.max(startSec, s.start);+ const hi = Math.min(endSec, s.end);+ if (hi > lo) covered += hi - lo;+ }+ return covered / span >= 0.85;}// Whisper hallucinates stock phrases over silence/noise; drop the usual suspectsdiff --git a/package.json b/package.jsonindex ab18bd9..36573a0 100644--- a/package.json+++ b/package.json@@ -14,7 +14,7 @@"build": "tsc --noEmit && vite build","typecheck": "tsc --noEmit","typecheck:evals": "tsc --noEmit -p evals/tsconfig.json",- "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test common/audio.test.ts electron/recording-controls-bounds.test.ts electron/recorder/controller.test.ts electron/frames/extractor.test.ts electron/narration/audio-analysis.test.ts electron/sessions.test.ts",+ "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test common/audio.test.ts electron/recording-controls-bounds.test.ts electron/recorder/controller.test.ts electron/frames/extractor.test.ts electron/narration/audio-analysis.test.ts electron/narration/transcribe.test.ts electron/sessions.test.ts","eval": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/run.ts","eval:builder": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/builder/run.ts","eval:skill": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/skillbuilder/run.ts",