π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(voice): attribute transcript segments to capture steps
Luis Ortiz1ca57293c8b8
diff --git a/src/core/capture/voice/__tests__/attribute.test.ts b/src/core/capture/voice/__tests__/attribute.test.tsnew file mode 100644index 0000000..87e2469--- /dev/null+++ b/src/core/capture/voice/__tests__/attribute.test.ts@@ -0,0 +1,83 @@+import { describe, expect, it } from 'vitest';+import { assignSegments, makeToAbsolute } from '../attribute';+import { absoluteSeconds, type Batch, type StepWindow } from '../types';++const steps: StepWindow[] = [+ { stepId: 's1', from: absoluteSeconds(0), to: absoluteSeconds(10) },+ { stepId: 's2', from: absoluteSeconds(10), to: absoluteSeconds(20) },+];++const batch = (ranges: [number, number][]): Batch => ({+ start: absoluteSeconds(ranges[0][0]),+ end: absoluteSeconds(ranges[ranges.length - 1][1]),+ segments: ranges.map(([start, end]) => ({+ start: absoluteSeconds(start),+ end: absoluteSeconds(end),+ })),+});++const scores = { no_speech_prob: 0.01, avg_logprob: -0.2, compression_ratio: 1.4 };++describe('makeToAbsolute', () => {+ it('maps batch time back across removed silence', () => {+ const toAbs = makeToAbsolute(+ batch([+ [2, 5],+ [15, 18],+ ]),+ );+ expect(toAbs(0)).toBeCloseTo(2);+ expect(toAbs(3)).toBeCloseTo(5);+ expect(toAbs(4)).toBeCloseTo(16);+ });+});++describe('assignSegments', () => {+ const whole = batch([[0, 20]]);++ it('uses a segment verbatim when it sits inside one step', () => {+ const result = assignSegments(+ { segments: [{ start: 1, end: 5, text: 'Open the settings menu.', ...scores }] },+ whole,+ steps,+ );+ expect(result.byStep.get('s1')).toEqual(['Open the settings menu.']);+ expect(result.verbatim).toBe(1);+ expect(result.split).toBe(0);+ });++ it('splits a segment by word when it straddles a step boundary', () => {+ const result = assignSegments(+ {+ segments: [{ start: 8, end: 12, text: 'click edit then save', ...scores }],+ words: [+ { word: 'click', start: 8, end: 8.5 },+ { word: 'edit', start: 9, end: 9.5 },+ { word: 'then', start: 11, end: 11.4 },+ { word: 'save', start: 11.5, end: 12 },+ ],+ },+ whole,+ steps,+ );+ expect(result.byStep.get('s1')).toEqual(['click edit']);+ expect(result.byStep.get('s2')).toEqual(['then save']);+ expect(result.split).toBe(1);+ });++ it('drops a rejected segment entirely', () => {+ const result = assignSegments(+ { segments: [{ start: 1, end: 2, text: 'Thanks for watching!', ...scores }] },+ whole,+ steps,+ );+ expect(result.byStep.size).toBe(0);+ expect(result.rejected).toBe(1);+ });++ it('drops an unscored segment', () => {+ const result = assignSegments({ segments: [{ start: 1, end: 2, text: 'Open settings.' }] }, whole, steps);+ expect(result.byStep.size).toBe(0);+ expect(result.rejected).toBe(1);+ });+});diff --git a/src/core/capture/voice/attribute.ts b/src/core/capture/voice/attribute.tsnew file mode 100644index 0000000..de8ca6b--- /dev/null+++ b/src/core/capture/voice/attribute.ts@@ -0,0 +1,81 @@+import { rejectReason } from './filter';+import {+ type AbsoluteSeconds,+ absoluteSeconds,+ type Batch,+ type StepWindow,+ type TranscriptionResponse,+} from './types';++export function makeToAbsolute(batch: Batch): (batchTime: number) => AbsoluteSeconds {+ const bounds: { offset: number; start: number; length: number }[] = [];+ let elapsed = 0;+ for (const s of batch.segments) {+ bounds.push({ offset: elapsed, start: s.start, length: s.end - s.start });+ elapsed += s.end - s.start;+ }+ return (batchTime) => {+ for (const b of bounds) {+ if (batchTime <= b.offset + b.length) {+ return absoluteSeconds(b.start + (batchTime - b.offset));+ }+ }+ const last = bounds[bounds.length - 1];+ return absoluteSeconds(last ? last.start + last.length : batchTime);+ };+}++export interface AssignResult {+ byStep: Map<string, string[]>;+ verbatim: number;+ split: number;+ rejected: number;+}++export function assignSegments(response: TranscriptionResponse, batch: Batch, steps: StepWindow[]): AssignResult {+ const toAbsolute = makeToAbsolute(batch);+ const byStep = new Map<string, string[]>();+ const add = (stepId: string, text: string) => {+ if (!text) return;+ const existing = byStep.get(stepId);+ if (existing) existing.push(text);+ else byStep.set(stepId, [text]);+ };++ let verbatim = 0;+ let split = 0;+ let rejected = 0;++ for (const segment of response.segments ?? []) {+ if (rejectReason(segment)) {+ rejected += 1;+ continue;+ }+ const start = toAbsolute(segment.start);+ const end = toAbsolute(segment.end);+ const spanned = steps.filter((s) => start < s.to && end > s.from);++ if (spanned.length <= 1) {+ const step = spanned[0] ?? steps.find((s) => start >= s.from && start <= s.to);+ if (step) {+ add(step.stepId, segment.text.trim());+ verbatim += 1;+ }+ continue;+ }++ split += 1;+ const grouped = new Map<string, string[]>();+ for (const word of response.words ?? []) {+ if (word.start < segment.start || word.start > segment.end) continue;+ const at = toAbsolute(word.start);+ const step = steps.find((s) => at >= s.from && at <= s.to) ?? spanned[spanned.length - 1];+ const existing = grouped.get(step.stepId);+ if (existing) existing.push(word.word.trim());+ else grouped.set(step.stepId, [word.word.trim()]);+ }+ for (const [stepId, words] of grouped) add(stepId, words.join(' ').trim());+ }++ return { byStep, verbatim, split, rejected };+}