Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(voice): attribute transcript segments to capture steps

Luis Ortiz1ca57293c8b8
diff --git a/src/core/capture/voice/__tests__/attribute.test.ts b/src/core/capture/voice/__tests__/attribute.test.ts
new file mode 100644
index 0000000..87e2469
--- /dev/null
+++ b/src/core/capture/voice/__tests__/attribute.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, it } from 'vitest';
+import { assignSegments, makeToAbsolute } from '../attribute';
+import { absoluteSeconds, type Batch, type StepWindow } from '../types';
+
+const steps: StepWindow[] = [
+ { stepId: 's1', from: absoluteSeconds(0), to: absoluteSeconds(10) },
+ { stepId: 's2', from: absoluteSeconds(10), to: absoluteSeconds(20) },
+];
+
+const batch = (ranges: [number, number][]): Batch => ({
+ start: absoluteSeconds(ranges[0][0]),
+ end: absoluteSeconds(ranges[ranges.length - 1][1]),
+ segments: ranges.map(([start, end]) => ({
+ start: absoluteSeconds(start),
+ end: absoluteSeconds(end),
+ })),
+});
+
+const scores = { no_speech_prob: 0.01, avg_logprob: -0.2, compression_ratio: 1.4 };
+
+describe('makeToAbsolute', () => {
+ it('maps batch time back across removed silence', () => {
+ const toAbs = makeToAbsolute(
+ batch([
+ [2, 5],
+ [15, 18],
+ ]),
+ );
+ expect(toAbs(0)).toBeCloseTo(2);
+ expect(toAbs(3)).toBeCloseTo(5);
+ expect(toAbs(4)).toBeCloseTo(16);
+ });
+});
+
+describe('assignSegments', () => {
+ const whole = batch([[0, 20]]);
+
+ it('uses a segment verbatim when it sits inside one step', () => {
+ const result = assignSegments(
+ { segments: [{ start: 1, end: 5, text: 'Open the settings menu.', ...scores }] },
+ whole,
+ steps,
+ );
+ expect(result.byStep.get('s1')).toEqual(['Open the settings menu.']);
+ expect(result.verbatim).toBe(1);
+ expect(result.split).toBe(0);
+ });
+
+ it('splits a segment by word when it straddles a step boundary', () => {
+ const result = assignSegments(
+ {
+ segments: [{ start: 8, end: 12, text: 'click edit then save', ...scores }],
+ words: [
+ { word: 'click', start: 8, end: 8.5 },
+ { word: 'edit', start: 9, end: 9.5 },
+ { word: 'then', start: 11, end: 11.4 },
+ { word: 'save', start: 11.5, end: 12 },
+ ],
+ },
+ whole,
+ steps,
+ );
+ expect(result.byStep.get('s1')).toEqual(['click edit']);
+ expect(result.byStep.get('s2')).toEqual(['then save']);
+ expect(result.split).toBe(1);
+ });
+
+ it('drops a rejected segment entirely', () => {
+ const result = assignSegments(
+ { segments: [{ start: 1, end: 2, text: 'Thanks for watching!', ...scores }] },
+ whole,
+ steps,
+ );
+ expect(result.byStep.size).toBe(0);
+ expect(result.rejected).toBe(1);
+ });
+
+ it('drops an unscored segment', () => {
+ const result = assignSegments({ segments: [{ start: 1, end: 2, text: 'Open settings.' }] }, whole, steps);
+ expect(result.byStep.size).toBe(0);
+ expect(result.rejected).toBe(1);
+ });
+});
diff --git a/src/core/capture/voice/attribute.ts b/src/core/capture/voice/attribute.ts
new file mode 100644
index 0000000..de8ca6b
--- /dev/null
+++ b/src/core/capture/voice/attribute.ts
@@ -0,0 +1,81 @@
+import { rejectReason } from './filter';
+import {
+ type AbsoluteSeconds,
+ absoluteSeconds,
+ type Batch,
+ type StepWindow,
+ type TranscriptionResponse,
+} from './types';
+
+export function makeToAbsolute(batch: Batch): (batchTime: number) => AbsoluteSeconds {
+ const bounds: { offset: number; start: number; length: number }[] = [];
+ let elapsed = 0;
+ for (const s of batch.segments) {
+ bounds.push({ offset: elapsed, start: s.start, length: s.end - s.start });
+ elapsed += s.end - s.start;
+ }
+ return (batchTime) => {
+ for (const b of bounds) {
+ if (batchTime <= b.offset + b.length) {
+ return absoluteSeconds(b.start + (batchTime - b.offset));
+ }
+ }
+ const last = bounds[bounds.length - 1];
+ return absoluteSeconds(last ? last.start + last.length : batchTime);
+ };
+}
+
+export interface AssignResult {
+ byStep: Map<string, string[]>;
+ verbatim: number;
+ split: number;
+ rejected: number;
+}
+
+export function assignSegments(response: TranscriptionResponse, batch: Batch, steps: StepWindow[]): AssignResult {
+ const toAbsolute = makeToAbsolute(batch);
+ const byStep = new Map<string, string[]>();
+ const add = (stepId: string, text: string) => {
+ if (!text) return;
+ const existing = byStep.get(stepId);
+ if (existing) existing.push(text);
+ else byStep.set(stepId, [text]);
+ };
+
+ let verbatim = 0;
+ let split = 0;
+ let rejected = 0;
+
+ for (const segment of response.segments ?? []) {
+ if (rejectReason(segment)) {
+ rejected += 1;
+ continue;
+ }
+ const start = toAbsolute(segment.start);
+ const end = toAbsolute(segment.end);
+ const spanned = steps.filter((s) => start < s.to && end > s.from);
+
+ if (spanned.length <= 1) {
+ const step = spanned[0] ?? steps.find((s) => start >= s.from && start <= s.to);
+ if (step) {
+ add(step.stepId, segment.text.trim());
+ verbatim += 1;
+ }
+ continue;
+ }
+
+ split += 1;
+ const grouped = new Map<string, string[]>();
+ for (const word of response.words ?? []) {
+ if (word.start < segment.start || word.start > segment.end) continue;
+ const at = toAbsolute(word.start);
+ const step = steps.find((s) => at >= s.from && at <= s.to) ?? spanned[spanned.length - 1];
+ const existing = grouped.get(step.stepId);
+ if (existing) existing.push(word.word.trim());
+ else grouped.set(step.stepId, [word.word.trim()]);
+ }
+ for (const [stepId, words] of grouped) add(stepId, words.join(' ').trim());
+ }
+
+ return { byStep, verbatim, split, rejected };
+}