Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

10.7 KB
Raw Blame
import type { Video } from "@cap/web-domain";
import {
	EDIT_TRANSCRIPT_VERSION,
	type EditTranscript,
	type EditTranscriptWord,
	editTranscriptWordsToCaptionVtt,
} from "@/lib/edit-transcript";
import type { NormalizedSegmentEntry } from "@/lib/segments-audio";
import { planSegmentsAudioExtraction } from "@/lib/segments-audio";

/**
 * Pure logic for the provisional live transcription of instant-mode
 * recordings: which audio segments form the next chunk, how chunk words are
 * placed on the recording timeline, and the `transcription.live.json`
 * artifact that the share page reads while the canonical transcription
 * doesn't exist yet.
 */

export const LIVE_TRANSCRIBE = {
	/** First chunk fires fast so a transcript exists almost immediately. */
	INITIAL_CHUNK_SECONDS: 5,
	/** Steady-state window. Small on purpose: the untranscribed tail at stop
	 * averages half this, and that tail is all that stands between "stop" and
	 * a complete transcript on the share page. */
	MAX_CHUNK_SECONDS: 10,
	GROW_AFTER_CHUNKS: 2,
	/** Catch-up bound: one chunk never covers more audio than this. */
	MAX_CHUNK_TAKE_SECONDS: 60,
	/** Hard cost cap: stop live-transcribing after this much audio; the
	 * canonical post-stop transcription covers the full recording anyway. */
	MAX_TRANSCRIBED_SECONDS: 60 * 60,
	/** Hard backstop on chunk count (1h at 10s windows = 360). */
	MAX_CHUNKS: 420,
	POLL_INTERVAL_MS: 3_000,
	/** One waiting step holds its invocation at most this long. */
	MAX_POLL_MS_PER_STEP: 120_000,
	/** Consecutive empty waiting steps before giving up (~8 minutes). */
	MAX_IDLE_STEPS: 4,
	/** Attempts per chunk before it is skipped so one poison chunk can never
	 * wedge the loop or burn retries forever. */
	MAX_CHUNK_ATTEMPTS: 2,
} as const;

export type LiveChunkDecision =
	| { action: "wait" }
	| { action: "no-audio" }
	| { action: "done" }
	| {
			action: "chunk";
			entries: NormalizedSegmentEntry[];
			startMs: number;
			durationMs: number;
	  };

/**
 * Choose the next contiguous run of unprocessed audio segments. Segments
 * after an index gap are never taken (a missing upload would silently shift
 * every later word), so a mid-recording gap pauses live transcription and a
 * gap in a completed manifest ends it.
 */
export function planNextLiveChunk(options: {
	manifest: Video.SegmentManifestType;
	lastProcessedIndex: number;
	targetSeconds: number;
	maxTakeSeconds?: number;
}): LiveChunkDecision {
	const { manifest, lastProcessedIndex, targetSeconds } = options;
	const maxTakeSeconds =
		options.maxTakeSeconds ?? LIVE_TRANSCRIBE.MAX_CHUNK_TAKE_SECONDS;

	const plan = planSegmentsAudioExtraction(manifest, {
		requireComplete: false,
	});
	if (plan.status === "no-audio") {
		return manifest.is_complete ? { action: "no-audio" } : { action: "wait" };
	}
	if (plan.status !== "ok") {
		return { action: "wait" };
	}

	let startMs = 0;
	const pending: NormalizedSegmentEntry[] = [];
	let expectedIndex = plan.entries[0]?.index === 0 ? 0 : 1;

	for (const entry of plan.entries) {
		if (entry.index !== expectedIndex) break;
		expectedIndex++;

		if (entry.index <= lastProcessedIndex) {
			startMs += entry.duration * 1000;
		} else {
			pending.push(entry);
		}
	}

	if (pending.length === 0) {
		return manifest.is_complete ? { action: "done" } : { action: "wait" };
	}

	const taken: NormalizedSegmentEntry[] = [];
	let takenSeconds = 0;
	for (const entry of pending) {
		if (taken.length > 0 && takenSeconds + entry.duration > maxTakeSeconds) {
			break;
		}
		taken.push(entry);
		takenSeconds += entry.duration;
		if (takenSeconds >= maxTakeSeconds) break;
	}

	const tookEverything = taken.length === pending.length;
	if (!manifest.is_complete && tookEverything && takenSeconds < targetSeconds) {
		return { action: "wait" };
	}

	return {
		action: "chunk",
		entries: taken,
		startMs: Math.round(startMs),
		durationMs: Math.round(takenSeconds * 1000),
	};
}

export interface LiveChunkWordInput {
	text?: unknown;
	start?: unknown;
	end?: unknown;
	confidence?: unknown;
	speaker?: unknown;
}

/** Place a chunk's AssemblyAI words (chunk-relative ms) on the recording
 * timeline. Words outside the chunk bounds are clamped, invalid ones dropped. */
export function offsetChunkWords(
	words: readonly LiveChunkWordInput[] | null | undefined,
	chunkStartMs: number,
	chunkDurationMs: number,
): EditTranscriptWord[] {
	const result: EditTranscriptWord[] = [];

	for (const [index, word] of (words ?? []).entries()) {
		const text = typeof word.text === "string" ? word.text.trim() : "";
		const start = typeof word.start === "number" ? word.start : null;
		const end = typeof word.end === "number" ? word.end : null;
		if (!text || start === null || end === null || !Number.isFinite(start)) {
			continue;
		}

		const clamp = (value: number) =>
			Math.min(Math.max(Math.round(value), 0), chunkDurationMs);
		const startMs = chunkStartMs + clamp(start);
		const endMs = chunkStartMs + Math.max(clamp(end), clamp(start));

		result.push({
			id: `live-${chunkStartMs}-${index}`,
			text,
			startMs,
			endMs,
			confidence: typeof word.confidence === "number" ? word.confidence : null,
			speaker: typeof word.speaker === "string" ? word.speaker : null,
			channel: null,
		});
	}

	return result;
}

/**
 * AssemblyAI reports speech-free audio as a transcript error when language
 * detection is enabled. For chunked live transcription that's a valid empty
 * chunk (pauses in narration are normal), not a failure to retry.
 */
export function isNoSpokenAudioError(transcript: {
	status: string;
	error?: string | null;
}): boolean {
	return (
		transcript.status === "error" &&
		/no spoken audio/i.test(transcript.error ?? "")
	);
}

export const LIVE_TRANSCRIPT_VERSION = 1;

export type LiveTranscriptState = "active" | "complete" | "stopped";

export interface LiveTranscriptArtifact {
	version: typeof LIVE_TRANSCRIPT_VERSION;
	state: LiveTranscriptState;
	languageCode: string | null;
	lastAudioSegmentIndex: number;
	transcribedDurationMs: number;
	words: EditTranscriptWord[];
	vtt: string;
	updatedAt: string;
	/** A chunk was skipped after repeated failures, so some speech is missing.
	 * Disqualifies the artifact from being promoted to the canonical
	 * transcript; the full-pass fallback covers the recording instead. */
	hasGaps?: boolean;
}

export function getLiveTranscriptObjectKey(ownerId: string, videoId: string) {
	return `${ownerId}/${videoId}/transcription.live.json`;
}

/** Sentinel for "nothing processed yet": 0 is a legitimate segment index in
 * 0-based manifests, so the cursor starts below every real index. */
export const LIVE_TRANSCRIPT_NO_SEGMENTS = -1;

export function createEmptyLiveTranscript(
	nowIso: string,
): LiveTranscriptArtifact {
	return {
		version: LIVE_TRANSCRIPT_VERSION,
		state: "active",
		languageCode: null,
		lastAudioSegmentIndex: LIVE_TRANSCRIPT_NO_SEGMENTS,
		transcribedDurationMs: 0,
		words: [],
		vtt: editTranscriptWordsToCaptionVtt([]),
		updatedAt: nowIso,
		hasGaps: false,
	};
}

export function parseLiveTranscript(
	value: string,
): LiveTranscriptArtifact | null {
	try {
		const parsed = JSON.parse(value) as Partial<LiveTranscriptArtifact>;
		if (
			parsed?.version !== LIVE_TRANSCRIPT_VERSION ||
			!Array.isArray(parsed.words) ||
			typeof parsed.vtt !== "string" ||
			typeof parsed.lastAudioSegmentIndex !== "number" ||
			typeof parsed.transcribedDurationMs !== "number"
		) {
			return null;
		}
		return {
			version: LIVE_TRANSCRIPT_VERSION,
			state:
				parsed.state === "complete" || parsed.state === "stopped"
					? parsed.state
					: "active",
			languageCode:
				typeof parsed.languageCode === "string" ? parsed.languageCode : null,
			lastAudioSegmentIndex: parsed.lastAudioSegmentIndex,
			transcribedDurationMs: parsed.transcribedDurationMs,
			words: parsed.words,
			vtt: parsed.vtt,
			updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "",
			hasGaps: parsed.hasGaps === true,
		};
	} catch {
		return null;
	}
}

/**
 * Whether the live transcript fully and faithfully covers the recording, so
 * it can be promoted to the canonical transcript instead of paying for a
 * second full transcription pass. Anything short of complete gap-free
 * coverage means the full-pass fallback runs exactly as before.
 */
export function canPromoteLiveTranscript(
	artifact: LiveTranscriptArtifact,
	manifest: Video.SegmentManifestType,
): { ok: true } | { ok: false; reason: string } {
	if (!manifest.is_complete) {
		return { ok: false, reason: "manifest is not complete" };
	}
	if (artifact.hasGaps) {
		return { ok: false, reason: "live transcript has skipped chunks" };
	}

	const plan = planSegmentsAudioExtraction(manifest);
	if (plan.status === "no-audio") {
		return { ok: false, reason: "recording has no audio" };
	}
	if (plan.status !== "ok") {
		return { ok: false, reason: plan.reason };
	}

	const lastIndex = plan.entries[plan.entries.length - 1]?.index;
	if (lastIndex === undefined || artifact.lastAudioSegmentIndex !== lastIndex) {
		return {
			ok: false,
			reason: `covered up to segment ${artifact.lastAudioSegmentIndex} of ${lastIndex}`,
		};
	}

	// A manifest index gap means audio the recording had but we never saw.
	let expected = plan.entries[0]?.index ?? 1;
	for (const entry of plan.entries) {
		if (entry.index !== expected) {
			return { ok: false, reason: `segment gap at index ${expected}` };
		}
		expected++;
	}

	return { ok: true };
}

/**
 * Shape the accumulated live words as a canonical edit transcript (v3). The
 * caption VTT derives from the same words via editTranscriptWordsToCaptionVtt,
 * exactly as the full-pass path does.
 */
export function liveTranscriptToEditTranscript(
	artifact: LiveTranscriptArtifact,
	speechModelUsed: string,
): EditTranscript {
	return {
		version: EDIT_TRANSCRIPT_VERSION,
		speechModelUsed,
		durationMs: Math.max(0, Math.round(artifact.transcribedDurationMs)),
		languageCode: artifact.languageCode,
		words: artifact.words,
	};
}

/**
 * Merge one transcribed chunk into the artifact. Idempotent per chunk range:
 * a retried chunk first evicts any words at or after its start.
 */
export function applyChunkToLiveTranscript(
	artifact: LiveTranscriptArtifact,
	chunk: {
		startMs: number;
		durationMs: number;
		lastAudioSegmentIndex: number;
		words: EditTranscriptWord[];
		languageCode: string | null;
		nowIso: string;
	},
): LiveTranscriptArtifact {
	const words = [
		...artifact.words.filter((word) => word.startMs < chunk.startMs),
		...chunk.words,
	];

	return {
		...artifact,
		languageCode: artifact.languageCode ?? chunk.languageCode,
		lastAudioSegmentIndex: Math.max(
			artifact.lastAudioSegmentIndex,
			chunk.lastAudioSegmentIndex,
		),
		transcribedDurationMs: Math.max(
			artifact.transcribedDurationMs,
			chunk.startMs + chunk.durationMs,
		),
		words,
		vtt: editTranscriptWordsToCaptionVtt(words),
		updatedAt: chunk.nowIso,
	};
}