AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(platform): add brain dump STT quality gate and eval corpus (#13792)
Abhimanyu Yadava3f5677724b4
diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/EVAL.md b/autogpt_platform/backend/backend/api/features/onboarding_dump/EVAL.mdindex d37a186e3c..66c6e0cd10 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/EVAL.md+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/EVAL.md@@ -41,6 +41,57 @@ single-request byte cap also need `ffmpeg` on `PATH`.roughly 3 hours of audio per run. Run it before a transcription change ships,not on every commit.+## Quality-gate scoring++A corpus can also score the quality gate (`quality.py`) by shipping a+`gate_manifest.json` next to the audio:++```json+{"stt-wer-clean-01": "pass", "gate-reject-silence": "reject"}+```++Every scored file's machine transcript is pushed through the real+`quality.check_transcript_quality`, and manifest entries **without** a+`.txt` reference (silence, noise) are transcribed just to see what the STT+hallucinates — the gate must reject the result, whatever it is. One wrong+verdict fails the run, exactly like a WER breach: a garbage dump that+personalizes and a real dump that bounces are both release blockers. The+LLM half of the gate needs the usual chat key (`OPEN_ROUTER_API_KEY` or an+OpenAI key) on top of the STT key.++## The committed baseline corpus++`eval_data/` in this directory is a ready-to-run baseline corpus+(~3 MB of ogg/opus — audio, `.txt` references and the gate manifest):++```bash+poetry run brain-dump-eval --dir backend/api/features/onboarding_dump/eval_data+```++It is built from Mini LibriSpeech (LibriVox, public domain, exact+reference transcripts — no privacy concern, which is why this corpus *can*+be committed). Every filename states what the file tests:++- `stt-wer-clean-01..13` — WER on clean speech, the accuracy baseline+- `stt-wer-moderate-noise-15db-01..06` — WER with background noise mixed+ at a measured 15 dB SNR (fan / quiet room)+- `stt-wer-loud-noise-5db-01..06` — WER with loud noise at 5 dB SNR+ (noise nearly as loud as the voice)+- `gate-reject-{silence,white-noise,pink-noise,brown-noise}` — no speech+ at all; the quality gate must reject whatever the STT hallucinates++All 25 speech clips must also *pass* the gate. Regenerate or vary the+corpus with:++```bash+poetry run python -m backend.api.features.onboarding_dump.eval_corpus \+ --out backend/api/features/onboarding_dump/eval_data+```++Read speech scores far better than rambling speech, so treat its WER as a+floor check, not the release answer — the real-dump corpus below stays+private and uncommitted.+## The corpusPut the recordings and their reference transcripts in one flat directory,diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_eval.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_eval.pyindex 1fb902813b..afd321c21f 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_eval.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_eval.py@@ -19,7 +19,8 @@ from pathlib import Pathfrom pydantic import BaseModel-from backend.api.features.onboarding_dump import transcription+from backend.api.features.onboarding_dump import brain_dump_gate_eval, transcription+from backend.api.features.onboarding_dump.brain_dump_gate_eval import GateReportfrom backend.api.features.onboarding_dump.brain_dump_wer import (WordErrors,compute_word_errors,@@ -68,6 +69,10 @@ class FileResult(BaseModel):segment_count: int = 0seams: list[Seam] = []error: str | None = None+ transcript: str = ""+ """The full machine transcript — kept so the quality gate can be scored+ on exactly what the pipeline produced, and so a bad WER row can be+ diagnosed from the JSON report without re-running the corpus."""class EvalReport(BaseModel):@@ -80,6 +85,9 @@ class EvalReport(BaseModel):passed: boolaudio_without_reference: list[str]reference_without_audio: list[str]+ quality_gate: GateReport | None = None+ """Present when the corpus ships a ``gate_manifest.json`` — every+ verdict must match it for the run to pass."""class DumpPair(BaseModel):@@ -133,9 +141,15 @@ def main() -> None:async def run_eval(directory: Path, duration_secs: float | None = None) -> EvalReport:discovery = discover_pairs(directory)+ manifest = brain_dump_gate_eval.load_manifest(directory)started = time.monotonic()results = [await evaluate_pair(pair, duration_secs) for pair in discovery.pairs]scored = [result for result in results if result.error is None]+ quality_gate = (+ await _score_quality_gate(directory, discovery, scored, manifest)+ if manifest is not None+ else None+ )pooled = WordErrors(substitutions=sum(r.errors.substitutions for r in scored),insertions=sum(r.errors.insertions for r in scored),@@ -152,9 +166,42 @@ async def run_eval(directory: Path, duration_secs: float | None = None) -> EvalRgate=WER_RELEASE_GATE,passed=bool(scored)and len(scored) == len(results)- and pooled.wer < WER_RELEASE_GATE,- audio_without_reference=discovery.audio_without_reference,+ and pooled.wer < WER_RELEASE_GATE+ and (quality_gate is None or quality_gate.passed),+ # Reference-less audio that the gate manifest claims is deliberate+ # (silence, noise) is the gate's input, not a corpus mistake.+ audio_without_reference=[+ name+ for name in discovery.audio_without_reference+ if manifest is None or name not in manifest+ ],reference_without_audio=discovery.reference_without_audio,+ quality_gate=quality_gate,+ )+++async def _score_quality_gate(+ directory: Path,+ discovery: PairDiscovery,+ scored: list[FileResult],+ manifest: dict[str, brain_dump_gate_eval.GateExpectation],+) -> GateReport:+ """Run the real quality gate over everything the manifest covers."""+ results = [+ await brain_dump_gate_eval.evaluate_transcript(+ result.name, manifest.get(result.name, "pass"), result.transcript+ )+ for result in scored+ ]+ gate_only = brain_dump_gate_eval.discover_gate_only(+ directory, manifest, {pair.name for pair in discovery.pairs}, AUDIO_EXTENSIONS+ )+ results += [+ await brain_dump_gate_eval.evaluate_gate_only(path, manifest[path.stem])+ for path in gate_only+ ]+ return brain_dump_gate_eval.build_report(+ results, [result.name for result in scored], manifest)@@ -189,6 +236,7 @@ async def evaluate_pair(model=run.model,segment_count=run.segment_count,seams=run.seams,+ transcript=run.transcript,)@@ -275,13 +323,18 @@ def render_report(report: EvalReport) -> str:lines = ["",f"Brain-dump transcription WER — {len(report.files)} file(s)",- "",- f"{'file':<20}{'WER':>9}{'sub':>6}{'ins':>6}{'del':>6}"- f"{'ref':>8}{'secs':>8} {'segments':<10}model",]- lines.extend(_render_row(result) for result in report.files)+ lines.extend(+ brain_dump_gate_eval.render_table(+ ["file", "WER", "sub", "ins", "del", "ref", "secs", "segments", "model"],+ [_render_row(result) for result in report.files],+ right_align=frozenset({1, 2, 3, 4, 5, 6}),+ )+ )lines.extend(_render_seams(report))lines.extend(_render_mismatches(report))+ if report.quality_gate is not None:+ lines.extend(brain_dump_gate_eval.render_gate_report(report.quality_gate))lines.extend(["",@@ -298,15 +351,20 @@ def render_report(report: EvalReport) -> str:return "\n".join(lines)-def _render_row(result: FileResult) -> str:+def _render_row(result: FileResult) -> list[str]:if result.error:- return f"{result.name:<20}{'ERROR':>9} {result.error[:60]}"- return (- f"{result.name:<20}{result.errors.wer:>9.2%}"- f"{result.errors.substitutions:>6}{result.errors.insertions:>6}"- f"{result.errors.deletions:>6}{result.errors.reference_words:>8}"- f"{result.latency_secs:>8.1f} {result.segment_count:<10}{result.model}"- )+ return [result.name, "ERROR", "", "", "", "", "", "", result.error[:60]]+ return [+ result.name,+ f"{result.errors.wer:.2%}",+ str(result.errors.substitutions),+ str(result.errors.insertions),+ str(result.errors.deletions),+ str(result.errors.reference_words),+ f"{result.latency_secs:.1f}",+ str(result.segment_count),+ result.model,+ ]def _render_seams(report: EvalReport) -> list[str]:diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval.pynew file mode 100644index 0000000000..3b0296a303--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval.py@@ -0,0 +1,194 @@+"""Quality-gate scoring for the brain-dump eval corpus.++The WER harness answers "did the STT hear the words?"; this answers "did+the quality gate make the right call on the result?". A corpus opts in by+shipping a ``gate_manifest.json`` next to the audio:++ {"dump01-clean": "pass", "garbage-silence": "reject", ...}++Every scored WER file's transcript is pushed through the real+``quality.check_transcript_quality``, and manifest entries with no ``.txt``+reference (silence, noise) are transcribed just to see what the STT+hallucinates — the gate must reject it, whatever it is. One wrong verdict+fails the whole eval run: a garbage dump that personalizes, or a real dump+that bounces, are both release blockers.+"""++import logging+from pathlib import Path+from typing import Literal++from pydantic import BaseModel, TypeAdapter++from backend.api.features.onboarding_dump import quality, transcription++logger = logging.getLogger(__name__)++GATE_MANIFEST_NAME = "gate_manifest.json"+TRANSCRIPT_PREVIEW_CHARS = 80++GateExpectation = Literal["pass", "reject"]+++class GateFileResult(BaseModel):+ name: str+ expected: GateExpectation+ outcome: GateExpectation+ error_code: str | None = None+ correct: bool+ transcript_preview: str = ""+ transcription_error: str | None = None+++class GateReport(BaseModel):+ results: list[GateFileResult]+ passed: bool+ unlisted: list[str] = []+ """Scored audio files the manifest forgot — reported so a new clip+ can't silently skip the gate check."""+++def load_manifest(directory: Path) -> dict[str, GateExpectation] | None:+ path = directory / GATE_MANIFEST_NAME+ if not path.exists():+ return None+ return TypeAdapter(dict[str, GateExpectation]).validate_json(+ path.read_text(encoding="utf-8")+ )+++def build_report(+ results: list[GateFileResult],+ scored_names: list[str],+ manifest: dict[str, GateExpectation],+) -> GateReport:+ unlisted = sorted(set(scored_names) - set(manifest))+ return GateReport(+ results=results,+ unlisted=unlisted,+ passed=all(result.correct for result in results) and not unlisted,+ )+++async def evaluate_transcript(+ name: str, expected: GateExpectation, transcript: str+) -> GateFileResult:+ error_code = await quality.check_transcript_quality(transcript)+ outcome: GateExpectation = "pass" if error_code is None else "reject"+ return GateFileResult(+ name=name,+ expected=expected,+ outcome=outcome,+ error_code=error_code,+ correct=outcome == expected,+ transcript_preview=transcript.strip()[:TRANSCRIPT_PREVIEW_CHARS],+ )+++async def evaluate_gate_only(+ audio_path: Path, expected: GateExpectation+) -> GateFileResult:+ """Transcribe a reference-less file (silence, noise) and gate the result.++ A transcription failure counts as an empty transcript rather than an+ eval error: in production that request ends in a non-personalized+ ``failed`` status, which is the same protection the gate provides —+ but it is still recorded, because "the STT errored" and "the gate+ caught a hallucination" are different findings.+ """+ transcription_error: str | None = None+ try:+ result = await transcription.transcribe(+ audio_path.read_bytes(), audio_path.name+ )+ transcript = result.text+ except (+ transcription.TranscriptionUnavailableError,+ transcription.TranscriptionFailedError,+ OSError,+ ) as e:+ transcription_error = str(e)+ transcript = ""+ logger.warning("Gate-only transcription failed for %s: %s", audio_path.name, e)+ scored = await evaluate_transcript(audio_path.stem, expected, transcript)+ return scored.model_copy(update={"transcription_error": transcription_error})+++def discover_gate_only(+ directory: Path,+ manifest: dict[str, GateExpectation],+ paired_names: set[str],+ audio_extensions: tuple[str, ...],+) -> list[Path]:+ """Manifest entries that have audio but no ``.txt`` reference."""+ return sorted(+ path+ for path in directory.iterdir()+ if path.suffix.lower() in audio_extensions+ and path.stem in manifest+ and path.stem not in paired_names+ )+++def render_gate_report(report: GateReport) -> list[str]:+ rows = [+ [+ result.name,+ result.expected,+ result.outcome,+ "ok" if result.correct else "WRONG",+ _gate_detail(result),+ ]+ for result in report.results+ ]+ lines = [+ "",+ f"Quality gate — {len(report.results)} file(s)",+ *render_table(["file", "expected", "outcome", "result", "detail"], rows),+ ]+ if report.unlisted:+ lines.append(f"Not in {GATE_MANIFEST_NAME}: {', '.join(report.unlisted)}")+ lines.append(+ "Gate verdicts must all match the manifest — "+ + ("PASS" if report.passed else "FAIL")+ )+ return lines+++def _gate_detail(result: GateFileResult) -> str:+ detail = result.error_code or result.transcript_preview or "(empty transcript)"+ if result.transcription_error:+ detail += f" [stt error: {result.transcription_error[:40]}]"+ return detail+++def render_table(+ headers: list[str], rows: list[list[str]], right_align: frozenset[int] = frozenset()+) -> list[str]:+ """Box-drawn table with column widths sized to the content.++ Shared by the WER and quality-gate report sections so both stay+ readable however long the corpus filenames get.+ """+ widths = [+ max(len(header), *(len(row[column]) for row in rows)) if rows else len(header)+ for column, header in enumerate(headers)+ ]++ def line(cells: list[str]) -> str:+ padded = [+ cell.rjust(widths[i]) if i in right_align else cell.ljust(widths[i])+ for i, cell in enumerate(cells)+ ]+ return "│ " + " │ ".join(padded) + " │"++ def rule(left: str, mid: str, right: str) -> str:+ return left + mid.join("─" * (width + 2) for width in widths) + right++ return [+ rule("┌", "┬", "┐"),+ line(headers),+ rule("├", "┼", "┤"),+ *[line(row) for row in rows],+ rule("└", "┴", "┘"),+ ]diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval_test.pynew file mode 100644index 0000000000..2f79b774ca--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/brain_dump_gate_eval_test.py@@ -0,0 +1,232 @@+"""Tests for the quality-gate half of the brain-dump eval.++The contract: a corpus with a ``gate_manifest.json`` gets every transcript+gated with the real classifier interface (mocked here), garbage files with+no reference are transcribed just for the gate, and one wrong verdict —+or a file the manifest forgot — fails the whole run.+"""++import json+from pathlib import Path+from unittest.mock import AsyncMock++import pytest+from pydantic import ValidationError++from backend.api.features.onboarding_dump import (+ brain_dump_eval,+ brain_dump_gate_eval,+ quality,+ transcription,+)+from backend.api.features.onboarding_dump.transcription import TranscriptionResult++GOOD_TRANSCRIPT = "I run a bakery and want the weekly order emails handled."+GARBAGE_TRANSCRIPT = "Thank you for watching."+++def _result(text: str) -> TranscriptionResult:+ return TranscriptionResult(text=text, model="gpt-4o-transcribe")+++def _write_manifest(directory: Path, entries: dict[str, str]) -> None:+ (directory / brain_dump_gate_eval.GATE_MANIFEST_NAME).write_text(+ json.dumps(entries), encoding="utf-8"+ )+++def _gate_by_transcript(monkeypatch) -> None:+ """A stand-in gate: rejects the known garbage transcript, passes the rest."""++ async def check(transcript: str) -> str | None:+ return "insufficient_content" if transcript == GARBAGE_TRANSCRIPT else None++ monkeypatch.setattr(quality, "check_transcript_quality", check)+++def test_load_manifest_returns_none_without_a_file(tmp_path: Path):+ assert brain_dump_gate_eval.load_manifest(tmp_path) is None+++def test_load_manifest_rejects_unknown_expectations(tmp_path: Path):+ _write_manifest(tmp_path, {"a": "pass", "b": "reject", "c": "maybe"})+ with pytest.raises(ValidationError):+ brain_dump_gate_eval.load_manifest(tmp_path)+++def test_load_manifest_accepts_valid_entries(tmp_path: Path):+ _write_manifest(tmp_path, {"a": "pass", "b": "reject"})+ assert brain_dump_gate_eval.load_manifest(tmp_path) == {"a": "pass", "b": "reject"}+++async def test_run_eval_gates_scored_transcripts_and_garbage_audio(+ tmp_path: Path, monkeypatch+):+ (tmp_path / "dump01.webm").write_bytes(b"audio")+ (tmp_path / "dump01.txt").write_text(GOOD_TRANSCRIPT, encoding="utf-8")+ (tmp_path / "garbage-silence.wav").write_bytes(b"audio")+ _write_manifest(tmp_path, {"dump01": "pass", "garbage-silence": "reject"})+ transcribe = AsyncMock(+ side_effect=[_result(GOOD_TRANSCRIPT), _result(GARBAGE_TRANSCRIPT)]+ )+ monkeypatch.setattr(transcription, "transcribe", transcribe)+ _gate_by_transcript(monkeypatch)++ report = await brain_dump_eval.run_eval(tmp_path)++ assert report.passed is True+ assert report.quality_gate is not None+ assert report.quality_gate.passed is True+ by_name = {result.name: result for result in report.quality_gate.results}+ assert by_name["dump01"].outcome == "pass"+ assert by_name["garbage-silence"].outcome == "reject"+ assert by_name["garbage-silence"].error_code == "insufficient_content"+ # The garbage clip is the gate's input, not a corpus mistake.+ assert report.audio_without_reference == []+ # Both files hit the pipeline: one for WER, one gate-only.+ assert transcribe.await_count == 2+++async def test_run_eval_fails_when_a_good_dump_is_rejected(tmp_path: Path, monkeypatch):+ (tmp_path / "dump01.webm").write_bytes(b"audio")+ (tmp_path / "dump01.txt").write_text(GARBAGE_TRANSCRIPT, encoding="utf-8")+ _write_manifest(tmp_path, {"dump01": "pass"})+ monkeypatch.setattr(+ transcription, "transcribe", AsyncMock(return_value=_result(GARBAGE_TRANSCRIPT))+ )+ _gate_by_transcript(monkeypatch)++ report = await brain_dump_eval.run_eval(tmp_path)++ # WER is 0% — only the gate verdict is wrong, and that alone fails.+ assert report.pooled.wer == 0.0+ assert report.quality_gate is not None+ assert report.quality_gate.passed is False+ assert report.passed is False+ assert "WRONG" in brain_dump_eval.render_report(report)+++async def test_run_eval_fails_when_garbage_slips_through_the_gate(+ tmp_path: Path, monkeypatch+):+ (tmp_path / "garbage-noise.wav").write_bytes(b"audio")+ (tmp_path / "dump01.webm").write_bytes(b"audio")+ (tmp_path / "dump01.txt").write_text(GOOD_TRANSCRIPT, encoding="utf-8")+ _write_manifest(tmp_path, {"dump01": "pass", "garbage-noise": "reject"})+ # The STT hallucinates something the gate then (wrongly) accepts.+ transcribe = AsyncMock(+ side_effect=[_result(GOOD_TRANSCRIPT), _result(GOOD_TRANSCRIPT)]+ )+ monkeypatch.setattr(transcription, "transcribe", transcribe)+ _gate_by_transcript(monkeypatch)++ report = await brain_dump_eval.run_eval(tmp_path)++ assert report.quality_gate is not None+ assert report.quality_gate.passed is False+ assert report.passed is False+++async def test_run_eval_flags_scored_files_missing_from_the_manifest(+ tmp_path: Path, monkeypatch+):+ (tmp_path / "dump01.webm").write_bytes(b"audio")+ (tmp_path / "dump01.txt").write_text(GOOD_TRANSCRIPT, encoding="utf-8")+ _write_manifest(tmp_path, {"something-else": "pass"})+ monkeypatch.setattr(+ transcription, "transcribe", AsyncMock(return_value=_result(GOOD_TRANSCRIPT))+ )+ _gate_by_transcript(monkeypatch)++ report = await brain_dump_eval.run_eval(tmp_path)++ assert report.quality_gate is not None+ assert report.quality_gate.unlisted == ["dump01"]+ assert report.quality_gate.passed is False+ assert report.passed is False+++async def test_run_eval_without_a_manifest_skips_the_gate(tmp_path: Path, monkeypatch):+ (tmp_path / "dump01.webm").write_bytes(b"audio")+ (tmp_path / "dump01.txt").write_text(GOOD_TRANSCRIPT, encoding="utf-8")+ monkeypatch.setattr(+ transcription, "transcribe", AsyncMock(return_value=_result(GOOD_TRANSCRIPT))+ )+ gate = AsyncMock()+ monkeypatch.setattr(quality, "check_transcript_quality", gate)++ report = await brain_dump_eval.run_eval(tmp_path)++ assert report.quality_gate is None+ assert report.passed is True+ gate.assert_not_awaited()+++async def test_gate_only_transcription_failure_counts_as_empty_transcript(+ tmp_path: Path, monkeypatch+):+ """An STT error on garbage audio still ends non-personalized in+ production, so the eval scores the gate on the empty transcript — but+ records the error, because the two findings are different."""+ audio = tmp_path / "garbage-silence.wav"+ audio.write_bytes(b"audio")+ monkeypatch.setattr(+ transcription,+ "transcribe",+ AsyncMock(+ side_effect=transcription.TranscriptionFailedError("audio too short")+ ),+ )++ async def check(transcript: str) -> str | None:+ return None if transcript else "no_usable_speech"++ monkeypatch.setattr(quality, "check_transcript_quality", check)++ result = await brain_dump_gate_eval.evaluate_gate_only(audio, "reject")++ assert result.outcome == "reject"+ assert result.correct is True+ assert result.transcription_error == "audio too short"+++ "expected,outcome_correct", [("pass", True), ("reject", False)]+)+async def test_evaluate_transcript_compares_outcome_to_expectation(+ monkeypatch, expected: str, outcome_correct: bool+):+ _gate_by_transcript(monkeypatch)++ result = await brain_dump_gate_eval.evaluate_transcript(+ "dump01", expected, GOOD_TRANSCRIPT+ )++ assert result.outcome == "pass"+ assert result.correct is outcome_correct+++# --- shipped corpus consistency (pure filesystem, no credentials) ---------++EVAL_DATA_DIR = Path(__file__).parent / "eval_data"+++def test_shipped_corpus_matches_its_manifest():+ """Corpus drift (renamed/removed audio, a clip the manifest forgot,+ a reference file on a garbage clip) must fail CI without needing an+ LLM or STT credential."""+ manifest = brain_dump_gate_eval.load_manifest(EVAL_DATA_DIR)+ assert manifest, "shipped corpus must carry a gate manifest"++ audio = {path.stem for path in EVAL_DATA_DIR.glob("*.ogg")}+ references = {path.stem for path in EVAL_DATA_DIR.glob("*.txt")}++ assert set(manifest) == audio, "manifest keys and audio files must match 1:1"+ assert references <= audio, "reference files must match an audio file"+ for stem in sorted(audio):+ if stem.startswith("gate-reject-"):+ assert manifest[stem] == "reject"+ assert stem not in references, f"{stem} must not carry a reference"+ else:+ assert manifest[stem] == "pass"+ assert stem in references, f"{stem} needs a .txt reference"diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_corpus.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_corpus.pynew file mode 100644index 0000000000..a9bd7ec3a8--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_corpus.py@@ -0,0 +1,357 @@+"""Build a brain-dump eval corpus from public-domain speech.++Downloads Mini LibriSpeech (LibriVox recordings, public domain — exact+per-utterance reference transcripts included), stitches consecutive+utterances into ~1-minute clips, and writes the audio + ``.txt`` pairs the+WER harness (``brain_dump_eval.py``) consumes.++The corpus is deliberately mixed, per the eval plan:++- clean clips — the "very good" baseline;+- clips remixed with pink/white noise at moderate (15 dB SNR) and loud+ (5 dB SNR) levels, so the report shows how far accuracy degrades with a+ fan, a street, a café;+- garbage clips (silence, pure noise) with **no** reference text — these+ exist for the quality gate, which must reject whatever the STT+ hallucinates from them. Expectations land in ``gate_manifest.json``.++Read speech is not rambling speech — treat the numbers as a floor check,+not a substitute for the real-dump corpus EVAL.md asks for.++Usage (from ``autogpt_platform/backend``):++ poetry run python -m backend.api.features.onboarding_dump.eval_corpus \+ --out /tmp/brain-dump-eval-corpus+"""++import argparse+import json+import re+import shutil+import subprocess+import tarfile+import tempfile+import urllib.request+from pathlib import Path++from pydantic import BaseModel++from backend.api.features.onboarding_dump.brain_dump_gate_eval import GATE_MANIFEST_NAME++SOURCE_URL = "https://www.openslr.org/resources/31/dev-clean-2.tar.gz"+CACHE_DIR = Path.home() / ".cache" / "brain-dump-eval"++TOTAL_CLIPS = 25+# words per clip before it is closed out — ~110 words of read speech is+# roughly 45-70 seconds, comfortably past the gate's clear-pass threshold.+MIN_CLIP_WORDS = 110+MAX_CLIPS_PER_CHAPTER = 2++# clean / moderate-noise / loud-noise split over the 25 speech clips.+MODERATE_NOISE_CLIPS = 6+LOUD_NOISE_CLIPS = 6+MODERATE_SNR_DB = 15.0+LOUD_SNR_DB = 5.0+++# Every filename states what the file tests. The speech clips measure STT+# accuracy (WER) at a given noise level and must pass the quality gate;+# the ``gate-reject-*`` clips contain no speech at all and exist purely to+# prove the gate rejects whatever the STT hallucinates from them.+CLEAN_LABEL = "stt-wer-clean"+MODERATE_LABEL = "stt-wer-moderate-noise-15db"+LOUD_LABEL = "stt-wer-loud-noise-5db"+GARBAGE_CLIPS = (+ ("gate-reject-silence", "anullsrc=r=16000:cl=mono", 8),+ (+ "gate-reject-white-noise",+ "anoisesrc=color=white:sample_rate=16000:amplitude=0.3",+ 10,+ ),+ (+ "gate-reject-pink-noise",+ "anoisesrc=color=pink:sample_rate=16000:amplitude=0.4",+ 12,+ ),+ (+ "gate-reject-brown-noise",+ "anoisesrc=color=brown:sample_rate=16000:amplitude=0.5",+ 10,+ ),+)+++class Utterance(BaseModel):+ audio_path: Path+ text: str+++class Clip(BaseModel):+ name: str+ utterances: list[Utterance]+ snr_db: float | None = None++ @property+ def reference(self) -> str:+ return " ".join(u.text for u in self.utterances)+++def main() -> None:+ parser = argparse.ArgumentParser(+ description="Assemble a mixed clean/noisy WER + quality-gate corpus."+ )+ parser.add_argument("--out", required=True, type=Path)+ parser.add_argument("--source-url", default=SOURCE_URL)+ parser.add_argument("--cache", type=Path, default=CACHE_DIR)+ args = parser.parse_args()++ if shutil.which("ffmpeg") is None:+ raise SystemExit("ffmpeg is required on PATH")++ extracted = download_and_extract(args.source_url, args.cache)+ clips = plan_clips(collect_chapters(extracted))+ args.out.mkdir(parents=True, exist_ok=True)++ manifest: dict[str, str] = {}+ for clip in clips:+ build_clip(clip, args.out)+ manifest[clip.name] = "pass"+ print(f" {clip.name}.ogg ({len(clip.reference.split())} ref words)")+ for name, source, seconds in GARBAGE_CLIPS:+ build_garbage_clip(args.out / f"{name}.ogg", source, seconds)+ manifest[name] = "reject"+ print(f" {name}.ogg (gate must reject)")++ (args.out / GATE_MANIFEST_NAME).write_text(+ json.dumps(manifest, indent=2) + "\n", encoding="utf-8"+ )+ print(+ f"\n{len(clips)} speech clips + {len(GARBAGE_CLIPS)} garbage clips "+ f"in {args.out}\nRun: poetry run brain-dump-eval --dir {args.out}"+ )+++def download_and_extract(url: str, cache: Path) -> Path:+ cache.mkdir(parents=True, exist_ok=True)+ tarball = cache / url.rsplit("/", 1)[-1]+ if not tarball.exists():+ print(f"Downloading {url} ...")+ with urllib.request.urlopen(url) as response, open(tarball, "wb") as out:+ shutil.copyfileobj(response, out)+ root = cache / "extracted"+ if not root.exists():+ print(f"Extracting {tarball.name} ...")+ with tarfile.open(tarball) as archive:+ archive.extractall(root, members=_safe_members(archive, root))+ return root+++def _safe_members(archive: tarfile.TarFile, root: Path) -> list[tarfile.TarInfo]:+ """Regular files that resolve inside ``root`` — nothing else.++ Explicit validation instead of ``filter="data"``: the filter argument+ only exists on Python 3.11.4+ while this package supports 3.10, and+ the corpus needs no links, devices or directories anyway.+ """+ resolved_root = root.resolve()+ members = []+ for member in archive.getmembers():+ if not member.isreg():+ continue+ target = (root / member.name).resolve()+ if not target.is_relative_to(resolved_root):+ raise ValueError(f"Unsafe path in archive: {member.name}")+ members.append(member)+ return members+++def collect_chapters(root: Path) -> list[list[Utterance]]:+ """Every chapter's utterances, in spoken order.++ LibriSpeech lays a chapter out as ``<spk>/<ch>/<spk>-<ch>-NNNN.flac``+ plus one ``<spk>-<ch>.trans.txt`` of ``ID TEXT`` lines; consecutive+ utterance ids are consecutive audiobook speech, which is what makes+ concatenating them sound like one continuous take.+ """+ chapters = []+ for trans in sorted(root.rglob("*.trans.txt")):+ utterances = []+ for line in trans.read_text(encoding="utf-8").splitlines():+ utterance_id, _, text = line.partition(" ")+ flac = trans.parent / f"{utterance_id}.flac"+ if text.strip() and flac.exists():+ utterances.append(Utterance(audio_path=flac, text=text.strip()))+ if utterances:+ chapters.append(utterances)+ return chapters+++def plan_clips(chapters: list[list[Utterance]]) -> list[Clip]:+ """Fold chapters into ``TOTAL_CLIPS`` clips and assign the noise mix.++ Chapters are interleaved (at most ``MAX_CLIPS_PER_CHAPTER`` each) so+ the corpus spans many speakers rather than exhausting the first one.+ """+ groups: list[list[Utterance]] = []+ for chapter in chapters:+ taken = 0+ group: list[Utterance] = []+ for utterance in chapter:+ group.append(utterance)+ if sum(len(u.text.split()) for u in group) >= MIN_CLIP_WORDS:+ groups.append(group)+ group = []+ taken += 1+ if taken == MAX_CLIPS_PER_CHAPTER:+ break+ if len(groups) >= TOTAL_CLIPS:+ break+ if len(groups) < TOTAL_CLIPS:+ raise SystemExit(+ f"corpus source only yielded {len(groups)} clips of {TOTAL_CLIPS}"+ )++ clips = []+ counters: dict[str, int] = {}+ for index, group in enumerate(groups[:TOTAL_CLIPS]):+ snr, label = None, CLEAN_LABEL+ if index >= TOTAL_CLIPS - LOUD_NOISE_CLIPS:+ snr, label = LOUD_SNR_DB, LOUD_LABEL+ elif index >= TOTAL_CLIPS - LOUD_NOISE_CLIPS - MODERATE_NOISE_CLIPS:+ snr, label = MODERATE_SNR_DB, MODERATE_LABEL+ counters[label] = counters.get(label, 0) + 1+ clips.append(+ Clip(name=f"{label}-{counters[label]:02d}", utterances=group, snr_db=snr)+ )+ return clips+++def build_clip(clip: Clip, out_dir: Path) -> None:+ with tempfile.TemporaryDirectory() as workdir:+ clean = Path(workdir) / "clean.wav"+ concat_utterances(clip.utterances, clean)+ source = clean+ if clip.snr_db is not None:+ source = Path(workdir) / "mixed.wav"+ mix_noise(clean, source, clip.snr_db, Path(workdir))+ encode_opus(source, out_dir / f"{clip.name}.ogg")+ (out_dir / f"{clip.name}.txt").write_text(clip.reference + "\n", encoding="utf-8")+++def encode_opus(source: Path, target: Path) -> None:+ """ogg/opus at voice bitrate — what the pipeline's own splitter emits.++ Keeps a 29-clip corpus at ~4 MB instead of ~34 MB of wav, small enough+ to commit next to the harness.+ """+ _ffmpeg("-i", str(source), "-c:a", "libopus", "-b:a", "24k", str(target))+++def concat_utterances(utterances: list[Utterance], target: Path) -> None:+ list_file = target.with_suffix(".list")+ list_file.write_text(+ "".join(f"file '{u.audio_path}'\n" for u in utterances), encoding="utf-8"+ )+ _ffmpeg(+ "-f",+ "concat",+ "-safe",+ "0",+ "-i",+ str(list_file),+ "-ac",+ "1",+ "-ar",+ "16000",+ str(target),+ )+++def mix_noise(clean: Path, target: Path, snr_db: float, workdir: Path) -> None:+ """Overlay pink noise at a measured signal-to-noise ratio.++ Both RMS levels are measured with ``astats`` rather than assumed from+ generator amplitudes, so "5 dB SNR" in a filename is actually true of+ the audio.+ """+ noise = workdir / "noise.wav"+ duration = probe_duration(clean)+ _ffmpeg(+ "-f",+ "lavfi",+ "-i",+ f"anoisesrc=color=pink:sample_rate=16000:amplitude=0.3:duration={duration}",+ "-ac",+ "1",+ str(noise),+ )+ gain_db = (measure_rms_db(clean) - snr_db) - measure_rms_db(noise)+ _ffmpeg(+ "-i",+ str(clean),+ "-i",+ str(noise),+ "-filter_complex",+ f"[1:a]volume={gain_db:.2f}dB[n];"+ "[0:a][n]amix=inputs=2:duration=first:normalize=0",+ str(target),+ )+++def build_garbage_clip(target: Path, source: str, seconds: int) -> None:+ _ffmpeg(+ "-f",+ "lavfi",+ "-i",+ source,+ "-t",+ str(seconds),+ "-ac",+ "1",+ "-c:a",+ "libopus",+ "-b:a",+ "24k",+ str(target),+ )+++def probe_duration(path: Path) -> float:+ stderr = _ffmpeg_stderr("-i", str(path), "-f", "null", "-")+ match = re.search(r"Duration: (\d+):(\d+):(\d+\.\d+)", stderr)+ if match is None:+ raise SystemExit(f"could not probe duration of {path.name}")+ hours, minutes, seconds = match.groups()+ return int(hours) * 3600 + int(minutes) * 60 + float(seconds)+++def measure_rms_db(path: Path) -> float:+ stderr = _ffmpeg_stderr(+ "-i", str(path), "-af", "astats=measure_perchannel=none", "-f", "null", "-"+ )+ match = re.search(r"RMS level dB: (-?\d+\.?\d*)", stderr)+ if match is None:+ raise SystemExit(f"could not measure RMS of {path.name}")+ return float(match.group(1))+++def _ffmpeg(*args: str) -> None:+ subprocess.run(+ ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", *args],+ check=True,+ capture_output=True,+ )+++def _ffmpeg_stderr(*args: str) -> str:+ result = subprocess.run(+ ["ffmpeg", "-hide_banner", *args],+ check=False,+ capture_output=True,+ text=True,+ )+ return result.stderr+++if __name__ == "__main__":+ main()diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-brown-noise.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-brown-noise.oggnew file mode 100644index 0000000000..bfdfa6016dBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-brown-noise.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-pink-noise.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-pink-noise.oggnew file mode 100644index 0000000000..d4917d27d8Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-pink-noise.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-silence.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-silence.oggnew file mode 100644index 0000000000..cf4bdb2c31Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-silence.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-white-noise.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-white-noise.oggnew file mode 100644index 0000000000..f5c226391bBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate-reject-white-noise.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate_manifest.json b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate_manifest.jsonnew file mode 100644index 0000000000..2b56571a5b--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/gate_manifest.json@@ -0,0 +1,31 @@+{+ "stt-wer-clean-01": "pass",+ "stt-wer-clean-02": "pass",+ "stt-wer-clean-03": "pass",+ "stt-wer-clean-04": "pass",+ "stt-wer-clean-05": "pass",+ "stt-wer-clean-06": "pass",+ "stt-wer-clean-07": "pass",+ "stt-wer-clean-08": "pass",+ "stt-wer-clean-09": "pass",+ "stt-wer-clean-10": "pass",+ "stt-wer-clean-11": "pass",+ "stt-wer-clean-12": "pass",+ "stt-wer-clean-13": "pass",+ "stt-wer-moderate-noise-15db-01": "pass",+ "stt-wer-moderate-noise-15db-02": "pass",+ "stt-wer-moderate-noise-15db-03": "pass",+ "stt-wer-moderate-noise-15db-04": "pass",+ "stt-wer-moderate-noise-15db-05": "pass",+ "stt-wer-moderate-noise-15db-06": "pass",+ "stt-wer-loud-noise-5db-01": "pass",+ "stt-wer-loud-noise-5db-02": "pass",+ "stt-wer-loud-noise-5db-03": "pass",+ "stt-wer-loud-noise-5db-04": "pass",+ "stt-wer-loud-noise-5db-05": "pass",+ "stt-wer-loud-noise-5db-06": "pass",+ "gate-reject-silence": "reject",+ "gate-reject-white-noise": "reject",+ "gate-reject-pink-noise": "reject",+ "gate-reject-brown-noise": "reject"+}diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.oggnew file mode 100644index 0000000000..fcabc20dc3Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.txtnew file mode 100644index 0000000000..95c9fa2889--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-01.txt@@ -0,0 +1 @@+BECAUSE YOU WERE SLEEPING INSTEAD OF CONQUERING THE LOVELY ROSE PRINCESS HAS BECOME A FIDDLE WITHOUT A BOW WHILE POOR SHAGGY SITS THERE A COOING DOVE HE HAS GONE AND GONE FOR GOOD ANSWERED POLYCHROME WHO HAD MANAGED TO SQUEEZE INTO THE ROOM BESIDE THE DRAGON AND HAD WITNESSED THE OCCURRENCES WITH MUCH INTEREST I HAVE REMAINED A PRISONER ONLY BECAUSE I WISHED TO BE ONE AND WITH THIS HE STEPPED FORWARD AND BURST THE STOUT CHAINS AS EASILY AS IF THEY HAD BEEN THREADS THE LITTLE GIRL HAD BEEN ASLEEP BUT SHE HEARD THE RAPS AND OPENED THE DOOR THE KING HAS FLED IN DISGRACE AND YOUR FRIENDS ARE ASKING FOR YOUdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.oggnew file mode 100644index 0000000000..9a3200efb9Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.txtnew file mode 100644index 0000000000..aaee784f83--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-02.txt@@ -0,0 +1 @@+I BEGGED RUGGEDO LONG AGO TO SEND HIM AWAY BUT HE WOULD NOT DO SO I ALSO OFFERED TO HELP YOUR BROTHER TO ESCAPE BUT HE WOULD NOT GO HE EATS AND SLEEPS VERY STEADILY REPLIED THE NEW KING I HOPE HE DOESN'T WORK TOO HARD SAID SHAGGY HE DOESN'T WORK AT ALL IN FACT THERE IS NOTHING HE CAN DO IN THESE DOMINIONS AS WELL AS OUR NOMES WHOSE NUMBERS ARE SO GREAT THAT IT WORRIES US TO KEEP THEM ALL BUSY NOT EXACTLY RETURNED KALIKO WHERE IS MY BROTHER NOW INQUIRED SHAGGY IN THE METAL FOREST WHERE IS THAT THE METAL FOREST IS IN THE GREAT DOMED CAVERN THE LARGEST IN ALL OUR DOMINIONS REPLIED KALIKOdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.oggnew file mode 100644index 0000000000..ef18981839Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.txtnew file mode 100644index 0000000000..e1eb1a3480--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-03.txt@@ -0,0 +1 @@+A MAN SAID TO THE UNIVERSE SIR I EXIST SWEAT COVERED BRION'S BODY TRICKLING INTO THE TIGHT LOINCLOTH THAT WAS THE ONLY GARMENT HE WORE THE CUT ON HIS CHEST STILL DRIPPING BLOOD THE ACHE OF HIS OVERSTRAINED EYES EVEN THE SOARING ARENA AROUND HIM WITH THE THOUSANDS OF SPECTATORS WERE TRIVIALITIES NOT WORTH THINKING ABOUT HIS INSTANT OF PANIC WAS FOLLOWED BY A SMALL SHARP BLOW HIGH ON HIS CHEST ONE MINUTE A VOICE SAID AND THE TIME BUZZER SOUNDED A MINUTE IS NOT A VERY LARGE MEASURE OF TIME AND HIS BODY NEEDED EVERY FRACTION OF IT THE BUZZER'S WHIRR TRIGGERED HIS MUSCLES INTO COMPLETE RELAXATION ONLY HIS HEART AND LUNGS WORKED ON AT A STRONG MEASURED RATEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.oggnew file mode 100644index 0000000000..a3da0950a2Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.txtnew file mode 100644index 0000000000..35106253f6--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-04.txt@@ -0,0 +1 @@+HE WAS IN REVERIE SLIDING ALONG THE BORDERS OF CONSCIOUSNESS THE CONTESTANTS IN THE TWENTIES NEEDED UNDISTURBED REST THEREFORE NIGHTS IN THE DORMITORIES WERE AS QUIET AS DEATH PARTICULARLY SO ON THIS LAST NIGHT WHEN ONLY TWO OF THE LITTLE CUBICLES WERE OCCUPIED THE THOUSANDS OF OTHERS STANDING WITH DARK EMPTY DOORS THE OTHER VOICE SNAPPED WITH A HARSH URGENCY CLEARLY USED TO COMMAND I'M HERE BECAUSE THE MATTER IS OF UTMOST IMPORTANCE AND BRANDD IS THE ONE I MUST SEE NOW STAND ASIDE THE TWENTIES HE MUST HAVE DRAWN HIS GUN BECAUSE THE INTRUDER SAID QUICKLY PUT THAT AWAY YOU'RE BEING A FOOL OUT THERE WAS SILENCE THEN AND STILL WONDERING BRION WAS ONCE MORE ASLEEPdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.oggnew file mode 100644index 0000000000..3f9e125d65Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.txtnew file mode 100644index 0000000000..d201117c08--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-05.txt@@ -0,0 +1 @@+THE LAST TWO DAYS OF THE VOYAGE BARTLEY FOUND ALMOST INTOLERABLE EMERGING AT EUSTON AT HALF PAST THREE O'CLOCK IN THE AFTERNOON ALEXANDER HAD HIS LUGGAGE SENT TO THE SAVOY AND DROVE AT ONCE TO BEDFORD SQUARE SHE BLUSHED AND SMILED AND FUMBLED HIS CARD IN HER CONFUSION BEFORE SHE RAN UPSTAIRS THE ROOM WAS EMPTY WHEN HE ENTERED A COAL FIRE WAS CRACKLING IN THE GRATE AND THE LAMPS WERE LIT FOR IT WAS ALREADY BEGINNING TO GROW DARK OUTSIDE SHE CALLED HIS NAME ON THE THRESHOLD BUT IN HER SWIFT FLIGHT ACROSS THE ROOM SHE FELT A CHANGE IN HIM AND CAUGHT HERSELF UP SO DEFTLY THAT HE COULD NOT TELL JUST WHEN SHE DID ITdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.oggnew file mode 100644index 0000000000..f493bce68aBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.txtnew file mode 100644index 0000000000..dc7910cdc2--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-06.txt@@ -0,0 +1 @@+SHE MERELY BRUSHED HIS CHEEK WITH HER LIPS AND PUT A HAND LIGHTLY AND JOYOUSLY ON EITHER SHOULDER I NEVER DREAMED IT WOULD BE YOU BARTLEY WHEN DID YOU COME BARTLEY AND HOW DID IT HAPPEN YOU HAVEN'T SPOKEN A WORD SHE LOOKED AT HIS HEAVY SHOULDERS AND BIG DETERMINED HEAD THRUST FORWARD LIKE A CATAPULT IN LEASH I'LL DO ANYTHING YOU WISH ME TO BARTLEY SHE SAID TREMULOUSLY HE PULLED UP A WINDOW AS IF THE AIR WERE HEAVY HILDA WATCHED HIM FROM HER CORNER TREMBLING AND SCARCELY BREATHING DARK SHADOWS GROWING ABOUT HER EYES IT IT HASN'T ALWAYS MADE YOU MISERABLE HAS IT ALWAYS BUT IT'S WORSE NOWdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.oggnew file mode 100644index 0000000000..dd370d4f25Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.txtnew file mode 100644index 0000000000..2d3bc2ab18--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-07.txt@@ -0,0 +1 @@+ON THE LAST SATURDAY IN APRIL THE NEW YORK TIMES PUBLISHED AN ACCOUNT OF THE STRIKE COMPLICATIONS WHICH WERE DELAYING ALEXANDER'S NEW JERSEY BRIDGE AND STATED THAT THE ENGINEER HIMSELF WAS IN TOWN AND AT HIS OFFICE ON WEST TENTH STREET OVER THE FIREPLACE THERE WAS A LARGE OLD FASHIONED GILT MIRROR HE ROSE AND CROSSED THE ROOM QUICKLY OF COURSE I KNOW BARTLEY SHE SAID AT LAST THAT AFTER THIS YOU WON'T OWE ME THE LEAST CONSIDERATION BUT WE SAIL ON TUESDAY I SAW THAT INTERVIEW IN THE PAPER YESTERDAY TELLING WHERE YOU WERE AND I THOUGHT I HAD TO SEE YOU THAT'S ALL GOOD NIGHT I'M GOING NOWdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.oggnew file mode 100644index 0000000000..f4f74f998fBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.txtnew file mode 100644index 0000000000..fa53171eaa--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-08.txt@@ -0,0 +1 @@+LET ME TAKE OFF YOUR COAT AND YOUR BOOTS THEY'RE OOZING WATER IF YOU'D SENT ME A NOTE OR TELEPHONED ME OR ANYTHING I TOLD MYSELF THAT IF I WERE REALLY THINKING OF YOU AND NOT OF MYSELF A LETTER WOULD BE BETTER THAN NOTHING HE PAUSED THEY NEVER DID TO ME OH BARTLEY DID YOU WRITE TO ME ALEXANDER SLIPPED HIS ARM ABOUT HER I THINK I HAVE FELT THAT YOU WERE COMING HE BENT HIS FACE OVER HER HAIR AND I SHE WHISPERED I FELT THAT YOU WERE FEELING THAT BUT WHEN I CAME I THOUGHT I HAD BEEN MISTAKEN I'VE BEEN UP IN CANADA WITH MY BRIDGE AND I ARRANGED NOT TO COME TO NEW YORK UNTIL AFTER YOU HAD GONEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.oggnew file mode 100644index 0000000000..01c0b5f4b7Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.txtnew file mode 100644index 0000000000..40f1d96a1d--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-09.txt@@ -0,0 +1 @@+HE HAD NEVER BEEN FATHER LOVER HUSBAND FRIEND THE HEART OF THAT EX CONVICT WAS FULL OF VIRGINITY HIS SISTER AND HIS SISTER'S CHILDREN HAD LEFT HIM ONLY A VAGUE AND FAR OFF MEMORY WHICH HAD FINALLY ALMOST COMPLETELY VANISHED HE HAD MADE EVERY EFFORT TO FIND THEM AND NOT HAVING BEEN ABLE TO FIND THEM HE HAD FORGOTTEN THEM HE SUFFERED ALL THE PANGS OF A MOTHER AND HE KNEW NOT WHAT IT MEANT FOR THAT GREAT AND SINGULAR MOVEMENT OF A HEART WHICH BEGINS TO LOVE IS A VERY OBSCURE AND A VERY SWEET THING ONLY AS HE WAS FIVE AND FIFTY AND COSETTE EIGHT YEARS OF AGE ALL THAT MIGHT HAVE BEEN LOVE IN THE WHOLE COURSE OF HIS LIFE FLOWED TOGETHER INTO A SORT OF INEFFABLE LIGHTdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.oggnew file mode 100644index 0000000000..74dd1f1cf3Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.txtnew file mode 100644index 0000000000..e43f53314a--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-10.txt@@ -0,0 +1 @@+COSETTE ON HER SIDE HAD ALSO UNKNOWN TO HERSELF BECOME ANOTHER BEING POOR LITTLE THING SHE FELT THAT WHICH SHE HAD NEVER FELT BEFORE A SENSATION OF EXPANSION THE MAN NO LONGER PRODUCED ON HER THE EFFECT OF BEING OLD OR POOR SHE THOUGHT JEAN VALJEAN HANDSOME JUST AS SHE THOUGHT THE HOVEL PRETTY NATURE A DIFFERENCE OF FIFTY YEARS HAD SET A PROFOUND GULF BETWEEN JEAN VALJEAN AND COSETTE DESTINY FILLED IN THIS GULF TO MEET WAS TO FIND EACH OTHER WHEN THESE TWO SOULS PERCEIVED EACH OTHER THEY RECOGNIZED EACH OTHER AS NECESSARY TO EACH OTHER AND EMBRACED EACH OTHER CLOSELY MOREOVER JEAN VALJEAN HAD CHOSEN HIS REFUGE WELLdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.oggnew file mode 100644index 0000000000..bd92a210d2Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.txtnew file mode 100644index 0000000000..bd893e9c66--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-11.txt@@ -0,0 +1 @@+FUCHS BROUGHT UP A SACK OF POTATOES AND A PIECE OF CURED PORK FROM THE CELLAR AND GRANDMOTHER PACKED SOME LOAVES OF SATURDAY'S BREAD A JAR OF BUTTER AND SEVERAL PUMPKIN PIES IN THE STRAW OF THE WAGON BOX OCCASIONALLY ONE OF THE HORSES WOULD TEAR OFF WITH HIS TEETH A PLANT FULL OF BLOSSOMS AND WALK ALONG MUNCHING IT THE FLOWERS NODDING IN TIME TO HIS BITES AS HE ATE DOWN TOWARD THEM IT'S NO BETTER THAN A BADGER HOLE NO PROPER DUGOUT AT ALL NOW WHY IS THAT OTTO PRESENTLY AGAINST ONE OF THOSE BANKS I SAW A SORT OF SHED THATCHED WITH THE SAME WINE COLORED GRASS THAT GREW EVERYWHEREdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.oggnew file mode 100644index 0000000000..d59d48ffffBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.txtnew file mode 100644index 0000000000..47dfde3125--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-12.txt@@ -0,0 +1 @@+VERY GLAD VERY GLAD SHE EJACULATED YOU'LL GET FIXED UP COMFORTABLE AFTER WHILE MISSUS SHIMERDA MAKE GOOD HOUSE MY GRANDMOTHER ALWAYS SPOKE IN A VERY LOUD TONE TO FOREIGNERS AS IF THEY WERE DEAF SHE MADE MISSUS SHIMERDA UNDERSTAND THE FRIENDLY INTENTION OF OUR VISIT AND THE BOHEMIAN WOMAN HANDLED THE LOAVES OF BREAD AND EVEN SMELLED THEM AND EXAMINED THE PIES WITH LIVELY CURIOSITY EXCLAIMING MUCH GOOD MUCH THANK THE FAMILY HAD BEEN LIVING ON CORNCAKES AND SORGHUM MOLASSES FOR THREE DAYS I REMEMBERED WHAT THE CONDUCTOR HAD SAID ABOUT HER EYES HER SKIN WAS BROWN TOO AND IN HER CHEEKS SHE HAD A GLOW OF RICH DARK COLORdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.oggnew file mode 100644index 0000000000..881c94c833Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.txtnew file mode 100644index 0000000000..48a1a6a3e8--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-clean-13.txt@@ -0,0 +1 @@+THE TWO STRAY KITTENS GRADUALLY MAKE THEMSELVES AT HOME SOMEHOW OR OTHER CAT HAS TAUGHT THEM THAT HE'S IN CHARGE HERE AND HE JUST CHASES THEM FOR FUN NOW AND AGAIN WHEN HE'S NOT BUSY SLEEPING SHE DOESN'T PICK THEM UP BUT JUST HAVING THEM IN THE ROOM SURE DOESN'T GIVE HER ASTHMA WHEN ARE YOU GETTING RID OF THESE CATS I'M NOT FIXING TO START AN ANNEX TO KATE'S CAT HOME RIGHT AWAY WHEN I BRING HOME MY NEW PROGRAM HE SAYS HOW COME YOU'RE TAKING ONE LESS COURSE THIS HALF I EXPLAIN THAT I'M TAKING MUSIC AND ALSO BIOLOGY ALGEBRA ENGLISH AND FRENCH MUSIC HE SNORTS POP IT'S A COURSEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.oggnew file mode 100644index 0000000000..80481726e4Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.txtnew file mode 100644index 0000000000..65bb3c10d7--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-01.txt@@ -0,0 +1 @@+THEY MADE ME THINK OF DEFEATED ARMIES RETREATING OR OF GHOSTS WHO WERE TRYING DESPERATELY TO GET IN FOR SHELTER AND THEN WENT MOANING ON I COULD NOT TAKE MY EYES OFF THE MAN IN THE BED THE SHARP SMELL OF SPIRITS WENT THROUGH THE ROOM IT SEEMED TO ME THAT HE DESPISED HIM FOR BEING SO SIMPLE AND DOCILE THE SICK MAN RAGED AND SHOOK HIS FIST HE SEEMED TO BE CURSING PEOPLE WHO HAD WRONGED HIM QUICKLY IT WAS COVERED WITH BRIGHT RED SPOTS I THOUGHT I HAD NEVER SEEN ANY BLOOD SO BRIGHT HE LAY PATIENTLY FIGHTING FOR BREATH LIKE A CHILD WITH CROUP ANTONIA'S FATHER UNCOVERED ONE OF HIS LONG BONY LEGS AND RUBBED IT RHYTHMICALLYdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.oggnew file mode 100644index 0000000000..0a89c8a6c7Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.txtnew file mode 100644index 0000000000..532b12d9dd--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-02.txt@@ -0,0 +1 @@+THROUGHOUT THIS CENTURY THE POWER OF THE CHURCH WAS CONSTANTLY ON THE INCREASE AND IS VISIBLE IN MANY IMPORTANT CHANGES THE ANCESTORS OF THE PRESENT PRETENDER CONGAL SURNAMED THE SQUINT EYED HAD TWICE RECEIVED AND CHERISHED THE LICENTIOUS BARDS WHEN UNDER THE BAN OF TARA AND HIS POPULARITY WITH THAT STILL POWERFUL ORDER WAS ONE PROP OF HIS AMBITION IT IS PRETTY CLEAR ALSO THAT THE LAST RALLY OF DRUIDISM AGAINST CHRISTIANITY TOOK PLACE BEHIND HIS BANNER ON THE PLAIN OF MOIRA THE POETS OF SUCCEEDING AGES HAVE DWELT MUCH IN DETAIL ON THE OCCURRENCES OF THIS MEMORABLE DAY LIKE THE TWO KINGS OF SPARTA THEY REIGNED JOINTLY DIVIDING BETWEEN THEM THE LABOURS AND CARES OF STATEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.oggnew file mode 100644index 0000000000..2bc688b477Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.txtnew file mode 100644index 0000000000..00c78e2306--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-03.txt@@ -0,0 +1 @@+IT WAS THE SEASON WHEN THE ANCIENT SUN GOD HAD BEEN ACCUSTOMED TO RECEIVE HIS ANNUAL OBLATIONS AND WE CAN WELL BELIEVE THAT THOSE WHOSE HEARTS STILL TREMBLED AT THE NAME OF BEL MUST HAVE CONNECTED THE ECLIPSE AND THE PLAGUE WITH THE REVOLUTION IN THE NATIONAL WORSHIP AND THE OVERTHROW OF THE ANCIENT GODS ON THAT PLAIN OF PROSTRATION WHERE THEY HAD SO LONG RECEIVED THE HOMAGE OF AN ENTIRE PEOPLE LASTLY THE ROYAL BROTHERS FELL THEMSELVES VICTIMS TO THE EPIDEMIC WHICH SO SADLY SIGNALIZES THEIR REIGN THE ONLY CONFLICTS THAT OCCURRED ON IRISH SOIL WITH A PICTISH OR AN ANGLO SAXON FORCE IF WE EXCEPT THOSE WHO FORMED A CONTINGENT OF CONGAL'S ARMY AT MOIRA OCCURRED IN THE TIME OF THE HOSPITABLE FINNACTAdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.oggnew file mode 100644index 0000000000..5e39b5122cBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.txtnew file mode 100644index 0000000000..cb53c6c7c8--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-04.txt@@ -0,0 +1 @@+IF THE READER WILL EXCUSE ME I WILL SAY NOTHING OF MY ANTECEDENTS NOR OF THE CIRCUMSTANCES WHICH LED ME TO LEAVE MY NATIVE COUNTRY THE NARRATIVE WOULD BE TEDIOUS TO HIM AND PAINFUL TO MYSELF IT WILL BE SEEN THAT I DID NOT SUCCEED IN MY DESIGN AND THAT HOWEVER MUCH I MAY HAVE MET WITH THAT WAS NEW AND STRANGE I HAVE BEEN UNABLE TO REAP ANY PECUNIARY ADVANTAGE NO ONE WHO IS HIMSELF HONEST WILL DOUBT MY BEING SO I REACHED MY DESTINATION IN ONE OF THE LAST MONTHS OF EIGHTEEN SIXTY EIGHT BUT I DARE NOT MENTION THE SEASON LEST THE READER SHOULD GATHER IN WHICH HEMISPHERE I WASdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.oggnew file mode 100644index 0000000000..b28185f72dBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.txtnew file mode 100644index 0000000000..9601247dbb--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-05.txt@@ -0,0 +1 @@+SHEEP AND CATTLE WERE INTRODUCED AND BRED WITH EXTREME RAPIDITY MEN TOOK UP THEIR FIFTY THOUSAND OR ONE HUNDRED THOUSAND ACRES OF COUNTRY GOING INLAND ONE BEHIND THE OTHER TILL IN A FEW YEARS THERE WAS NOT AN ACRE BETWEEN THE SEA AND THE FRONT RANGES WHICH WAS NOT TAKEN UP AND STATIONS EITHER FOR SHEEP OR CATTLE WERE SPOTTED ABOUT AT INTERVALS OF SOME TWENTY OR THIRTY MILES OVER THE WHOLE COUNTRY I WAS DELIGHTED WITH THE COUNTRY AND THE MANNER OF LIFE I WAS TO SEE THE SHEEP NOT NECESSARILY CLOSE AT HAND NOR TO GET THEM IN A SINGLE MOB BUT TO SEE ENOUGH OF THEM HERE AND THERE TO FEEL EASY THAT NOTHING HAD GONE WRONG THIS WAS NO DIFFICULT MATTER FOR THERE WERE NOT ABOVE EIGHT HUNDRED OF THEM AND BEING ALL BREEDING EWES THEY WERE PRETTY QUIETdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.oggnew file mode 100644index 0000000000..8e36200b1cBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.txtnew file mode 100644index 0000000000..e956ba8f38--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-loud-noise-5db-06.txt@@ -0,0 +1 @@+I IMAGINE THERE WERE SEVERAL KINDS OF OLD FASHIONED CHRISTMASES BUT IT COULD HARDLY BE WORSE THAN A CHOP IN MY CHAMBERS OR HORROR OF HORRORS AT THE CLUB OR MY COUSIN LUCY'S NOTION OF WHAT SHE CALLS THE FESTIVE SEASON FESTIVE YES THE REPLY WAS WRITTEN IN A SPRAWLING FEMININE HAND IT WAS A LITTLE VAGUE IT APPEARED THAT THE TERMS WOULD BE FIVE GUINEAS BUT THERE WAS NO MENTION OF THE LENGTH OF TIME WHICH THAT FEE WOULD COVER THE WHOLE THING WAS A TRIFLE ODD THERE WAS NOTHING SAID ABOUT THE SORT OF ACCOMMODATION WHICH WOULD BE PROVIDED NOTHING ABOUT THE KIND OF ESTABLISHMENT WHICH WAS MAINTAINED OR THE TABLE WHICH WAS KEPTdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.oggnew file mode 100644index 0000000000..377bcbea93Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.txtnew file mode 100644index 0000000000..e07cb7fb97--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-01.txt@@ -0,0 +1 @@+HE DOES AND FOR ONCE I WIN A ROUND I KEEP MUSIC FOR THIS SEMESTER I'LL BE LUCKY IF I HAVE TIME TO BREATHE SOMETIMES SCHOOLS DO LET KIDS TAKE A LOT OF SOFT COURSES AND THEN THEY'RE OUT ON A LIMB LATER HUH SO HE CARES HUH BESIDES SAYS TOM HALF THE REASON YOU AND YOUR FATHER ARE ALWAYS BICKERING IS THAT YOU'RE SO MUCH ALIKE ME LIKE HIM SURE AS LONG AS THERE'S A BONE ON THE FLOOR THE TWO OF YOU WORRY IT I GET THE PILLOWS COMFORTABLY ARRANGED ON THE FLOOR WITH A BIG BOTTLE OF SODA AND A BAG OF POPCORN WITHIN EASY REACH POP GOES RIGHT ON TUNING HIS CHANNELdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.oggnew file mode 100644index 0000000000..5ddef34870Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.txtnew file mode 100644index 0000000000..5f3aa8b1bb--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-02.txt@@ -0,0 +1 @@+THEY SAT ABOUT THE HOUSE MOST OF THE DAY AS IF IT WERE SUNDAY GREASING THEIR BOOTS MENDING THEIR SUSPENDERS PLAITING WHIPLASHES ANYWAY HE WOULD NEVER ALLOW ONE OF HIS HORSES TO BE PUT TO SUCH A STRAIN I HAD WANTED TO GET SOME PICTURE BOOKS FOR YULKA AND ANTONIA EVEN YULKA WAS ABLE TO READ A LITTLE NOW SHE CUT SQUARES OF COTTON CLOTH AND WE SEWED THEM TOGETHER INTO A BOOK ON THE WHITE PAGES I GROUPED SUNDAY SCHOOL CARDS AND ADVERTISING CARDS WHICH I HAD BROUGHT FROM MY OLD COUNTRY WHEN HE MOUNTED HIS HORSE AT THE DOOR I SAW THAT HE HAD A HATCHET SLUNG TO HIS BELT AND HE GAVE GRANDMOTHER A MEANING LOOK WHICH TOLD ME HE WAS PLANNING A SURPRISE FOR MEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.oggnew file mode 100644index 0000000000..9b808af67dBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.txtnew file mode 100644index 0000000000..9734f60f61--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-03.txt@@ -0,0 +1 @@+I PUT ON MY CAP AND RAN OUT TO MEET JAKE HE USED TO HELP MY FATHER CUT CHRISTMAS TREES FOR ME IN VIRGINIA AND HE HAD NOT FORGOTTEN HOW MUCH I LIKED THEM BY THE TIME WE HAD PLACED THE COLD FRESH SMELLING LITTLE TREE IN A CORNER OF THE SITTING ROOM IT WAS ALREADY CHRISTMAS EVE FROM UNDER THE LINING HE NOW PRODUCED A COLLECTION OF BRILLIANTLY COLORED PAPER FIGURES SEVERAL INCHES HIGH AND STIFF ENOUGH TO STAND ALONE I CAN SEE THEM NOW EXACTLY AS THEY LOOKED WORKING ABOUT THE TABLE IN THE LAMPLIGHT JAKE WITH HIS HEAVY FEATURES SO RUDELY MOULDED THAT HIS FACE SEEMED SOMEHOW UNFINISHED OTTO WITH HIS HALF EAR AND THE SAVAGE SCAR THAT MADE HIS UPPER LIP CURL SO FEROCIOUSLY UNDER HIS TWISTED MUSTACHEdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.oggnew file mode 100644index 0000000000..e9264da974Binary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.txtnew file mode 100644index 0000000000..35be59d23d--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-04.txt@@ -0,0 +1 @@+SHE WAS FOUR YEARS OLDER THAN I TO BE SURE AND HAD SEEN MORE OF THE WORLD BUT I WAS A BOY AND SHE WAS A GIRL AND I RESENTED HER PROTECTING MANNER THIS CHANGE CAME ABOUT FROM AN ADVENTURE WE HAD TOGETHER ONE DAY WHEN I RODE OVER TO THE SHIMERDAS I FOUND ANTONIA STARTING OFF ON FOOT FOR RUSSIAN PETER'S HOUSE TO BORROW A SPADE AMBROSCH NEEDED THERE HAD BEEN ANOTHER BLACK FROST THE NIGHT BEFORE AND THE AIR WAS CLEAR AND HEADY AS WINE IT WAS ON ONE OF THESE GRAVEL BEDS THAT I MET MY ADVENTURE I WHIRLED ROUND AND THERE ON ONE OF THOSE DRY GRAVEL BEDS WAS THE BIGGEST SNAKE I HAD EVER SEENdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.oggnew file mode 100644index 0000000000..975ad1018fBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.txtnew file mode 100644index 0000000000..789dfe5900--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-05.txt@@ -0,0 +1 @@+I KNOW I AM JUST AWFUL JIM I WAS SO SCARED I NEVER KNOW YOU WAS SO BRAVE JIM SHE WENT ON COMFORTINGLY A FAINT FETID SMELL CAME FROM HIM AND A THREAD OF GREEN LIQUID OOZED FROM HIS CRUSHED HEAD LOOK TONY THAT'S HIS POISON I SAID I EXPLAINED TO ANTONIA HOW THIS MEANT THAT HE WAS TWENTY FOUR YEARS OLD THAT HE MUST HAVE BEEN THERE WHEN WHITE MEN FIRST CAME LEFT ON FROM BUFFALO AND INDIAN TIMES WE DECIDED THAT ANTONIA SHOULD RIDE DUDE HOME AND I WOULD WALK I FOLLOWED WITH THE SPADE OVER MY SHOULDER DRAGGING MY SNAKE OTTO FUCHS WAS THE FIRST ONE WE METdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.ogg b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.oggnew file mode 100644index 0000000000..0a33e7ddcaBinary files /dev/null and b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.ogg differdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.txt b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.txtnew file mode 100644index 0000000000..c17290cb57--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/eval_data/stt-wer-moderate-noise-15db-06.txt@@ -0,0 +1 @@+PETER TOLD HIS TROUBLES TO MISTER SHIMERDA HE WAS UNABLE TO MEET A NOTE WHICH FELL DUE ON THE FIRST OF NOVEMBER HAD TO PAY AN EXORBITANT BONUS ON RENEWING IT AND TO GIVE A MORTGAGE ON HIS PIGS AND HORSES AND EVEN HIS MILK COW PETER COULD GIVE NO VERY CLEAR ACCOUNT OF HIS TRANSACTIONS WITH CUTTER SHE ASKED PETER TO WAIT A MOMENT AND WHEN SHE CAME BACK FROM THE KITCHEN SHE BROUGHT A BAG OF SANDWICHES AND DOUGHNUTS FOR US WE LAY STILL AND DID NOT TALK THE LITTLE HOUSE ON THE HILLSIDE WAS SO MUCH THE COLOR OF THE NIGHT THAT WE COULD NOT SEE IT AS WE CAME UP THE DRAWdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/intro.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/intro.pyindex 2c2a718e65..76bde96035 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/intro.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/intro.py@@ -10,14 +10,13 @@ fallback when Langfuse is unconfigured or unreachable."""import asyncio-import jsonimport loggingimport os-import refrom langfuse import get_clientfrom backend.api.features.onboarding_dump.models import SuggestedPrompt+from backend.api.features.onboarding_dump.parsing import parse_response_jsonfrom backend.util.clients import get_openai_clientfrom backend.util.settings import Settings@@ -156,7 +155,7 @@ async def generate_intro(transcript: str) -> tuple[str, list[SuggestedPrompt]]:),timeout=_TIMEOUT_SECONDS,)- data = _parse_response_json(response.choices[0].message.content or "")+ data = parse_response_json(response.choices[0].message.content or "")except Exception as e: # degrades to the template belowlogger.warning("Brain dump greeting generation failed (attempt %s): %s",@@ -205,31 +204,6 @@ async def _fetch_langfuse_prompt() -> str | None:return None-def _parse_response_json(content: str) -> dict | None:- """Parse the model's JSON, tolerating markdown fences and preamble.-- Anthropic models have no OpenAI-style JSON mode, so the contract is- prompt-level ("return ONLY valid JSON") and the parser forgives the- two ways that commonly bends: a ```json fence around the object, or- stray prose before/after it.- """- text = content.strip()- if text.startswith("```"):- text = re.sub(r"^```[a-zA-Z]*\s*", "", text)- text = re.sub(r"\s*```$", "", text)- try:- data = json.loads(text)- except json.JSONDecodeError:- start, end = text.find("{"), text.rfind("}")- if start == -1 or end <= start:- return None- try:- data = json.loads(text[start : end + 1])- except json.JSONDecodeError:- return None- return data if isinstance(data, dict) else None--def _parse_prompts(raw: object) -> list[SuggestedPrompt]:if not isinstance(raw, list):return []diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/parsing.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/parsing.pynew file mode 100644index 0000000000..1faf70e709--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/parsing.py@@ -0,0 +1,29 @@+"""Shared LLM-output parsing for the brain-dump pipeline."""++import json+import re+++def parse_response_json(content: str) -> dict | None:+ """Parse the model's JSON, tolerating markdown fences and preamble.++ Anthropic models have no OpenAI-style JSON mode, so the contract is+ prompt-level ("return ONLY valid JSON") and the parser forgives the+ two ways that commonly bends: a ```json fence around the object, or+ stray prose before/after it.+ """+ text = content.strip()+ if text.startswith("```"):+ text = re.sub(r"^```[a-zA-Z]*\s*", "", text)+ text = re.sub(r"\s*```$", "", text)+ try:+ data = json.loads(text)+ except json.JSONDecodeError:+ start, end = text.find("{"), text.rfind("}")+ if start == -1 or end <= start:+ return None+ try:+ data = json.loads(text[start : end + 1])+ except json.JSONDecodeError:+ return None+ return data if isinstance(data, dict) else Nonediff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/quality.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/quality.pynew file mode 100644index 0000000000..435e3c6ab2--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/quality.py@@ -0,0 +1,175 @@+"""Quality gate between transcription and personalized generation.++A transcription request succeeding is not the same as the user saying+anything. Silence, background noise, an STT hallucination or four seconds+of filler all come back as a "successful" transcript — and everything+downstream (greeting, suggested prompts, provider picks) would confidently+personalize from it.++Layered on purpose so the common cases stay free:++1. Deterministic reject — unmistakably unusable text (empty, no letters,+ a looping/repeated transcript) never reaches a model.+2. Deterministic pass — clearly substantial text (a real rambling dump)+ never pays for a model call either.+3. Everything in between gets one cheap LLM judgement: "is there enough+ concrete information here to personalize from?". Length alone cannot+ make this call — "Automate my Shopify refund emails" is six words and+ must pass, "uh hello hello testing" must not.++The gate never judges grammar, accent, fluency or language: a coherent+dump in any language passes. And it never deletes anything — the caller+keeps the audio and transcript on the row, exactly like every other+failure in this pipeline.+"""++import asyncio+import logging+import os+import zlib+from typing import Literal++from backend.api.features.onboarding_dump.parsing import parse_response_json+from backend.util.clients import get_openai_client++logger = logging.getLogger(__name__)++# Stored in ``OnboardingBrainDump.errorCode`` (a plain string column, so no+# enum migration) and read by the frontend to show the recovery screen.+NO_USABLE_SPEECH = "no_usable_speech"+INSUFFICIENT_CONTENT = "insufficient_content"+QUALITY_ERROR_CODES = frozenset({NO_USABLE_SPEECH, INSUFFICIENT_CONTENT})++_MODEL = os.environ.get("BRAIN_DUMP_QUALITY_MODEL", "anthropic/claude-haiku-4-5")+# The gate runs inside the finalize request, after transcription already+# spent part of the frontend proxy's 30s budget. A 100-token Haiku verdict+# typically lands in a couple of seconds; a stalled provider must degrade+# to the recoverable reject quickly instead of eating the rest of the+# request budget.+_TIMEOUT_SECONDS = 8++# Above this many words a transcript is clearly substantial and skips the+# LLM entirely — the vast majority of real dumps land here, which is what+# keeps the gate's cost near zero. ``split()`` can't see words in+# space-less scripts (Chinese, Japanese, ...), so those get a+# character-based route instead — but only when the chars-per-"word"+# ratio proves the script really is space-less, or 25 words of English+# could slip past the word threshold on characters alone. Both routes+# apply *after* the repetition checks, so a looping transcript cannot+# buy its way through on sheer length.+CLEAR_PASS_WORDS = 40+CLEAR_PASS_CHARS = 100+SPACELESS_CHARS_PER_WORD = 10++# The classic STT failure on silence/noise is a decoder loop: the same+# word or phrase over and over. Both checks are standard Whisper-style+# signals — a low unique-word ratio and a text that zlib collapses.+REPETITION_MIN_WORDS = 20+MIN_UNIQUE_WORD_RATIO = 0.3+COMPRESSION_MIN_CHARS = 80+MAX_COMPRESSION_RATIO = 2.4++_PROMPT = """A new user of an automation platform was asked to describe \+their work and what they'd like automated. Below is the raw transcript \+(spoken or typed). Decide whether it contains enough concrete information \+to personalize a greeting or propose work for them — anything about who \+they are, what they do, tools they use, or tasks they want handled counts.++The transcript may be in ANY language; a coherent non-English answer \+counts fully. Ignore grammar, fluency, accent artifacts and how \+interesting the work sounds. Judge ONLY whether there is usable substance.++Answer "usable": false when the text is only filler, greetings, mic \+testing, repeated fragments, or generic phrases a speech model produces \+from silence or background noise (e.g. "thank you for watching").++Return ONLY valid JSON: {{"usable": true}} or {{"usable": false}}.++Transcript:+{transcript}+"""+++async def check_transcript_quality(transcript: str) -> str | None:+ """Return an error code when ``transcript`` can't support personalization.++ ``None`` means the dump is usable and the personalized pipeline may+ run. Deterministic answers are preferred in both directions; only the+ ambiguous middle costs an LLM call.+ """+ text = transcript.strip()+ verdict = _deterministic_verdict(text)+ if verdict == "reject":+ return NO_USABLE_SPEECH+ if verdict == "pass":+ return None+ return await _semantic_verdict(text)+++def _deterministic_verdict(text: str) -> Literal["pass", "reject", "ambiguous"]:+ if not text or not any(ch.isalnum() for ch in text):+ return "reject"++ words = [word for word in text.split() if any(ch.isalnum() for ch in word)]+ if not words:+ return "reject"+ if len(words) >= REPETITION_MIN_WORDS:+ unique_ratio = len({w.lower() for w in words}) / len(words)+ if unique_ratio < MIN_UNIQUE_WORD_RATIO:+ return "reject"+ if (+ len(text) >= COMPRESSION_MIN_CHARS+ and _compression_ratio(text) > MAX_COMPRESSION_RATIO+ ):+ return "reject"++ alnum_chars = sum(1 for ch in text if ch.isalnum())+ is_spaceless_script = alnum_chars / len(words) >= SPACELESS_CHARS_PER_WORD+ if len(words) >= CLEAR_PASS_WORDS or (+ is_spaceless_script and alnum_chars >= CLEAR_PASS_CHARS+ ):+ return "pass"+ return "ambiguous"+++def _compression_ratio(text: str) -> float:+ """How far zlib collapses ``text`` — high means the decoder was looping."""+ raw = text.encode("utf-8")+ return len(raw) / len(zlib.compress(raw))+++async def _semantic_verdict(text: str) -> str | None:+ """One cheap LLM call for the short-but-nonempty middle band.++ Every failure path here — no client, timeout, malformed output —+ resolves to ``INSUFFICIENT_CONTENT`` rather than a silent pass: an+ ambiguous dump must never produce confident personalized output just+ because the judge was unavailable. The outcome is recoverable by+ design (the user is offered retry / type / skip), which is the safe+ documented behavior for a quality-model outage.+ """+ client = get_openai_client(prefer_openrouter=True)+ if client is None:+ logger.warning("Brain dump quality check: no LLM client configured")+ return INSUFFICIENT_CONTENT++ try:+ response = await asyncio.wait_for(+ client.chat.completions.create(+ model=_MODEL,+ messages=[{"role": "user", "content": _PROMPT.format(transcript=text)}],+ temperature=0,+ max_tokens=100,+ ),+ timeout=_TIMEOUT_SECONDS,+ )+ content = response.choices[0].message.content or ""+ except Exception as e:+ logger.warning("Brain dump quality check failed: %s", e)+ return INSUFFICIENT_CONTENT++ data = parse_response_json(content)+ if not isinstance(data, dict) or not isinstance(data.get("usable"), bool):+ logger.warning("Brain dump quality check: malformed verdict")+ return INSUFFICIENT_CONTENT+ return None if data["usable"] else INSUFFICIENT_CONTENTdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/quality_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/quality_test.pynew file mode 100644index 0000000000..a381c7221f--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/quality_test.py@@ -0,0 +1,294 @@+"""Tests for the brain-dump quality gate.++The layering is the contract under test: unmistakable garbage and clearly+substantial dumps must be decided without a model call, only the ambiguous+middle may pay for one, and every judge failure resolves to a recoverable+rejection rather than a silent pass.+"""++import asyncio+from unittest.mock import AsyncMock, MagicMock++import pytest+from pytest_mock import MockerFixture++from backend.api.features.onboarding_dump import quality++SUBSTANTIAL_DUMP = (+ "So I run a small bakery in Lisbon with my sister. Every Friday we get "+ "about forty wholesale orders by email and I retype them into a "+ "spreadsheet, then chase the suppliers on WhatsApp, then invoice "+ "everyone through our accounting tool at the end of the month. Honestly "+ "the invoicing and the order retyping are the parts I would love to "+ "never touch again."+)++SPANISH_DUMP = (+ "Trabajo como gestora de redes sociales para tres restaurantes en "+ "Madrid. Cada semana programo las publicaciones, respondo los "+ "comentarios, preparo un informe con las métricas de cada local y se lo "+ "envío a los dueños por correo. Me encantaría automatizar los informes "+ "y las respuestas más repetitivas para dedicar ese tiempo a las "+ "campañas nuevas."+)+++def _llm_client(content: str) -> MagicMock:+ response = MagicMock()+ response.choices = [MagicMock(message=MagicMock(content=content))]+ client = MagicMock()+ client.chat.completions.create = AsyncMock(return_value=response)+ return client+++def llm(mocker: MockerFixture):+ def install(client: MagicMock | None) -> MagicMock:+ return mocker.patch.object(+ quality, "get_openai_client", MagicMock(return_value=client)+ )++ return install+++# --- deterministic rejects: no model call ---------------------------------+++ "transcript",+ [+ "",+ " \n\t ",+ "... !!! ,,, --- ???",+ ". . . . . . . . . .",+ ],+)+async def test_empty_and_symbol_only_transcripts_reject_without_llm(+ llm, transcript: str+):+ client_factory = llm(_llm_client('{"usable": true}'))+ assert (+ await quality.check_transcript_quality(transcript) == quality.NO_USABLE_SPEECH+ )+ client_factory.assert_not_called()+++async def test_repeated_word_loop_rejects_without_llm(llm):+ client_factory = llm(_llm_client('{"usable": true}'))+ looped = "you " * 30+ assert await quality.check_transcript_quality(looped) == quality.NO_USABLE_SPEECH+ client_factory.assert_not_called()+++async def test_repeated_phrase_loop_rejects_without_llm(llm):+ client_factory = llm(_llm_client('{"usable": true}'))+ looped = "thank you for watching " * 12+ assert await quality.check_transcript_quality(looped) == quality.NO_USABLE_SPEECH+ client_factory.assert_not_called()+++async def test_compressible_character_garbage_rejects_without_llm(llm):+ # One "word", so the unique-ratio check can't see it — the zlib+ # compression check is what catches a decoder stuck on a character.+ client_factory = llm(_llm_client('{"usable": true}'))+ garbage = "a" * 200+ assert await quality.check_transcript_quality(garbage) == quality.NO_USABLE_SPEECH+ client_factory.assert_not_called()+++# --- deterministic passes: no model call ----------------------------------+++async def test_substantial_english_dump_passes_without_llm(llm):+ client_factory = llm(_llm_client('{"usable": false}'))+ assert await quality.check_transcript_quality(SUBSTANTIAL_DUMP) is None+ client_factory.assert_not_called()+++async def test_substantial_non_english_dump_passes_without_llm(llm):+ client_factory = llm(_llm_client('{"usable": false}'))+ assert await quality.check_transcript_quality(SPANISH_DUMP) is None+ client_factory.assert_not_called()+++async def test_spaceless_script_dump_passes_on_characters(llm):+ # split() sees a handful of "words" here; the character count is what+ # recognizes a substantial CJK dump.+ client_factory = llm(_llm_client('{"usable": false}'))+ chinese = (+ "我在上海经营一家小型设计工作室,主要为餐饮品牌做菜单和门店视觉设计。"+ "每周我要整理客户反馈、更新项目进度表、给供应商发对账单,还要在社交媒体"+ "上发布作品集。我最想把整理反馈和发对账单这两件事自动化,因为它们每周"+ "都要花掉我好几个小时。"+ )+ assert await quality.check_transcript_quality(chinese) is None+ client_factory.assert_not_called()+++# --- threshold seams: exact boundaries route correctly --------------------++# 45 distinct natural words, sliced to sit exactly on CLEAR_PASS_WORDS.+# fmt: off+_DISTINCT_WORDS = [+ "my", "bakery", "ships", "fresh", "sourdough", "daily", "while",+ "managing", "wholesale", "orders", "refunds", "invoices", "suppliers",+ "deliveries", "roster", "payroll", "marketing", "newsletters",+ "customers", "complaints", "reviews", "inventory", "flour", "butter",+ "yeast", "ovens", "repairs", "budget", "taxes", "accounting",+ "spreadsheets", "emails", "scheduling", "drivers", "routes",+ "packaging", "labels", "promotions", "seasonal", "catering", "events",+ "quarterly", "forecasts", "vendors", "contracts",+]+# fmt: on+++async def test_exactly_clear_pass_words_passes_without_llm(llm):+ client_factory = llm(_llm_client('{"usable": false}'))+ text = " ".join(_DISTINCT_WORDS[: quality.CLEAR_PASS_WORDS])+ assert await quality.check_transcript_quality(text) is None+ client_factory.assert_not_called()+++async def test_one_word_below_clear_pass_still_asks_the_llm(llm):+ llm(_llm_client('{"usable": false}'))+ text = " ".join(_DISTINCT_WORDS[: quality.CLEAR_PASS_WORDS - 1])+ assert await quality.check_transcript_quality(text) == quality.INSUFFICIENT_CONTENT+++async def test_unique_ratio_just_below_threshold_rejects(llm):+ client_factory = llm(_llm_client('{"usable": true}'))+ # 20 words, 5 unique → 0.25, just under MIN_UNIQUE_WORD_RATIO.+ text = " ".join(["ab", "cd", "ef", "gh", "ij"] * 4)+ assert await quality.check_transcript_quality(text) == quality.NO_USABLE_SPEECH+ client_factory.assert_not_called()+++async def test_unique_ratio_at_threshold_is_not_a_repetition_reject(llm):+ llm(_llm_client('{"usable": true}'))+ # 20 words, 6 unique → exactly 0.3: the check is strictly-less-than,+ # so this falls through to the LLM instead of rejecting.+ text = " ".join(["ab", "cd", "ef", "gh", "ij", "kl"] * 3 + ["ab", "cd"])+ assert await quality.check_transcript_quality(text) is None+++async def test_compression_seam_separates_looping_from_prose(llm):+ llm(_llm_client('{"usable": false}'))+ looping = "banana" * 20+ prose = (+ "Automate the weekly refund emails my Shopify store keeps sending "+ "to unhappy overseas customers every single Monday."+ )+ # Self-check the fixtures actually straddle the threshold.+ assert quality._compression_ratio(looping) > quality.MAX_COMPRESSION_RATIO+ assert quality._compression_ratio(prose) <= quality.MAX_COMPRESSION_RATIO+ assert await quality.check_transcript_quality(looping) == quality.NO_USABLE_SPEECH+ assert await quality.check_transcript_quality(prose) == quality.INSUFFICIENT_CONTENT+++# --- the ambiguous middle: one LLM call decides ---------------------------+++async def test_short_meaningful_request_passes_via_llm(llm):+ llm(_llm_client('{"usable": true}'))+ short = "Automate my Shopify refund emails."+ assert await quality.check_transcript_quality(short) is None+++async def test_short_filler_rejects_via_llm(llm):+ llm(_llm_client('{"usable": false}'))+ filler = "Uh hello hello, testing, can you hear me?"+ assert (+ await quality.check_transcript_quality(filler) == quality.INSUFFICIENT_CONTENT+ )+++async def test_hallucinated_outro_rejects_via_llm(llm):+ llm(_llm_client('{"usable": false}'))+ assert (+ await quality.check_transcript_quality("Thank you for watching.")+ == quality.INSUFFICIENT_CONTENT+ )+++async def test_llm_verdict_tolerates_markdown_fence(llm):+ llm(_llm_client('```json\n{"usable": true}\n```'))+ assert await quality.check_transcript_quality("Automate my invoices.") is None+++# --- judge failures: never a silent pass ----------------------------------+++async def test_no_llm_client_rejects_ambiguous_input(llm):+ llm(None)+ assert (+ await quality.check_transcript_quality("Automate my invoices.")+ == quality.INSUFFICIENT_CONTENT+ )+++async def test_no_llm_client_still_passes_clear_input(llm):+ llm(None)+ assert await quality.check_transcript_quality(SUBSTANTIAL_DUMP) is None++[email protected]("content", ["not json at all", '{"usable": "yes"}', "{}"])+async def test_malformed_llm_verdict_rejects(llm, content: str):+ llm(_llm_client(content))+ assert (+ await quality.check_transcript_quality("Automate my invoices.")+ == quality.INSUFFICIENT_CONTENT+ )+++async def test_empty_choices_rejects(llm):+ response = MagicMock()+ response.choices = []+ client = MagicMock()+ client.chat.completions.create = AsyncMock(return_value=response)+ llm(client)+ assert (+ await quality.check_transcript_quality("Automate my invoices.")+ == quality.INSUFFICIENT_CONTENT+ )+++async def test_llm_timeout_rejects(llm):+ client = MagicMock()+ client.chat.completions.create = AsyncMock(side_effect=asyncio.TimeoutError())+ llm(client)+ assert (+ await quality.check_transcript_quality("Automate my invoices.")+ == quality.INSUFFICIENT_CONTENT+ )+++async def test_llm_error_rejects(llm):+ client = MagicMock()+ client.chat.completions.create = AsyncMock(side_effect=RuntimeError("boom"))+ llm(client)+ assert (+ await quality.check_transcript_quality("Automate my invoices.")+ == quality.INSUFFICIENT_CONTENT+ )diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/recommend.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/recommend.pyindex fb3d87e8e0..f0c0f419c7 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/recommend.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/recommend.py@@ -15,8 +15,8 @@ from backend.api.features.integrations.models import (get_all_provider_names,get_provider_description,)-from backend.api.features.onboarding_dump.intro import _parse_response_jsonfrom backend.api.features.onboarding_dump.models import RecommendedProvider+from backend.api.features.onboarding_dump.parsing import parse_response_jsonfrom backend.util.clients import get_openai_clientlogger = logging.getLogger(__name__)@@ -89,7 +89,7 @@ async def generate_recommendations(transcript: str) -> list[RecommendedProvider]logger.warning("Brain dump recommendation generation failed: %s", e)return []- data = _parse_response_json(response.choices[0].message.content or "")+ data = parse_response_json(response.choices[0].message.content or "")return _parse_recommendations(data, set(known))diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.pyindex 5a16820184..b71f6c25d8 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py@@ -254,6 +254,18 @@ def generation(mocker: MockerFixture) -> dict[str, AsyncMock]:return {"generate_intro": intro_mock, "generate_recommendations": recommend_mock}[email protected](autouse=True)+def quality_gate(mocker: MockerFixture) -> AsyncMock:+ """The quality gate passes by default so route tests stay about the+ routes; rejection tests flip the return value."""+ mock = AsyncMock(return_value=None)+ mocker.patch(+ "backend.api.features.onboarding_dump.quality.check_transcript_quality",+ new=mock,+ )+ return mock++def upload_part(part_index: int = 0,content: bytes = b"chunk",@@ -422,6 +434,43 @@ def test_typed_finalize_skips_audio_and_extracts_directly(stt_create.assert_not_awaited()+def test_quality_rejected_voice_finalize_reports_the_error_code(+ dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+):+ """The API state the frontend branches on: a 200 envelope with+ ``failed`` + a quality error code, distinct from a transcription+ failure — with transcript and audio still on the row."""+ upload_part()+ quality_gate.return_value = "insufficient_content"++ response = finalize()++ assert response.status_code == 200+ body = response.json()+ assert body["status"] == BrainDumpStatus.failed+ assert body["error_code"] == "insufficient_content"+ assert dumps.row is not None+ assert dumps.row.status == BrainDumpStatus.failed+ assert dumps.row.errorCode == "insufficient_content"+ assert dumps.row.transcript == TRANSCRIPT+ assert dumps.row.audioPath is not None+ extraction["upsert_business_understanding"].assert_not_awaited()+++def test_quality_rejected_typed_finalize_reports_the_error_code(+ dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+):+ quality_gate.return_value = "insufficient_content"++ response = finalize(input_mode="typed", text="hello hello testing")++ assert response.status_code == 200+ body = response.json()+ assert body["status"] == BrainDumpStatus.failed+ assert body["error_code"] == "insufficient_content"+ extraction["upsert_business_understanding"].assert_not_awaited()++def test_a_duration_past_the_ceiling_is_rejected(stt_create: AsyncMock):"""``duration_secs`` is not just metadata — it bounds the split loop.diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/service.pyindex 18bd8af6d9..0f63d8400d 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py@@ -17,6 +17,7 @@ from backend.api.features.onboarding_dump import (db,intro,prompts,+ quality,recommend,storage,transcription,@@ -182,6 +183,14 @@ async def finalize_voice_dump(transcript=result.text,transcriptLang=result.language,)+ # The transcript is stored *before* the gate so a rejected dump keeps+ # its text (and audio) for recovery and debugging — the gate only+ # decides whether the personalized pipeline below may run on it.+ quality_error = await quality.check_transcript_quality(result.text)+ if quality_error is not None:+ return await _quality_rejection(+ user_id, recording_id, quality_error, BrainDumpInputMode.voice+ )# Finalize only does the audio work. Extraction and the Sonnet# greeting are LLM calls that can outlive the frontend proxy's 30s# request timeout, so they run after this response is sent — the@@ -252,6 +261,16 @@ async def finalize_typed_dump()return _pipeline_response(current.status, current.transcript, current.inputMode)+ # Typed dumps skip transcription but not the gate: a keyboard can+ # produce "asdf asdf asdf" just as easily as a silent mic produces a+ # hallucinated transcript. Only the claim winner reaches this point,+ # so concurrent retries never run duplicate quality checks.+ quality_error = await quality.check_transcript_quality(text)+ if quality_error is not None:+ return await _quality_rejection(+ user_id, recording_id, quality_error, BrainDumpInputMode.typed+ )+background_tasks.add_task(_run_background_jobs,user_id,@@ -264,6 +283,29 @@ async def finalize_typed_dump()+async def _quality_rejection(+ user_id: str,+ recording_id: str,+ error_code: str,+ input_mode: BrainDumpInputMode,+) -> FinalizeResponse:+ """Mark the row failed with the gate's verdict and build the response.++ Shared by the voice and typed paths so the reject contract (log, mark+ failed, surface the error code) cannot drift between them.+ """+ logger.info(+ f"Brain dump ({input_mode}) rejected by quality gate "+ f"for user {user_id} ({error_code})"+ )+ await db.mark_failed(user_id, recording_id, error_code)+ return FinalizeResponse(+ status=BrainDumpStatus.failed,+ input_mode=input_mode,+ error_code=error_code,+ )++async def finalize_skipped_dump(user_id: str, recording_id: str) -> FinalizeResponse:"""Record the skip so the copilot intro can take Path B.@@ -502,6 +544,11 @@ async def get_intro_card(user_id: str) -> IntroCardResponse:dump is Noneor dump.inputMode == BrainDumpInputMode.skippedor not (dump.transcript or "").strip()+ # A quality-rejected dump keeps its transcript on the row for+ # recovery, but that text is by definition not worth reflecting+ # back — falling through to Path A here would greet the user with+ # a fallback built from the very content the gate refused.+ or dump.errorCode in quality.QUALITY_ERROR_CODES):return IntroCardResponse(path="B",diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.pyindex f1ca3d7a74..858030cf29 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py@@ -197,6 +197,18 @@ def extraction(mocker: MockerFixture) -> dict[str, AsyncMock]:return mocks[email protected](autouse=True)+def quality_gate(mocker: MockerFixture) -> AsyncMock:+ """The quality gate passes by default so pipeline tests stay about the+ pipeline; rejection tests flip the return value."""+ mock = AsyncMock(return_value=None)+ mocker.patch(+ "backend.api.features.onboarding_dump.quality.check_transcript_quality",+ new=mock,+ )+ return mock++async def start_voice_take(dumps: DumpStore) -> None:await dumps.start_dump(USER_ID, RECORDING_ID, BrainDumpInputMode.voice)@@ -883,6 +895,119 @@ async def test_intro_card_falls_back_when_the_greeting_was_never_generated(assert card.prompts == intro.fallback_prompts()+async def test_quality_rejected_voice_dump_keeps_data_and_queues_nothing(+ dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+):+ await start_voice_take(dumps)+ quality_gate.return_value = "no_usable_speech"+ background = BackgroundTasks()++ response = await service.finalize_voice_dump(+ USER_ID, RECORDING_ID, 12.0, "audio/webm", background+ )+ await background()++ assert response.status == BrainDumpStatus.failed+ assert response.error_code == "no_usable_speech"+ assert dumps.row is not None+ assert dumps.row.status == BrainDumpStatus.failed+ assert dumps.row.errorCode == "no_usable_speech"+ # Rejection is not deletion: audio and transcript both survive.+ assert dumps.row.audioPath == AUDIO_PATH+ assert dumps.row.transcript == TRANSCRIPT+ # And nothing personalized was queued from the rejected content.+ assert background.tasks == []+ extraction["extract_business_understanding"].assert_not_awaited()+ extraction["upsert_business_understanding"].assert_not_awaited()+++async def test_quality_rejected_typed_dump_records_the_error(+ dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+):+ quality_gate.return_value = "insufficient_content"+ background = BackgroundTasks()++ response = await service.finalize_typed_dump(+ USER_ID, RECORDING_ID, "hello hello testing", background+ )+ await background()++ assert response.status == BrainDumpStatus.failed+ assert response.error_code == "insufficient_content"+ assert dumps.row is not None+ assert dumps.row.status == BrainDumpStatus.failed+ assert dumps.row.errorCode == "insufficient_content"+ assert dumps.row.transcript == "hello hello testing"+ assert background.tasks == []+ extraction["upsert_business_understanding"].assert_not_awaited()+++async def test_repeating_finalize_after_quality_rejection_does_not_rerun_the_gate(+ dumps: DumpStore, quality_gate: AsyncMock, storage_mocks: dict[str, AsyncMock]+):+ """A client retry of the same rejected take gets the stored verdict.++ The part buffer is gone by then, so re-running the pipeline would+ assemble nothing and mark the take lost; the claim guard answers with+ the row's failure instead, without a second quality check.+ """+ await start_voice_take(dumps)+ quality_gate.return_value = "no_usable_speech"+ await finalize_voice()++ response = await finalize_voice()++ assert response.status == BrainDumpStatus.failed+ assert quality_gate.await_count == 1+ storage_mocks["assemble_parts"].assert_awaited_once()+++async def test_a_new_take_after_quality_rejection_runs_the_pipeline(+ dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+):+ await start_voice_take(dumps)+ quality_gate.return_value = "insufficient_content"+ await finalize_voice()++ quality_gate.return_value = None+ await dumps.start_dump(USER_ID, "rec-2", BrainDumpInputMode.voice)+ background = BackgroundTasks()+ response = await service.finalize_voice_dump(+ USER_ID, "rec-2", 30.0, "audio/webm", background+ )+ await background()++ assert response.status == BrainDumpStatus.transcribed+ assert dumps.row is not None+ assert dumps.row.status == BrainDumpStatus.completed+ assert dumps.row.errorCode is None+ extraction["upsert_business_understanding"].assert_awaited_once()+++async def test_intro_card_takes_path_b_for_a_quality_rejected_dump(dumps: DumpStore):+ """The preserved transcript of a rejected dump must never leak into a+ Path A greeting — the gate already decided it isn't worth reflecting."""+ await dumps.start_dump(USER_ID, RECORDING_ID, BrainDumpInputMode.voice)+ await dumps.update_dump(+ USER_ID,+ RECORDING_ID,+ status=BrainDumpStatus.failed,+ transcript="Thank you for watching.",+ errorCode="insufficient_content",+ )++ card = await service.get_intro_card(USER_ID)++ assert card.path == "B"+ assert card.greeting == prompts.PATH_B_GREETING+ assert card.transcript is None++@pytest.mark.asyncioasync def test_intro_generation_degrades_instead_of_raising(mocker: MockerFixture):# The greeting must never be the reason onboarding fails, so a brokendiff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex c458e5b3d8..818eb05992 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -154,6 +154,15 @@ oauth-tool = "backend.cli.oauth_tool:cli"[tool.isort]profile = "black"+[tool.coverage.run]+# Offline eval harnesses: run manually with LLM credentials / ffmpeg+# (see onboarding_dump/EVAL.md), never exercised in CI.+omit = [+ "backend/api/features/onboarding_dump/brain_dump_eval.py",+ "backend/api/features/onboarding_dump/brain_dump_gate_eval.py",+ "backend/api/features/onboarding_dump/eval_corpus.py",+]+[tool.pytest-watcher]now = falseclear = truediff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsxindex d80dbde05c..1d25b4f81a 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx@@ -787,6 +787,72 @@ describe("onboarding brain dump — failure", () => {});});+describe("onboarding brain dump — insufficient content", () => {+ async function recordAndSend() {+ const partUploads: string[] = [];+ server.use(+ getUploadBrainDumpPartMockHandler200(() => {+ partUploads.push("part");+ return {+ recording_id: "r",+ part_index: 0,+ received_bytes: 9,+ total_bytes: 9,+ };+ }),+ );+ mockFlags = { "onboarding-brain-dump": true };+ landOnPainPointsStep();++ render(<OnboardingPage />);+ await screen.findByText(DUMP_HEADLINE);+ await userEvent.click(+ screen.getByRole("button", { name: "Start talking" }),+ );+ await waitFor(() => expect(partUploads).toHaveLength(1));+ await userEvent.click(+ await screen.findByRole("button", { name: "Send recording" }),+ );+ }++ it("shows the recovery choices without advancing or losing the take", async () => {+ finalizeReturns({ status: "failed", error_code: "insufficient_content" });++ await recordAndSend();++ expect(+ await screen.findByText("We didn't catch enough of that."),+ ).toBeDefined();+ // Its own copy, not the transcription-failure screen's.+ expect(screen.queryByText("That didn't go through.")).toBeNull();+ expect(screen.getByRole("button", { name: "Record again" })).toBeDefined();+ expect(screen.getByRole("button", { name: "Type instead" })).toBeDefined();+ expect(+ screen.getByRole("button", { name: "Continue without it" }),+ ).toBeDefined();+ // No accidental advance to the personalized greeting...+ expect(screen.queryByTestId("step-preparing")).toBeNull();+ expect(window.sessionStorage.getItem(INTRO_PATH_KEY)).toBeNull();+ // ...and the local parts survive until the user picks a next move.+ expect(recordingStoreState.parts).toHaveLength(1);+ });++ it("opens the typed composer from the recovery screen", async () => {+ finalizeReturns({ status: "failed", error_code: "no_usable_speech" });++ await recordAndSend();+ await screen.findByText("We didn't catch enough of that.");++ await userEvent.click(screen.getByRole("button", { name: "Type instead" }));++ expect(+ await screen.findByPlaceholderText(+ "What repeats every week? What would you hand off first?",+ ),+ ).toBeDefined();+ });+});+describe("onboarding step map integrity", () => {it("keeps the step constants identical regardless of the brain-dump flag", () => {expect(PAYWALL_FIRST_STEPS).toEqual({diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsxindex dc904ca3a3..e69d37618b 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx@@ -5,6 +5,7 @@ import { Button } from "@/components/atoms/Button/Button";import { Text } from "@/components/atoms/Text/Text";import { cn } from "@/lib/utils";import { FailureState } from "./components/FailureState";+import { InsufficientState } from "./components/InsufficientState";import { DEFAULT_GLASS_PARAMS } from "@/components/molecules/GlassOrb/GlassSurface";import { ElapsedTime } from "./components/ElapsedTime";import { MicButton, OrbScreen } from "./components/MicButton";@@ -19,6 +20,12 @@ import { ringProgress } from "./helpers";import { ScreenState, useBrainDumpStep } from "./useBrainDumpStep";const FAILURE_HEADLINE = "That didn't go through.";+// "Catch" fits a mishearing, not a typed answer — the typed reject asks+// for more instead of implying the system misheard.+const INSUFFICIENT_HEADLINES = {+ voice: "We didn't catch enough of that.",+ typed: "Tell us a bit more.",+} as const;const TIME_LIMIT_CAPTION ="That's 30 minutes — the most we record in one go. Saving all of it…";@@ -29,7 +36,10 @@ export function BrainDumpStep() {const isProcessing = dump.screen === "processing";const isMicScreen = dump.screen === "rest" || isRecording;const isTyping = dump.screen === "typing";- const showSubline = dump.screen !== "failed" && dump.screen !== "recovery";+ const showSubline =+ dump.screen !== "failed" &&+ dump.screen !== "recovery" &&+ dump.screen !== "insufficient";// rest → recording → processing all share one orb, so it is never// unmounted between them: only the glyph and the ring change.const orbScreen = toOrbScreen(dump.screen);@@ -99,7 +109,7 @@ export function BrainDumpStep() {><RevealItem><Text variant="h4">- {dump.screen === "failed" ? FAILURE_HEADLINE : dump.headline}+ {stateHeadline(dump.screen, dump.headline, dump.insufficientMode)}</Text></RevealItem>{showSubline && (@@ -193,6 +203,18 @@ export function BrainDumpStep() {</RevealItem>)}+ {dump.screen === "insufficient" && (+ <RevealItem>+ <InsufficientState+ mode={dump.insufficientMode}+ canRecord={!dump.isMicBlocked}+ onRecordAgain={dump.handleRestart}+ onTypeInstead={dump.showTyping}+ onSkip={dump.handleSkip}+ />+ </RevealItem>+ )}+{dump.screen === "typing" && (<RevealItem className="w-full"><TypedFallback@@ -254,10 +276,24 @@ export function BrainDumpStep() {}function toOrbScreen(screen: ScreenState): OrbScreen | null {- if (screen === "typing" || screen === "recovery") return null;+ // "insufficient" gets no orb: the orb's failed state retries the same+ // take, and re-submitting a rejected take can only be rejected again.+ if (screen === "typing" || screen === "recovery" || screen === "insufficient")+ return null;return screen;}+function stateHeadline(+ screen: ScreenState,+ restHeadline: string,+ insufficientMode: "voice" | "typed",+) {+ if (screen === "failed") return FAILURE_HEADLINE;+ if (screen === "insufficient")+ return INSUFFICIENT_HEADLINES[insufficientMode];+ return restHeadline;+}+function OrbCaption({screen,reachedTimeLimit,diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.tsindex 4ef78b2d09..60d24e1c6e 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts@@ -3,6 +3,7 @@ import {encouragementAt,formatElapsed,headline,+ isInsufficientDump,isPermissionDenied,pickMimeType,recordingFeedbackAt,@@ -130,3 +131,19 @@ describe("isPermissionDenied", () => {expect(isPermissionDenied(null)).toBe(false);});});++describe("isInsufficientDump", () => {+ it("recognises the quality gate's two rejection codes", () => {+ expect(isInsufficientDump("no_usable_speech")).toBe(true);+ expect(isInsufficientDump("insufficient_content")).toBe(true);+ });++ // Every other failure keeps the existing failure screen — including+ // the HTTP status number a non-200 response reports as its code.+ it("does not absorb ordinary failures", () => {+ expect(isInsufficientDump("transcription_failed")).toBe(false);+ expect(isInsufficientDump(undefined)).toBe(false);+ expect(isInsufficientDump(null)).toBe(false);+ expect(isInsufficientDump(500)).toBe(false);+ });+});diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.tsindex 9f1157a877..1f4e2e3028 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts@@ -536,6 +536,117 @@ describe("useBrainDumpStep — finishing a take", () => {});});+describe("useBrainDumpStep — insufficient content", () => {+ function insufficient(code = "insufficient_content") {+ return { status: 200, data: { status: "failed", error_code: code } };+ }++ it("shows the recovery screen instead of the failure one", async () => {+ recorderState.recordingId = "rec-1";+ finalizeBrainDump.mockResolvedValueOnce(insufficient("no_usable_speech"));+ const { result } = await renderStep();++ await act(async () => {+ await result.current.handleDone();+ });++ expect(result.current.screen).toBe("insufficient");+ expect(trackBrainDump).toHaveBeenCalledWith("transcription_failed", {+ error_code: "no_usable_speech",+ });+ // No accidental advance, and nothing personalized to advance to.+ expect(useOnboardingWizardStore.getState().currentStep).toBe(1);+ expect(window.sessionStorage.getItem(INTRO_PATH_KEY)).toBeNull();+ // The local parts are the user's backup until they pick a next move.+ expect(clearRecording).not.toHaveBeenCalled();+ expect(discardBrainDump).not.toHaveBeenCalled();+ });++ it("keeps a normal transcription failure on the failure screen", async () => {+ recorderState.recordingId = "rec-1";+ finalizeBrainDump.mockResolvedValueOnce({+ status: 200,+ data: { status: "failed", error_code: "transcription_failed" },+ });+ const { result } = await renderStep();++ await act(async () => {+ await result.current.handleDone();+ });++ expect(result.current.screen).toBe("failed");+ });++ it("recovers into a fresh take when the user records again", async () => {+ recorderState.recordingId = "rec-1";+ finalizeBrainDump.mockResolvedValueOnce(insufficient());+ const { result } = await renderStep();+ await act(async () => {+ await result.current.handleDone();+ });+ expect(result.current.screen).toBe("insufficient");++ await act(async () => {+ await result.current.handleRestart();+ });++ // The rejected take is discarded only now — after the user chose.+ expect(clearRecording).toHaveBeenCalledWith("rec-1");+ expect(discardBrainDump).toHaveBeenCalledWith({ recording_id: "rec-1" });+ expect(result.current.screen).toBe("recording");+ });++ it("opens the composer when the user types instead", async () => {+ recorderState.recordingId = "rec-1";+ finalizeBrainDump.mockResolvedValueOnce(insufficient());+ const { result } = await renderStep();+ await act(async () => {+ await result.current.handleDone();+ });++ act(() => {+ result.current.showTyping();+ });++ expect(result.current.screen).toBe("typing");+ });++ it("still lets the user skip through to the generic path", async () => {+ recorderState.recordingId = "rec-1";+ finalizeBrainDump.mockResolvedValueOnce(insufficient());+ const { result } = await renderStep();+ await act(async () => {+ await result.current.handleDone();+ });++ await act(async () => {+ await result.current.handleSkip();+ });++ expect(window.sessionStorage.getItem(INTRO_PATH_KEY)).toBe("B");+ expect(useOnboardingWizardStore.getState().currentStep).toBe(2);+ });++ it("returns a typed dump to recovery with the text preserved", async () => {+ finalizeBrainDump.mockResolvedValueOnce(insufficient());+ const { result } = await renderStep();+ act(() => {+ result.current.showTyping();+ });+ act(() => {+ result.current.setTypedText("hello hello testing");+ });++ await act(async () => {+ await result.current.handleSubmitTyped();+ });++ expect(result.current.screen).toBe("insufficient");+ expect(result.current.typedText).toBe("hello hello testing");+ expect(useOnboardingWizardStore.getState().currentStep).toBe(1);+ });+});+describe("useBrainDumpStep — restart", () => {it("throws away the old take locally and on the server before listening again", async () => {recorderState.recordingId = "rec-old";diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/InsufficientState.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/InsufficientState.tsxnew file mode 100644index 0000000000..2489c4005a--- /dev/null+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/InsufficientState.tsx@@ -0,0 +1,64 @@+"use client";++import { Button } from "@/components/atoms/Button/Button";+import { Text } from "@/components/atoms/Text/Text";++interface Props {+ mode: "voice" | "typed";+ canRecord: boolean;+ onRecordAgain: () => void;+ onTypeInstead: () => void;+ onSkip: () => void;+}++// The take went through — it just didn't say enough to personalize from.+// Calm and specific on purpose: nothing broke, nothing was lost, and all+// three ways forward stay open. The copy and the primary action follow+// how the user got here: a voice reject leads with a fresh take, a typed+// reject leads back to their preserved text.+export function InsufficientState({+ mode,+ canRecord,+ onRecordAgain,+ onTypeInstead,+ onSkip,+}: Props) {+ const recordIsPrimary = mode === "voice" && canRecord;+ return (+ <div+ role="alert"+ className="flex max-w-md flex-col items-center gap-4 text-center"+ >+ <Text variant="lead" className="!text-base !text-zinc-500">+ {mode === "voice"+ ? "We heard you, but there wasn't enough about your work to " ++ "personalize things yet. A few sentences about what you do — " ++ "in any language — is all it takes."+ : "There wasn't enough about your work to personalize things " ++ "yet. A few sentences about what you do — in any language — " ++ "is all it takes."}+ </Text>+ <div className="flex flex-wrap items-center justify-center gap-3">+ {canRecord && (+ <Button+ variant={recordIsPrimary ? "primary" : "secondary"}+ size="small"+ onClick={onRecordAgain}+ >+ Record again+ </Button>+ )}+ <Button+ variant={recordIsPrimary ? "secondary" : "primary"}+ size="small"+ onClick={onTypeInstead}+ >+ {mode === "typed" ? "Add more detail" : "Type instead"}+ </Button>+ <Button variant="ghost" size="small" onClick={onSkip}>+ Continue without it+ </Button>+ </div>+ </div>+ );+}diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.tsindex bece5a79e1..c4908a0d27 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts@@ -103,3 +103,19 @@ export function headline(name: string) {? `What keeps stealing your week, ${trimmed}?`: "What keeps stealing your week?";}++// The backend's quality gate rejects dumps with these codes when the+// transcription succeeded but carried nothing to personalize from —+// silence, filler, or an STT hallucination. Distinct from a transcription+// failure: the take went through fine, it just didn't say enough.+const INSUFFICIENT_DUMP_ERROR_CODES = [+ "no_usable_speech",+ "insufficient_content",+] as const;++export function isInsufficientDump(errorCode: unknown) {+ return (+ typeof errorCode === "string" &&+ INSUFFICIENT_DUMP_ERROR_CODES.some((code) => code === errorCode)+ );+}diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.tsindex e941a6fa54..d90deda1bd 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts@@ -7,7 +7,7 @@ import { useEffect, useRef, useState } from "react";import * as Sentry from "@sentry/nextjs";import { useOnboardingWizardStore } from "../../store";import { trackBrainDump } from "@/services/onboarding/brain-dump-analytics";-import { headline, SILENCE_NUDGE_SECONDS } from "./helpers";+import { headline, isInsufficientDump, SILENCE_NUDGE_SECONDS } from "./helpers";import {clearRecording,getMetaById,@@ -22,7 +22,8 @@ export type ScreenState =| "processing"| "typing"| "recovery"- | "failed";+ | "failed"+ | "insufficient";export function useBrainDumpStep() {const name = useOnboardingWizardStore((s) => s.name);@@ -31,6 +32,11 @@ export function useBrainDumpStep() {const recorder = useBrainDumpRecorder();const [screen, setScreen] = useState<ScreenState>("rest");+ // Which submission the quality gate last rejected — the recovery+ // screen's copy and primary action adapt to how the user got there.+ const [insufficientMode, setInsufficientMode] = useState<"voice" | "typed">(+ "voice",+ );const [typedText, setTypedText] = useState("");const [reachedTimeLimit, setReachedTimeLimit] = useState(false);@@ -200,13 +206,16 @@ export function useBrainDumpStep() {input_mode: "voice",});if (response.status !== 200 || response.data.status === "failed") {- trackBrainDump("transcription_failed", {- error_code:- response.status === 200- ? response.data.error_code- : response.status,- });- setScreen("failed");+ const errorCode =+ response.status === 200 ? response.data.error_code : response.status;+ const rejected = isInsufficientDump(errorCode);+ trackBrainDump("transcription_failed", { error_code: errorCode });+ // The recording itself went through — it just didn't carry enough+ // to personalize from, so the recovery screen offers a fresh take+ // instead of a retry of this one. Nothing is cleared until the+ // user picks their next move.+ if (rejected) setInsufficientMode("voice");+ setScreen(rejected ? "insufficient" : "failed");return;}} catch {@@ -293,7 +302,13 @@ export function useBrainDumpStep() {// the voice path — advancing on it would hand the user a copilot// home built from nothing.if (response.status !== 200 || response.data.status === "failed") {- setScreen("failed");+ const errorCode =+ response.status === 200 ? response.data.error_code : response.status;+ const rejected = isInsufficientDump(errorCode);+ if (rejected) setInsufficientMode("typed");+ // The typed text stays in the composer, so "Type instead" from+ // the recovery screen reopens it with nothing lost.+ setScreen(rejected ? "insufficient" : "failed");return;}} catch {@@ -440,6 +455,7 @@ export function useBrainDumpStep() {return {headline: headline(name),screen,+ insufficientMode,typedText,setTypedText,elapsedSeconds: recorder.elapsedSeconds,diff --git a/autogpt_platform/frontend/src/services/onboarding/__tests__/brain-dump-handoff.test.ts b/autogpt_platform/frontend/src/services/onboarding/__tests__/brain-dump-handoff.test.tsindex fb3a42ecf9..99ae6f8517 100644--- a/autogpt_platform/frontend/src/services/onboarding/__tests__/brain-dump-handoff.test.ts+++ b/autogpt_platform/frontend/src/services/onboarding/__tests__/brain-dump-handoff.test.ts@@ -1,4 +1,4 @@-import { beforeEach, describe, expect, it } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";import {clearWelcomePending,peekCapabilityCardsSeen,@@ -186,3 +186,25 @@ describe("capability cards seen", () => {expect(peekGreetingDone("user-1")).toBe(false);});});++describe("unavailable session storage", () => {+ it("setting a flag never throws, so the action it decorates proceeds", () => {+ vi.spyOn(window.sessionStorage, "setItem").mockImplementation(() => {+ throw new Error("storage disabled");+ });++ expect(() => setMicGlow()).not.toThrow();++ vi.restoreAllMocks();+ });++ it("taking a flag falls back to false instead of throwing", () => {+ vi.spyOn(window.sessionStorage, "getItem").mockImplementation(() => {+ throw new Error("storage disabled");+ });++ expect(takeMicGlow()).toBe(false);++ vi.restoreAllMocks();+ });+});diff --git a/autogpt_platform/frontend/src/services/onboarding/brain-dump-handoff.ts b/autogpt_platform/frontend/src/services/onboarding/brain-dump-handoff.tsindex df6bd8e1c0..b2fcfe1619 100644--- a/autogpt_platform/frontend/src/services/onboarding/brain-dump-handoff.ts+++ b/autogpt_platform/frontend/src/services/onboarding/brain-dump-handoff.ts@@ -102,14 +102,23 @@ export function setGreetingDone(userId: string | null | undefined) {function setFlag(key: string) {if (typeof window === "undefined") return;- window.sessionStorage.setItem(key, "1");+ try {+ window.sessionStorage.setItem(key, "1");+ } catch {+ // Storage can be unavailable (private mode, quota); attribution is+ // optional and must never block the action it decorates.+ }}function takeFlag(key: string) {if (typeof window === "undefined") return false;- const value = window.sessionStorage.getItem(key) === "1";- if (value) window.sessionStorage.removeItem(key);- return value;+ try {+ const value = window.sessionStorage.getItem(key) === "1";+ if (value) window.sessionStorage.removeItem(key);+ return value;+ } catch {+ return false;+ }}export function setIntroPath(path: IntroPath) {