AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): delete onboarding voice recordings once transcribed (#15485)
Krzysztof Czerwinskic9d7cb01feef
diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/db.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/db.pyindex 033d4efe8c..cfe94bc9a8 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/db.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/db.py@@ -188,3 +188,28 @@ async def mark_greeting_seen(user_id: str) -> None:"update": {"greetingSeen": True},},)+++# Postgres caps a statement at 32,767 bind parameters, and the first sweep+# after this shipped clears every recording ever stored.+_CLEAR_AUDIO_BATCH = 1000+++async def clear_audio_paths(audio_paths: list[str]) -> int:+ """Forget recordings that are about to be deleted from storage.++ Only ``audioPath`` goes: the transcript is kept while the account+ exists, and the size and duration say nothing about what was said.+ Returns how many rows pointed at one of ``audio_paths``.+ """+ batches = [+ audio_paths[start : start + _CLEAR_AUDIO_BATCH]+ for start in range(0, len(audio_paths), _CLEAR_AUDIO_BATCH)+ ]+ cleared = 0+ for batch in batches:+ cleared += await OnboardingBrainDump.prisma().update_many(+ where={"audioPath": {"in": batch}},+ data={"audioPath": None},+ )+ return cleareddiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/db_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/db_test.pyindex b554815b7b..5af5d33b06 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/db_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/db_test.py@@ -239,3 +239,40 @@ async def test_a_first_take_creates_the_row(mocker: MockerFixture, upsert: Asyncawait db.start_dump(USER_ID, RECORDING_ID, BrainDumpInputMode.voice)upsert.assert_awaited_once()+++async def test_clear_audio_paths_nulls_only_the_audio_in_batches(+ mocker: MockerFixture,+):+ update_many = AsyncMock(side_effect=lambda **_: 1)+ mocker.patch.object(+ OnboardingBrainDump,+ "prisma",+ MagicMock(return_value=MagicMock(update_many=update_many)),+ )+ paths = [+ f"gcs://bucket/uploads/users/u{i}/x/brain-dump-r.webm" for i in range(1001)+ ]++ cleared = await db.clear_audio_paths(paths)++ assert cleared == 2+ batches = [call.kwargs for call in update_many.await_args_list]+ assert [len(b["where"]["audioPath"]["in"]) for b in batches] == [1000, 1]+ assert all(b["data"] == {"audioPath": None} for b in batches)+++async def test_clear_audio_paths_with_nothing_to_clear_skips_the_query(+ mocker: MockerFixture,+):+ update_many = AsyncMock()+ mocker.patch.object(+ OnboardingBrainDump,+ "prisma",+ MagicMock(return_value=MagicMock(update_many=update_many)),+ )++ assert await db.clear_audio_paths([]) == 0+ update_many.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/models.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/models.pyindex 92f77ca627..43924f41d7 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/models.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/models.py@@ -44,6 +44,19 @@ ALLOWED_AUDIO_MIME_TYPES = frozenset(})+# Stored recordings are named ``brain-dump-{recording_id}.{ext}``. The+# retention sweep finds them in the bucket by this name, so the two must+# stay in step.+RECORDING_FILENAME_PREFIX = "brain-dump-"+AUDIO_FILE_EXTENSIONS = {+ "audio/webm": "webm",+ "audio/mp4": "m4a",+ "audio/mpeg": "mp3",+ "audio/wav": "wav",+ "audio/x-wav": "wav",+ "audio/ogg": "ogg",+}+class UploadPartResponse(BaseModel):recording_id: strdiff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/retention.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/retention.pynew file mode 100644index 0000000000..eeba90b014--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/retention.py@@ -0,0 +1,203 @@+"""Deleting brain-dump voice recordings after the retention window.++The privacy policy promises the recording is deleted as soon as it is+transcribed, within 48 hours at most; the transcript stays while the+account exists. The pipeline deletes a recording once its transcript is+stored (``service.finalize_voice_dump``). This sweep catches every other+one: takes that failed or never finished, which keep their recording so+the user can download it or retry, and transcribed ones whose delete+failed. It works from the bucket rather than from the table:++- A row only points at the user's *current* take. Re-recording resets+ ``audioPath``, and deleting the account cascades the row away, but+ neither touches the blob — so the bucket holds recordings no row knows.+- Recordings stored before this shipped carry no expiry the generic+ cleanup can read (``store_file`` used to drop it), so only their age in+ the bucket says when they are due.++What counts as a recording: ``POST /files/upload`` stores a user's file+under its own name in the same ``uploads/users/{user_id}/{uuid}/`` layout,+so a name like ``brain-dump-<id>.webm`` (the name a downloaded recording+is saved under) proves nothing. ``store_audio`` marks recordings in their+custom metadata, which an upload can't set. Recordings from before the+marker existed are recognised by name, but only if they were created+before ``_UNMARKED_RECORDINGS_BEFORE``, a date on or after the release+that added the marker. Each run counts the name-matched, unmarked objects+it skipped as too new (``unmarked_skipped``), so a recording missed that+way shows up in the log.++Rows that point at an expired recording are cleared *before* the blob is+deleted. If clearing fails nothing is deleted and the next run retries; if+a delete fails the row is already clear and the blob is found again next+run, so the database never points at a recording that is gone. (A blob+that goes some other way can leave a row behind; the download route+answers 404 for that and clears the row.)++Nothing else deletes recordings: the generic expiry cleanup reads only the+first page of the bucket listing. A run that fails raises out of the job,+which APScheduler logs at ERROR, so it reaches Sentry.+"""++import asyncio+import logging+import re+from datetime import datetime, timezone+from typing import Awaitable, Callable++from gcloud.aio import storage as async_gcs_storage+from pydantic import BaseModel++from backend.api.features.onboarding_dump.models import (+ AUDIO_FILE_EXTENSIONS,+ RECORDING_FILENAME_PREFIX,+)+from backend.api.features.onboarding_dump.storage import (+ RECORDING_MARKER_KEY,+ RECORDING_MARKER_VALUE,+ audio_max_age,+)+from backend.util.cloud_storage import get_cloud_storage_handler+from backend.util.gcs_utils import (+ fresh_gcs_client,+ is_not_found_error,+ object_created_at,+)++logger = logging.getLogger(__name__)++# ``store_file`` puts user files at ``uploads/users/{user_id}/{uuid}/{name}``.+_RECORDINGS_GLOB = f"uploads/users/*/*/{RECORDING_FILENAME_PREFIX}*"+_RECORDING_NAME = re.compile(+ r"^uploads/users/[^/]+/[^/]+/"+ + re.escape(RECORDING_FILENAME_PREFIX)+ + r"[A-Za-z0-9_-]{1,64}\.(?:"+ + "|".join(sorted(set(AUDIO_FILE_EXTENSIONS.values())))+ + r")$"+)+# Must not be earlier than the production release that started marking+# recordings: an unmarked recording stored in between would never be+# deleted (``unmarked_skipped`` counts what this date leaves out). Later+# only risks an upload named like a recording, stored after that release+# and before this date.+_UNMARKED_RECORDINGS_BEFORE = datetime(2027, 1, 1, tzinfo=timezone.utc)+_DELETE_CONCURRENCY = 10+# Keeps one run well inside the scheduler's 5-minute timeout. The first runs+# after this shipped have every recording ever stored to delete; at one run+# an hour that is 48,000 a day. Each run takes the most recently due first,+# so a take that just reached its max age never waits behind that backlog.+_MAX_PURGED_PER_RUN = 2000++ClearAudioPaths = Callable[[list[str]], Awaitable[int]]+++class RecordingPurgeResult(BaseModel):+ expired: int = 0+ rows_cleared: int = 0+ deleted: int = 0+ failed: int = 0+ # Named like a recording but unmarked, and created on or after+ # ``_UNMARKED_RECORDINGS_BEFORE``: a user's upload, or a recording the+ # cutoff missed.+ unmarked_skipped: int = 0+++async def purge_expired_recordings(+ clear_audio_paths: ClearAudioPaths,+ now: datetime | None = None,+) -> RecordingPurgeResult:+ """Delete every recording that reached ``audio_max_age()``.++ ``clear_audio_paths`` nulls ``audioPath`` on rows that point at the+ given paths: the database manager's in the scheduler, which has no+ Prisma connection of its own. Logs counts only, never paths: they+ carry user IDs.+ """+ handler = await get_cloud_storage_handler()+ bucket = handler.config.gcs_bucket_name+ if not bucket:+ # Without a bucket ``store_audio`` fails, so nothing was ever stored.+ return RecordingPurgeResult()++ cutoff = (now or datetime.now(timezone.utc)) - audio_max_age()+ async with fresh_gcs_client() as client:+ listing = await _list_expired_recordings(client, bucket, cutoff)+ result = RecordingPurgeResult(unmarked_skipped=listing.unmarked_skipped)+ names = listing.names+ if names:+ result.rows_cleared = await clear_audio_paths(+ [f"gcs://{bucket}/{n}" for n in names]+ )+ outcomes = await _delete_blobs(client, bucket, names)+ result.expired = len(names)+ result.deleted = sum(outcomes)+ result.failed = len(names) - result.deleted++ logger.info(f"[BrainDumpRetention] Purged expired recordings: {result}")+ return result+++class _Listing(BaseModel):+ names: list[str]+ unmarked_skipped: int+++async def _list_expired_recordings(+ client: async_gcs_storage.Storage, bucket: str, cutoff: datetime+) -> _Listing:+ """Up to ``_MAX_PURGED_PER_RUN`` due recordings, newest first.++ The bucket lists in name (user ID) order, so the whole listing is read+ before any are picked.+ """+ due: list[tuple[datetime, str]] = []+ unmarked_skipped = 0+ page_token: str | None = None+ while True:+ params = {"prefix": "uploads/users/", "matchGlob": _RECORDINGS_GLOB}+ if page_token:+ params["pageToken"] = page_token+ page = await client.list_objects(bucket, params=params)+ for item in page.get("items", []):+ if not _RECORDING_NAME.match(item.get("name", "")):+ continue+ created = object_created_at(item)+ if not _is_marked(item) and created >= _UNMARKED_RECORDINGS_BEFORE:+ unmarked_skipped += 1+ continue+ if created <= cutoff:+ due.append((created, item["name"]))+ page_token = page.get("nextPageToken")+ if not page_token:+ break+ due.sort(reverse=True)+ return _Listing(+ names=[name for _, name in due[:_MAX_PURGED_PER_RUN]],+ unmarked_skipped=unmarked_skipped,+ )+++def _is_marked(item: dict) -> bool:+ custom = item.get("metadata") or {}+ return custom.get(RECORDING_MARKER_KEY) == RECORDING_MARKER_VALUE+++async def _delete_blobs(+ client: async_gcs_storage.Storage, bucket: str, names: list[str]+) -> list[bool]:+ semaphore = asyncio.Semaphore(_DELETE_CONCURRENCY)++ async def delete(name: str) -> bool:+ async with semaphore:+ try:+ await client.delete(bucket, name)+ except Exception as e:+ # Already gone is what we wanted.+ if is_not_found_error(e):+ return True+ logger.warning(+ f"[BrainDumpRetention] Recording delete failed: {type(e).__name__}"+ )+ return False+ return True++ return await asyncio.gather(*(delete(name) for name in names))diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/retention_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/retention_test.pynew file mode 100644index 0000000000..4e11fff702--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/retention_test.py@@ -0,0 +1,329 @@+"""The 48-hour sweep that deletes brain-dump voice recordings.++The bucket is a fake listing; ``clear_audio_paths`` is a recorder standing+in for the database manager, so the order of "clear the row" and "delete+the blob" is observable.+"""++from contextlib import asynccontextmanager+from datetime import datetime, timedelta, timezone+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from pytest_mock import MockerFixture++from backend.api.features.onboarding_dump import retention, storage+from backend.util.gcs_utils_test import _URL_CONTAINING_404, _gcs_http_error++BUCKET = "private-bucket"+NOW = datetime(2026, 10, 13, 12, 0, tzinfo=timezone.utc)+# After every unmarked recording is gone: only the marker counts.+LATER = datetime(2027, 2, 1, 12, 0, tzinfo=timezone.utc)+MAX_AGE = timedelta(hours=42)+OLD = "uploads/users/user-1/uuid-a/brain-dump-rec-old.webm"+RECENT = "uploads/users/user-1/uuid-b/brain-dump-rec-new.webm"+MARKER = {storage.RECORDING_MARKER_KEY: storage.RECORDING_MARKER_VALUE}+++def _item(+ name: str, age: timedelta, *, now: datetime = NOW, marked: bool = True+) -> dict[str, Any]:+ created = (now - age).isoformat().replace("+00:00", "Z")+ item: dict[str, Any] = {"name": name, "timeCreated": created}+ if marked:+ item["metadata"] = {"expires_at": "2026-01-01T00:00:00+00:00", **MARKER}+ return item+++class Calls:+ def __init__(self) -> None:+ self.order: list[str] = []+ self.cleared: list[str] = []+++def calls() -> Calls:+ return Calls()+++def client(mocker: MockerFixture, calls: Calls) -> MagicMock:+ client = MagicMock()+ client.list_objects = AsyncMock(return_value={"items": []})++ async def delete(bucket: str, name: str) -> None:+ calls.order.append(f"delete:{name}")++ client.delete = AsyncMock(side_effect=delete)++ @asynccontextmanager+ async def fresh_client():+ yield client++ mocker.patch.object(retention, "fresh_gcs_client", fresh_client)+ mocker.patch.object(retention, "audio_max_age", return_value=MAX_AGE)+ handler = MagicMock()+ handler.config.gcs_bucket_name = BUCKET+ mocker.patch.object(+ retention, "get_cloud_storage_handler", AsyncMock(return_value=handler)+ )+ return client+++def _clear(calls: Calls, rows: int = 1) -> AsyncMock:+ async def clear(paths: list[str]) -> int:+ calls.order.append("clear")+ calls.cleared.extend(paths)+ return rows++ return AsyncMock(side_effect=clear)+++async def test_old_recording_is_cleared_from_its_row_then_deleted(+ client: MagicMock, calls: Calls+):+ client.list_objects.return_value = {+ "items": [+ _item(OLD, MAX_AGE),+ _item(RECENT, MAX_AGE - timedelta(minutes=1)),+ ]+ }++ result = await retention.purge_expired_recordings(_clear(calls), now=NOW)++ assert result == retention.RecordingPurgeResult(+ expired=1, rows_cleared=1, deleted=1, failed=0+ )+ assert calls.cleared == [f"gcs://{BUCKET}/{OLD}"]+ assert calls.order == ["clear", f"delete:{OLD}"]+ params: dict[str, Any] = client.list_objects.call_args.kwargs["params"]+ assert params["prefix"] == "uploads/users/"+++async def test_recent_recordings_and_other_uploads_are_left_alone(+ client: MagicMock, calls: Calls+):+ """Old blobs without expiry metadata are found by age, but only+ recordings: a user's agent-input upload in the same prefix stays."""+ client.list_objects.return_value = {+ "items": [+ _item(RECENT, timedelta(days=1)),+ _item("uploads/users/user-1/uuid-c/report.pdf", timedelta(days=90)),+ _item("uploads/users/user-1/uuid-d/brain-dump-notes.txt", timedelta(90)),+ _item("uploads/users/user-1/brain-dump-x.webm", timedelta(days=90)),+ ]+ }+ clear = _clear(calls)++ result = await retention.purge_expired_recordings(clear, now=NOW)++ assert result == retention.RecordingPurgeResult()+ clear.assert_not_awaited()+ client.delete.assert_not_awaited()+++async def test_unmarked_recordings_from_before_the_marker_are_purged_by_name(+ client: MagicMock, calls: Calls+):+ """Recordings stored before ``store_audio`` marked them, orphans+ included, are only recognisable by their name."""+ client.list_objects.return_value = {+ "items": [_item(OLD, timedelta(days=30), marked=False)]+ }++ result = await retention.purge_expired_recordings(_clear(calls), now=NOW)++ assert result.deleted == 1+ assert calls.order == ["clear", f"delete:{OLD}"]+++async def test_a_user_upload_named_like_a_recording_is_left_alone(+ client: MagicMock, calls: Calls+):+ """A downloaded recording uploaded again through ``/files/upload`` keeps+ its name but not the marker, so schedules and presets using it keep+ working."""+ reuploaded = "uploads/users/user-1/uuid-g/brain-dump-rec-old.webm"+ client.list_objects.return_value = {+ "items": [+ _item(reuploaded, timedelta(days=20), now=LATER, marked=False),+ _item(OLD, timedelta(days=20), now=LATER),+ ]+ }+ clear = _clear(calls)++ result = await retention.purge_expired_recordings(clear, now=LATER)++ assert result.expired == 1+ assert calls.cleared == [f"gcs://{BUCKET}/{OLD}"]+ assert calls.order == ["clear", f"delete:{OLD}"]+++def test_unmarked_recordings_are_recognised_until_2027():+ """Recordings are marked from this PR's release on. The cutoff has to be+ on or after that release, or an unmarked recording stored in between is+ never deleted."""+ assert retention._UNMARKED_RECORDINGS_BEFORE == datetime(+ 2027, 1, 1, tzinfo=timezone.utc+ )+++async def test_an_unmarked_recording_created_at_the_cutoff_is_kept_and_counted(+ client: MagicMock, calls: Calls, mocker: MockerFixture+):+ """Kept, since an upload named like a recording looks the same; counted,+ so a release that landed after the cutoff shows up in the log."""+ caplog_info = mocker.patch.object(retention.logger, "info")+ cutoff = retention._UNMARKED_RECORDINGS_BEFORE+ now = cutoff + timedelta(days=10)+ at_cutoff = "uploads/users/user-3/uuid-h/brain-dump-rec-3.webm"+ client.list_objects.return_value = {+ "items": [_item(at_cutoff, now - cutoff, now=now, marked=False)]+ }+ clear = _clear(calls)++ result = await retention.purge_expired_recordings(clear, now=now)++ assert result == retention.RecordingPurgeResult(unmarked_skipped=1)+ clear.assert_not_awaited()+ client.delete.assert_not_awaited()+ logged = caplog_info.call_args.args[0]+ assert "unmarked_skipped=1" in logged+ assert "user-3" not in logged and "uploads/" not in logged+++async def test_a_marked_recording_newer_than_the_cutoff_is_still_taken(+ client: MagicMock, calls: Calls+):+ cutoff = retention._UNMARKED_RECORDINGS_BEFORE+ now = cutoff + timedelta(days=10)+ newer = "uploads/users/user-4/uuid-i/brain-dump-rec-4.webm"+ client.list_objects.return_value = {+ "items": [_item(newer, timedelta(days=5), now=now)]+ }++ result = await retention.purge_expired_recordings(_clear(calls), now=now)++ assert result == retention.RecordingPurgeResult(+ expired=1, rows_cleared=1, deleted=1, failed=0, unmarked_skipped=0+ )+ assert calls.order == ["clear", f"delete:{newer}"]+++async def test_a_large_backlog_is_spread_over_several_runs(+ client: MagicMock, calls: Calls, mocker: MockerFixture+):+ mocker.patch.object(retention, "_MAX_PURGED_PER_RUN", 1)+ client.list_objects.return_value = {+ "items": [_item(OLD, timedelta(days=60)), _item(RECENT, timedelta(days=61))]+ }++ result = await retention.purge_expired_recordings(_clear(calls), now=NOW)++ assert result.expired == 1+ assert calls.order == ["clear", f"delete:{OLD}"]+++async def test_a_newly_due_take_does_not_wait_behind_the_backlog(+ client: MagicMock, calls: Calls, mocker: MockerFixture+):+ """At first deploy every recording ever stored is due, in name (user ID)+ order. A failed take that just reached its max age must still go this+ run, or it outlives 48 hours waiting behind recordings long overdue."""+ mocker.patch.object(retention, "_MAX_PURGED_PER_RUN", 1)+ failed_take = "uploads/users/user-9/uuid-c/brain-dump-rec-failed.webm"+ client.list_objects.side_effect = [+ {"items": [_item(OLD, timedelta(days=60))], "nextPageToken": "page-2"},+ {"items": [_item(failed_take, MAX_AGE + timedelta(minutes=5))]},+ ]++ result = await retention.purge_expired_recordings(_clear(calls), now=NOW)++ assert result.expired == 1+ assert calls.order == ["clear", f"delete:{failed_take}"]+++async def test_blob_already_gone_counts_as_deleted(client: MagicMock, calls: Calls):+ client.list_objects.return_value = {"items": [_item(OLD, timedelta(days=30))]}+ client.delete.side_effect = _gcs_http_error(404, _URL_CONTAINING_404)++ result = await retention.purge_expired_recordings(_clear(calls), now=NOW)++ assert result.deleted == 1+ assert result.failed == 0+++async def test_failed_delete_is_counted_and_the_rest_still_go(+ client: MagicMock, calls: Calls+):+ other = "uploads/users/user-2/uuid-e/brain-dump-rec-2.m4a"+ client.list_objects.return_value = {+ "items": [_item(OLD, timedelta(days=8)), _item(other, timedelta(days=8))]+ }++ async def delete(bucket: str, name: str) -> None:+ if name == OLD:+ raise _gcs_http_error(503, _URL_CONTAINING_404)++ client.delete.side_effect = delete++ result = await retention.purge_expired_recordings(_clear(calls, rows=2), now=NOW)++ assert result == retention.RecordingPurgeResult(+ expired=2, rows_cleared=2, deleted=1, failed=1+ )+++async def test_nothing_is_deleted_when_the_rows_cannot_be_cleared(+ client: MagicMock,+):+ client.list_objects.return_value = {"items": [_item(OLD, timedelta(days=8))]}++ with pytest.raises(RuntimeError):+ await retention.purge_expired_recordings(+ AsyncMock(side_effect=RuntimeError("db down")), now=NOW+ )+ client.delete.assert_not_awaited()+++async def test_every_page_of_the_bucket_is_swept(client: MagicMock, calls: Calls):+ """The first run after this ships purges every recording ever stored."""+ second = "uploads/users/user-2/uuid-f/brain-dump-rec-2.webm"+ client.list_objects.side_effect = [+ {"items": [_item(OLD, timedelta(days=60))], "nextPageToken": "page-2"},+ {"items": [_item(second, timedelta(days=60))]},+ ]++ result = await retention.purge_expired_recordings(_clear(calls, rows=1), now=NOW)++ assert result.expired == 2+ assert result.deleted == 2+ assert client.list_objects.call_args.kwargs["params"]["pageToken"] == "page-2"+++async def test_no_bucket_means_nothing_to_purge(mocker: MockerFixture):+ handler = MagicMock()+ handler.config.gcs_bucket_name = ""+ mocker.patch.object(+ retention, "get_cloud_storage_handler", AsyncMock(return_value=handler)+ )+ clear = AsyncMock()++ assert await retention.purge_expired_recordings(clear) == (+ retention.RecordingPurgeResult()+ )+ clear.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes.pyindex 4e7d36431c..cc93e4a675 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes.py@@ -98,8 +98,7 @@ async def upload_brain_dump_part(raise fastapi.HTTPException(status_code=413,detail=(- "Recording exceeds the "- f"{MAX_RECORDING_BYTES // (1024 * 1024)} MB limit"+ f"Recording exceeds the {MAX_RECORDING_BYTES // (1024 * 1024)} MB limit"),)@@ -119,8 +118,7 @@ async def upload_brain_dump_part(raise fastapi.HTTPException(status_code=413,detail=(- "Recording exceeds the "- f"{MAX_RECORDING_BYTES // (1024 * 1024)} MB limit"+ f"Recording exceeds the {MAX_RECORDING_BYTES // (1024 * 1024)} MB limit"),)@@ -256,14 +254,37 @@ async def complete_brain_dump_greeting(async def download_brain_dump_recording(user_id: Annotated[str, Security(get_user_id)],) -> RedirectResponse:- """Hand back the user's own recording.-- Kept working even when ``status`` is ``failed`` — "your recording is- safe" has to be true, and this is what makes it checkable.+ """Hand back the user's own recording, for as long as it is kept.++ Meant for a take that failed: "your recording is safe" has to be+ true, and this is what makes it checkable. A transcribed take has no+ recording left, since only the transcript is kept. A failed take's+ recording is deleted within 48 hours (see ``retention.py``), and one+ that is due is refused even before the sweep reaches it; a signed URL+ for one that is gone would only fail at the bucket, so say so here+ instead."""dump = await db.get_dump(user_id)if dump is None or not dump.audioPath:raise fastapi.HTTPException(status_code=404, detail="No recording found")+ try:+ available = await storage.audio_available(dump.audioPath)+ except Exception as e:+ # Type only: a GCS error carries the object URL, which holds the+ # user ID, and so would the traceback of a 500.+ logger.warning(+ f"Brain dump recording check failed at the bucket: {type(e).__name__}"+ )+ raise fastapi.HTTPException(+ status_code=503, detail="Recording temporarily unavailable, try again"+ ) from None+ if not available:+ # So ``has_audio`` stops offering it, even if the blob went some way+ # other than the sweep, which clears the row itself.+ await db.clear_audio_paths([dump.audioPath])+ raise fastapi.HTTPException(+ status_code=404, detail="Recording no longer available"+ )url = await storage.audio_download_url(user_id, dump.audioPath)return RedirectResponse(url=url, status_code=307)diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.pyindex 19f476e097..d3ddd8577c 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/routes_test.py@@ -24,6 +24,7 @@ from backend.api.features.onboarding_dump.models import ()from backend.api.features.store.exceptions import VirusDetectedErrorfrom backend.data.understanding import BusinessUnderstandingInput+from backend.util.gcs_utils_test import _gcs_http_errorapp = fastapi.FastAPI()app.include_router(routes.router)@@ -104,6 +105,12 @@ class DumpStore:self.statuses.append(status)return True+ async def clear_audio_paths(self, audio_paths: list[str]) -> int:+ if self.row is None or self.row.audioPath not in audio_paths:+ return 0+ self.row.audioPath = None+ return 1+async def claim_transition(self,user_id: str,@@ -187,6 +194,7 @@ def dumps(mocker: MockerFixture) -> DumpStore:mocker.patch(f"{module}.mark_failed", new=store.mark_failed)mocker.patch(f"{module}.claim_transition", new=store.claim_transition)mocker.patch(f"{module}.mark_greeting_seen", new=store.mark_greeting_seen)+ mocker.patch(f"{module}.clear_audio_paths", new=store.clear_audio_paths)return store@@ -200,6 +208,8 @@ def storage_mocks(mocker: MockerFixture) -> dict[str, AsyncMock]:"store_audio": AsyncMock(return_value="brain-dumps/rec-1.webm"),"discard_parts": AsyncMock(),"audio_download_url": AsyncMock(return_value="https://signed.example/audio"),+ "audio_available": AsyncMock(return_value=True),+ "delete_audio": AsyncMock(),}for name, mock in mocks.items():mocker.patch(f"{module}.{name}", new=mock)@@ -294,7 +304,10 @@ def finalize(**overrides: Any):def test_parts_upload_then_finalize_completes_with_full_transcript(- dumps: DumpStore, stt_create: AsyncMock, extraction: dict[str, AsyncMock]+ dumps: DumpStore,+ stt_create: AsyncMock,+ extraction: dict[str, AsyncMock],+ storage_mocks: dict[str, AsyncMock],):part = upload_part()assert part.status_code == 200@@ -328,7 +341,9 @@ def test_parts_upload_then_finalize_completes_with_full_transcript(]assert dumps.row is not Noneassert dumps.row.transcript == TRANSCRIPT- assert dumps.row.audioPath == "brain-dumps/rec-1.webm"+ # Only the transcript is kept: the recording went once it was stored.+ assert dumps.row.audioPath is None+ storage_mocks["delete_audio"].assert_awaited_once_with("brain-dumps/rec-1.webm")stt_create.assert_awaited_once()extraction["upsert_business_understanding"].assert_awaited_once()@@ -448,7 +463,9 @@ def test_quality_rejected_voice_finalize_reports_the_error_code():"""The API state the frontend branches on: a 200 envelope with``failed`` + a quality error code, distinct from a transcription- failure — with transcript and audio still on the row."""+ failure — with the transcript still on the row. The recording is not:+ it went once the transcript was stored, and a re-record starts a new+ take from the browser."""upload_part()quality_gate.return_value = "insufficient_content"@@ -462,7 +479,7 @@ def test_quality_rejected_voice_finalize_reports_the_error_code(assert dumps.row.status == BrainDumpStatus.failedassert dumps.row.errorCode == "insufficient_content"assert dumps.row.transcript == TRANSCRIPT- assert dumps.row.audioPath is not None+ assert dumps.row.audioPath is Noneextraction["upsert_business_understanding"].assert_not_awaited()@@ -518,7 +535,7 @@ def test_skipped_finalize_records_the_skip_without_a_transcript(dumps: DumpStoreassert dumps.row.transcript is None-def test_status_and_recording_reflect_the_stored_dump(dumps: DumpStore):+def test_a_transcribed_take_offers_no_recording(dumps: DumpStore):upload_part()finalize()@@ -528,17 +545,81 @@ def test_status_and_recording_reflect_the_stored_dump(dumps: DumpStore):"status": BrainDumpStatus.completed,"input_mode": BrainDumpInputMode.voice,"error_code": None,- "has_audio": True,+ "has_audio": False,# The greeting is stored in the same update that completes the# dump, so a completed voice dump is always greeting-ready."greeting_ready": True,}+ recording = client.get(RECORDING_URL, follow_redirects=False)+ assert recording.status_code == 404+ assert recording.json()["detail"] == "No recording found"+++def test_a_failed_take_keeps_its_recording_downloadable(+ dumps: DumpStore, stt_create: AsyncMock+):+ upload_part()+ stt_create.side_effect = RuntimeError("provider down")+ finalize()++ status = client.get(STATUS_URL)+ assert status.json()["status"] == BrainDumpStatus.failed+ assert status.json()["has_audio"] is True+recording = client.get(RECORDING_URL, follow_redirects=False)assert recording.status_code == 307assert recording.headers["location"] == "https://signed.example/audio"+def test_recording_deleted_by_retention_is_404_not_a_dead_link(+ dumps: DumpStore, storage_mocks: dict[str, AsyncMock], stt_create: AsyncMock+):+ upload_part()+ stt_create.side_effect = RuntimeError("provider down")+ finalize()+ storage_mocks["audio_available"].return_value = False++ recording = client.get(RECORDING_URL, follow_redirects=False)++ assert recording.status_code == 404+ assert recording.json()["detail"] == "Recording no longer available"+ storage_mocks["audio_download_url"].assert_not_awaited()+ # The status stops offering a download that can only 404.+ assert dumps.row is not None and dumps.row.audioPath is None+++def test_recording_check_failing_at_the_bucket_is_503_without_the_path(+ dumps: DumpStore,+ storage_mocks: dict[str, AsyncMock],+ stt_create: AsyncMock,+ mocker: MockerFixture,+ test_user_id: str,+):+ """A GCS outage or auth error must not become a 500 whose traceback+ carries the object URL, and with it the user ID."""+ upload_part()+ stt_create.side_effect = RuntimeError("provider down")+ finalize()+ storage_mocks["audio_available"].side_effect = _gcs_http_error(+ 503, f"https://storage.googleapis.com/storage/v1/b/b/o/{test_user_id}%2Fa.webm"+ )+ warning = mocker.patch.object(routes.logger, "warning")++ recording = client.get(RECORDING_URL, follow_redirects=False)++ assert recording.status_code == 503+ assert recording.json()["detail"] == (+ "Recording temporarily unavailable, try again"+ )+ storage_mocks["audio_download_url"].assert_not_awaited()+ # Still the user's recording: the row keeps pointing at it.+ assert dumps.row is not None and dumps.row.audioPath is not None+ logged = " ".join(str(c.args) for c in warning.call_args_list)+ assert "ClientResponseError" in logged+ assert test_user_id not in logged and "storage.googleapis.com" not in logged++def test_discard_clears_the_part_buffer_of_an_unfinished_take(dumps: DumpStore, storage_mocks: dict[str, AsyncMock]):diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/service.pyindex bd6e2e3f97..c93110c33e 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/service.py@@ -1,8 +1,10 @@"""The brain-dump pipeline: parts → audio → transcript → understanding.Every step advances ``OnboardingBrainDump.status`` so the loading screen-can narrate honestly, and no failure ever deletes the audio — a failed-dump stays downloadable and retryable.+can narrate honestly. Only the transcript is kept: the recording is deleted+as soon as the transcript is stored. Until then no failure deletes it, so a+failed take stays downloadable and retryable until the retention sweep+takes it (``storage.AUDIO_RETENTION``)."""import asyncio@@ -28,6 +30,8 @@ from backend.api.features.onboarding_dump import (transcription,)from backend.api.features.onboarding_dump.models import (+ AUDIO_FILE_EXTENSIONS,+ RECORDING_FILENAME_PREFIX,ExpertRecommendations,FinalizeResponse,IntroCardResponse,@@ -128,6 +132,13 @@ async def finalize_voice_dump(current.status, current.transcript, BrainDumpInputMode.voice)+ # A retry of a failed take stores the recording again, so the copy the+ # failed attempt stored has to go too once this one is transcribed.+ earlier_audio_path = (+ existing.audioPath+ if existing is not None and existing.recordingId == recording_id+ else None+ )await _admit_processing(user_id, recording_id)audio = await storage.assemble_parts(user_id, recording_id)if not audio:@@ -189,16 +200,22 @@ async def finalize_voice_dump(time.monotonic() - started_at,len(result.text),)- await db.update_dump(+ # The row forgets the recording in the same write that stores the+ # transcript, so it never points at a file that is about to go.+ transcript_saved = await db.update_dump(user_id,recording_id,status=BrainDumpStatus.transcribed,transcript=result.text,transcriptLang=result.language,+ audioPath=None,)+ if transcript_saved:+ await _delete_recordings([audio_path, earlier_audio_path])# The transcript is stored *before* the gate so a rejected dump keeps- # its text (and audio) for recovery and debugging — the gate only- # decides whether the personalized pipeline below may run on it.+ # its text for debugging — the gate only decides whether the+ # personalized pipeline below may run on it. The recording is not kept+ # for a rejected take either: the user re-records from the browser.quality_error = await quality.check_transcript_quality(result.text)if quality_error is not None:return await _quality_rejection(@@ -297,6 +314,26 @@ async def finalize_typed_dump()+async def _delete_recordings(audio_paths: list[str | None]) -> None:+ """Delete a transcribed take's recordings, best effort.++ The transcript is already stored, so a failed delete must not fail the+ take. The row no longer points at the files, and the retention sweep+ deletes them within ``storage.AUDIO_RETENTION``. The log carries no+ paths: they carry user IDs.+ """+ paths = [path for path in audio_paths if path]+ outcomes = await asyncio.gather(+ *(storage.delete_audio(path) for path in paths), return_exceptions=True+ )+ errors = [type(o).__name__ for o in outcomes if isinstance(o, BaseException)]+ if errors:+ logger.warning(+ f"Brain dump recordings not deleted after transcription ({len(errors)} "+ f"of {len(paths)}), left to the retention sweep: {', '.join(errors)}"+ )++async def _admit_processing(user_id: str, recording_id: str) -> None:try:await enforce_personalization_budget(user_id)@@ -700,15 +737,8 @@ def _pipeline_response(def _audio_filename(recording_id: str, mime_type: str | None) -> str:- extensions = {- "audio/webm": "webm",- "audio/mp4": "m4a",- "audio/mpeg": "mp3",- "audio/wav": "wav",- "audio/x-wav": "wav",- "audio/ogg": "ogg",- }- return f"brain-dump-{recording_id}.{extensions.get(mime_type or '', 'webm')}"+ extension = AUDIO_FILE_EXTENSIONS.get(mime_type or "", "webm")+ return f"{RECORDING_FILENAME_PREFIX}{recording_id}.{extension}"async def get_intro_card(user_id: str) -> IntroCardResponse:diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.pyindex ba85930c0a..9034f45853 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/service_test.py@@ -185,6 +185,7 @@ def storage_mocks(mocker: MockerFixture) -> dict[str, AsyncMock]:"assemble_parts": AsyncMock(return_value=b"opus-audio-bytes"),"store_audio": AsyncMock(return_value=AUDIO_PATH),"discard_parts": AsyncMock(),+ "delete_audio": AsyncMock(),}for name, mock in mocks.items():mocker.patch(f"{module}.{name}", new=mock)@@ -249,7 +250,7 @@ async def finalize_voice(duration_secs: float = 12.0, mime_type: str | None = No@pytest.mark.asyncioasync def test_transcription_failure_keeps_the_audio_and_records_the_error(- dumps: DumpStore, transcribe: AsyncMock+ dumps: DumpStore, transcribe: AsyncMock, storage_mocks: dict[str, AsyncMock]):await start_voice_take(dumps)transcribe.side_effect = RuntimeError("provider blew up")@@ -261,8 +262,75 @@ async def test_transcription_failure_keeps_the_audio_and_records_the_error(assert dumps.row is not Noneassert dumps.row.status == BrainDumpStatus.failedassert dumps.row.errorCode == "transcription_failed"- # The zero-lost-recordings guarantee: the audio is still addressable.+ # The zero-lost-recordings guarantee: the audio is still addressable,+ # until the retention sweep's 48 hours are up.assert dumps.row.audioPath == AUDIO_PATH+ storage_mocks["delete_audio"].assert_not_awaited()+++async def test_a_transcribed_take_deletes_its_recording_and_keeps_the_transcript(+ dumps: DumpStore, storage_mocks: dict[str, AsyncMock]+):+ """Only the transcript is kept, so the recording goes the moment the+ transcript is safely stored."""+ await start_voice_take(dumps)++ response = await finalize_voice()++ assert response.status == BrainDumpStatus.transcribed+ storage_mocks["delete_audio"].assert_awaited_once_with(AUDIO_PATH)+ assert dumps.row is not None+ assert dumps.row.audioPath is None+ assert dumps.row.transcript == TRANSCRIPT+ assert dumps.row.status == BrainDumpStatus.completed+++async def test_a_failed_recording_delete_never_fails_the_take(+ dumps: DumpStore,+ storage_mocks: dict[str, AsyncMock],+ caplog: pytest.LogCaptureFixture,+):+ """The retention sweep deletes the file within 48 hours instead."""+ await start_voice_take(dumps)+ storage_mocks["delete_audio"].side_effect = RuntimeError(f"boom {AUDIO_PATH}")++ response = await finalize_voice()++ assert response.status == BrainDumpStatus.transcribed+ assert dumps.row is not None+ assert dumps.row.status == BrainDumpStatus.completed+ assert dumps.row.transcript == TRANSCRIPT+ # The row stops pointing at it either way, so nothing offers it again.+ assert dumps.row.audioPath is None+ [logged] = [+ r.getMessage() for r in caplog.records if "not deleted" in r.getMessage()+ ]+ assert AUDIO_PATH not in logged+ assert USER_ID not in logged+++async def test_a_retried_take_deletes_the_copy_its_failed_attempt_stored(+ dumps: DumpStore, transcribe: AsyncMock, storage_mocks: dict[str, AsyncMock]+):+ """The retry re-uploads the take from the browser and stores it again,+ so once it is transcribed the failed attempt's copy has to go too."""+ await start_voice_take(dumps)+ transcribe.side_effect = RuntimeError("provider blew up")+ await finalize_voice()+ await start_voice_take(dumps)+ transcribe.side_effect = None+ retry_path = "brain-dumps/rec-1-retry.webm"+ storage_mocks["store_audio"].return_value = retry_path++ response = await finalize_voice()++ assert response.status == BrainDumpStatus.transcribed+ deleted = {c.args[0] for c in storage_mocks["delete_audio"].await_args_list}+ assert deleted == {AUDIO_PATH, retry_path}+ assert dumps.row is not None and dumps.row.audioPath is None@pytest.mark.asyncio@@ -597,7 +665,7 @@ async def test_a_voice_finalize_for_a_superseded_take_reports_its_own_take(@pytest.mark.asyncioasync def test_a_take_superseded_mid_transcription_never_writes_to_the_new_row(- dumps: DumpStore, transcribe: AsyncMock+ dumps: DumpStore, transcribe: AsyncMock, storage_mocks: dict[str, AsyncMock]):"""The old take loses the row the moment a second tab claims it.@@ -624,6 +692,9 @@ async def test_a_take_superseded_mid_transcription_never_writes_to_the_new_row(assert dumps.row.transcript is Noneassert dumps.row.greeting is Noneassert dumps.row.status == BrainDumpStatus.recording_uploaded+ # Its transcript was never saved, so its recording is left for the+ # retention sweep rather than deleted on the strength of a lost write.+ storage_mocks["delete_audio"].assert_not_awaited()@pytest.mark.asyncio@@ -1038,8 +1109,11 @@ async def test_intro_card_falls_back_when_the_greeting_was_never_generated(@pytest.mark.asyncio-async def test_quality_rejected_voice_dump_keeps_data_and_queues_nothing(- dumps: DumpStore, quality_gate: AsyncMock, extraction: dict[str, AsyncMock]+async def test_quality_rejected_voice_dump_keeps_its_transcript_and_queues_nothing(+ dumps: DumpStore,+ quality_gate: AsyncMock,+ extraction: dict[str, AsyncMock],+ storage_mocks: dict[str, AsyncMock],):await start_voice_take(dumps)quality_gate.return_value = "no_usable_speech"@@ -1055,9 +1129,12 @@ async def test_quality_rejected_voice_dump_keeps_data_and_queues_nothing(assert dumps.row is not Noneassert dumps.row.status == BrainDumpStatus.failedassert dumps.row.errorCode == "no_usable_speech"- # Rejection is not deletion: audio and transcript both survive.- assert dumps.row.audioPath == AUDIO_PATH+ # The transcript survives the rejection for debugging. The recording+ # does not: it went once the transcript was stored, and the user+ # re-records from the browser rather than from it.assert dumps.row.transcript == TRANSCRIPT+ assert dumps.row.audioPath is None+ storage_mocks["delete_audio"].assert_awaited_once_with(AUDIO_PATH)# And nothing personalized was queued from the rejected content.assert background.tasks == []extraction["extract_business_understanding"].assert_not_awaited()diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/storage.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/storage.pyindex 75ae89c495..392a3455bb 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/storage.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/storage.py@@ -14,10 +14,17 @@ never a recording.import base64import logging+from datetime import datetime, timedelta, timezonefrom typing import Any, Awaitable, castfrom backend.data import redis_clientfrom backend.util.cloud_storage import get_cloud_storage_handler+from backend.util.gcs_utils import (+ blob_created_at,+ fresh_gcs_client,+ is_not_found_error,+ parse_gcs_path,+)logger = logging.getLogger(__name__)@@ -26,9 +33,48 @@ logger = logging.getLogger(__name__)# accumulate.PART_BUFFER_TTL_SECONDS = 6 * 60 * 60-# The audio has to outlive the onboarding session by enough for the user-# to come back and download it. 48h is the ceiling ``store_file`` accepts.-AUDIO_EXPIRATION_HOURS = 48+# Privacy policy: "We delete the recording as soon as it is transcribed+# (within 48 hours at most) and keep the transcript while your account+# exists." The pipeline deletes a recording once its transcript is stored;+# ``retention.py`` deletes every other one (failed, unfinished or orphaned+# takes, and deletes that failed) once it reaches this age.+AUDIO_RETENTION = timedelta(hours=48)++# The blob's own expiry, in the custom metadata the generic cleanup reads.+# Don't count on it: that cleanup lists only the first page of the bucket+# (1,000 objects), so in a bucket of any size it never reaches recordings,+# and the sweep is the only thing that deletes them. It sits a day past the+# sweep's reach, so where the cleanup does reach it, it never races the sweep+# for a recording whose row still points at it.+AUDIO_EXPIRATION_HOURS = int(+ (AUDIO_RETENTION + timedelta(days=1)).total_seconds() // 3600+)++# Set only by ``store_audio``, so a user's own upload that happens to be named+# like a recording (say, one they downloaded) is never taken for one.+RECORDING_MARKER_KEY = "brain_dump_recording"+RECORDING_MARKER_VALUE = "true"+++# How often the scheduler runs the sweep. It is its own job rather than part+# of the generic file cleanup, which every scheduler start re-arms at start ++# interval, so frequent deploys could postpone it past 48 hours. This job+# keeps its next run across restarts (``Scheduler._register_recording_purge``).+RECORDING_PURGE_INTERVAL = timedelta(hours=1)++# A recording just short of the max age at one run goes at the next, once+# that run reaches it. The scheduler gives a run 5 minutes; the rest covers+# a run that starts late.+_PURGE_RUN_ALLOWANCE = timedelta(minutes=30)+++def audio_max_age() -> timedelta:+ """The age at which a recording is deleted and no longer served.++ Taking the sweep's interval and run time off the retention keeps the+ deletion inside ``AUDIO_RETENTION``.+ """+ return AUDIO_RETENTION - RECORDING_PURGE_INTERVAL - _PURGE_RUN_ALLOWANCEdef _buffer_key(user_id: str, recording_id: str, suffix: str) -> str:@@ -125,9 +171,48 @@ async def store_audio(user_id: str, content: bytes, filename: str) -> str:filename,expiration_hours=AUDIO_EXPIRATION_HOURS,user_id=user_id,+ delete_when_expired=True,+ custom_metadata={RECORDING_MARKER_KEY: RECORDING_MARKER_VALUE},)+async def audio_available(audio_path: str, now: datetime | None = None) -> bool:+ """Whether the stored recording can still be handed back.++ It can't once it is gone, nor once it is old enough for the sweep to+ delete: until the next run it is still in the bucket, but the policy+ already counts it as deleted. A path outside the configured private+ bucket can't be served either.+ """+ handler = await get_cloud_storage_handler()+ bucket, blob = parse_gcs_path(audio_path)+ if bucket != handler.config.gcs_bucket_name:+ return False+ created = await blob_created_at(bucket, blob)+ if created is None:+ return False+ return (now or datetime.now(timezone.utc)) - created < audio_max_age()+++async def delete_audio(audio_path: str) -> None:+ """Delete one stored recording; one that is already gone counts as deleted.++ Any other failure raises, for the caller to leave the file to the+ retention sweep. A path outside the configured private bucket is+ refused rather than deleted.+ """+ handler = await get_cloud_storage_handler()+ bucket, blob = parse_gcs_path(audio_path)+ if bucket != handler.config.gcs_bucket_name:+ raise ValueError("Recording is not in the configured private bucket")+ async with fresh_gcs_client() as client:+ try:+ await client.delete(bucket, blob)+ except Exception as e:+ if not is_not_found_error(e):+ raise++async def audio_download_url(user_id: str, audio_path: str) -> str:handler = await get_cloud_storage_handler()return await handler.generate_signed_url(diff --git a/autogpt_platform/backend/backend/api/features/onboarding_dump/storage_test.py b/autogpt_platform/backend/backend/api/features/onboarding_dump/storage_test.pyindex da9b435bf0..da0df2d723 100644--- a/autogpt_platform/backend/backend/api/features/onboarding_dump/storage_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding_dump/storage_test.py@@ -6,13 +6,18 @@ long recording and a silently corrupted or wrongly rejected upload — areexercised against a fake hash rather than mocked away."""+from contextlib import asynccontextmanager+from datetime import datetime, timedelta, timezonefrom typing import Any+from unittest.mock import AsyncMock, MagicMock+import aiohttpimport pytestfrom pytest_mock import MockerFixturefrom redis.cluster import key_slotfrom backend.api.features.onboarding_dump import storage+from backend.util.gcs_utils_test import _URL_CONTAINING_404, _gcs_http_errorUSER_ID = "user-1"RECORDING_ID = "rec-1"@@ -165,3 +170,141 @@ async def test_the_buffer_and_its_size_index_expire_and_are_dropped_together(assert redis.hashes == {}assert await storage.buffered_size(USER_ID, RECORDING_ID) == 0+++def _handler(mocker: MockerFixture, bucket: str = "private-bucket") -> MagicMock:+ handler = MagicMock()+ handler.config.gcs_bucket_name = bucket+ handler.store_file = AsyncMock(return_value="gcs://private-bucket/x")+ mocker.patch.object(+ storage, "get_cloud_storage_handler", AsyncMock(return_value=handler)+ )+ return handler+++async def test_recordings_are_stored_marked_with_an_expiry(+ mocker: MockerFixture,+):+ handler = _handler(mocker)++ await storage.store_audio(USER_ID, b"audio", "brain-dump-rec-1.webm")++ kwargs = handler.store_file.call_args.kwargs+ # A day past the sweep, so the generic cleanup never races it.+ assert kwargs["expiration_hours"] == 48 + 24+ assert kwargs["delete_when_expired"] is True+ assert kwargs["custom_metadata"] == {+ storage.RECORDING_MARKER_KEY: storage.RECORDING_MARKER_VALUE+ }+ assert kwargs["user_id"] == USER_ID+++def test_a_recording_due_just_after_one_sweep_goes_within_48_hours():+ """The sweep runs once per ``RECORDING_PURGE_INTERVAL``, so a recording+ just short of the max age at one run goes at the next, and that run+ gets half an hour to reach it."""+ assert storage.audio_max_age() == timedelta(hours=46, minutes=30)++ deleted_by = storage.audio_max_age() + storage.RECORDING_PURGE_INTERVAL++ assert deleted_by + timedelta(minutes=30) <= timedelta(hours=48)+++NOW = datetime(2026, 10, 13, 12, 0, tzinfo=timezone.utc)+AUDIO_PATH = "gcs://private-bucket/uploads/users/user-1/u/brain-dump-rec-1.webm"+++async def test_audio_available_checks_the_configured_bucket(mocker: MockerFixture):+ _handler(mocker)+ created = mocker.patch.object(+ storage, "blob_created_at", AsyncMock(return_value=None)+ )++ assert await storage.audio_available(AUDIO_PATH, now=NOW) is False+ created.assert_awaited_once_with(+ "private-bucket", "uploads/users/user-1/u/brain-dump-rec-1.webm"+ )+++async def test_audio_due_for_deletion_is_no_longer_served(mocker: MockerFixture):+ """Still in the bucket until the next sweep, but already past the policy."""+ _handler(mocker)+ created = mocker.patch.object(storage, "blob_created_at", AsyncMock())++ created.return_value = NOW - storage.audio_max_age() + timedelta(minutes=1)+ assert await storage.audio_available(AUDIO_PATH, now=NOW) is True++ created.return_value = NOW - storage.audio_max_age()+ assert await storage.audio_available(AUDIO_PATH, now=NOW) is False+++async def test_audio_in_another_bucket_counts_as_gone(mocker: MockerFixture):+ _handler(mocker)+ created = mocker.patch.object(storage, "blob_created_at", AsyncMock())++ path = "gcs://legacy-bucket/uploads/users/user-1/u/brain-dump-rec-1.webm"+ assert await storage.audio_available(path) is False+ created.assert_not_awaited()+++def _gcs_client(mocker: MockerFixture) -> MagicMock:+ client = MagicMock()+ client.delete = AsyncMock()++ @asynccontextmanager+ async def fresh_client():+ yield client++ mocker.patch.object(storage, "fresh_gcs_client", fresh_client)+ return client+++async def test_delete_audio_removes_the_recording_from_the_bucket(+ mocker: MockerFixture,+):+ _handler(mocker)+ client = _gcs_client(mocker)++ await storage.delete_audio(AUDIO_PATH)++ client.delete.assert_awaited_once_with(+ "private-bucket", "uploads/users/user-1/u/brain-dump-rec-1.webm"+ )+++async def test_delete_audio_counts_a_missing_recording_as_deleted(+ mocker: MockerFixture,+):+ _handler(mocker)+ client = _gcs_client(mocker)+ client.delete.side_effect = _gcs_http_error(404, _URL_CONTAINING_404)++ await storage.delete_audio(AUDIO_PATH)+++async def test_delete_audio_raises_any_other_failure(mocker: MockerFixture):+ """So the caller can log it and leave the file to the retention sweep."""+ _handler(mocker)+ client = _gcs_client(mocker)+ client.delete.side_effect = _gcs_http_error(503, _URL_CONTAINING_404)++ with pytest.raises(aiohttp.ClientResponseError):+ await storage.delete_audio(AUDIO_PATH)+++async def test_delete_audio_never_touches_another_bucket(mocker: MockerFixture):+ _handler(mocker)+ client = _gcs_client(mocker)++ path = "gcs://legacy-bucket/uploads/users/user-1/u/brain-dump-rec-1.webm"+ with pytest.raises(ValueError):+ await storage.delete_audio(path)+ client.delete.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex a3ea42ba95..3e0296d29b 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -40,6 +40,7 @@ from backend.api.features.library.triggers import (setup_triggered_preset,update_triggered_preset,)+from backend.api.features.onboarding_dump import db as onboarding_dump_dbfrom backend.api.features.orgs.db import get_user_default_team, resolve_default_tenancyfrom backend.api.features.search.embeddings import (cleanup_orphaned_embeddings,@@ -469,6 +470,9 @@ class DatabaseManager(AppService):# ============ Onboarding ============ #increment_onboarding_runs = _(increment_onboarding_runs)+ clear_brain_dump_audio_paths = _(+ onboarding_dump_db.clear_audio_paths, name="clear_brain_dump_audio_paths"+ )# ============ OAuth ============ #cleanup_expired_oauth_tokens = _(cleanup_expired_oauth_tokens)@@ -1017,6 +1021,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):# ============ Onboarding ============ #increment_onboarding_runs = d.increment_onboarding_runs+ clear_brain_dump_audio_paths = d.clear_brain_dump_audio_paths# ============ OAuth ============ #cleanup_expired_oauth_tokens = d.cleanup_expired_oauth_tokensdiff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex e506edcfc9..7d861b58a7 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -35,6 +35,11 @@ from pydantic import BaseModel, Field, ValidationErrorfrom sqlalchemy import MetaData, create_engine, or_from backend.api.features.experts.models import ExpertRoutine+from backend.api.features.onboarding_dump.retention import (+ RecordingPurgeResult,+ purge_expired_recordings,+)+from backend.api.features.onboarding_dump.storage import RECORDING_PURGE_INTERVALfrom backend.copilot.active_turns import ConcurrentTurnLimitErrorfrom backend.copilot.at_reset import (AT_RESET_JOB_NAMES,@@ -1392,6 +1397,16 @@ def cleanup_expired_files():run_async(cleanup_expired_files_async())+def purge_expired_recordings_job():+ """Delete brain-dump voice recordings that reached their max age."""+ run_async(_purge_expired_recordings())+++async def _purge_expired_recordings() -> RecordingPurgeResult:+ db = get_database_manager_async_client()+ return await purge_expired_recordings(db.clear_brain_dump_audio_paths)++def execute_community_rebuild(user_id: str, expert_id: str | None = None):"""Weekly community-rebuild cron body of one memory scope (P-1.7).@@ -1478,6 +1493,7 @@ def _morning_briefing_crontab(user_id: str) -> str:_POSTHOG_LIFECYCLE_JOB_ID = "sync_posthog_lifecycles"+RECORDING_PURGE_JOB_ID = "purge_expired_brain_dump_recordings"def _job_timezone_name(job: JobObj) -> str | None:@@ -2952,6 +2968,7 @@ class Scheduler(AppService):# jobstore, so the "leave an unchanged job alone" check can't work.self._register_idle_credential_sweep()self._register_posthog_lifecycle_sweep()+ self._register_recording_purge()self._register_memory_cron_reconcile()self._report_parked_jobs()self._start_schedule_index_backfill()@@ -4166,6 +4183,33 @@ class Scheduler(AppService):jobstore=Jobstores.EXECUTION.value,)+ def _register_recording_purge(self) -> None:+ """The brain-dump recording sweep, which keeps "deleted within 48+ hours" by running every ``RECORDING_PURGE_INTERVAL``.++ Registered like the PostHog sweep: replacing an interval job re-arms+ it at start + interval, so restarts closer together than the+ interval would postpone the sweep indefinitely. An unchanged job is+ left with the next run it has, and an overdue run fires on start.+ """+ trigger = IntervalTrigger(+ seconds=int(RECORDING_PURGE_INTERVAL.total_seconds()),+ timezone=timezone.utc,+ )+ existing = self.scheduler.get_job(+ RECORDING_PURGE_JOB_ID, jobstore=Jobstores.EXECUTION.value+ )+ if existing is not None and str(existing.trigger) == str(trigger):+ return+ self.scheduler.add_job(+ purge_expired_recordings_job,+ trigger,+ id=RECORDING_PURGE_JOB_ID,+ replace_existing=True,+ max_instances=1,+ jobstore=Jobstores.EXECUTION.value,+ )+def _register_memory_cron_reconcile(self) -> None:"""The memory cron reconcile (``reconcile_memory_scope_crons``), thelevel-triggered backstop of the registry's pause, resume and ensurediff --git a/autogpt_platform/backend/backend/executor/scheduler_file_cleanup_test.py b/autogpt_platform/backend/backend/executor/scheduler_file_cleanup_test.pynew file mode 100644index 0000000000..a8ed7b2085--- /dev/null+++ b/autogpt_platform/backend/backend/executor/scheduler_file_cleanup_test.py@@ -0,0 +1,171 @@+"""The scheduled cloud-storage cleanups: voice recordings, and the rest."""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, MagicMock++import pytest+from apscheduler.executors.base import run_job+from apscheduler.jobstores.memory import MemoryJobStore+from apscheduler.schedulers.background import BackgroundScheduler+from apscheduler.triggers.interval import IntervalTrigger+from pytest_mock import MockerFixture++from backend.api.features.onboarding_dump.retention import RecordingPurgeResult+from backend.api.features.onboarding_dump.storage import RECORDING_PURGE_INTERVAL++# Imported here, not in the test: the executor conftest patches a function+# ``db_manager`` wraps at import, and an import under that patch would wrap+# the mock.+from backend.data.db_manager import DatabaseManager, DatabaseManagerAsyncClient+from backend.executor import scheduler+from backend.util.service import EXPOSED_FLAG+++def _run(coro):+ return asyncio.run(coro)+++def test_the_generic_cleanup_leaves_recordings_to_their_own_job(+ mocker: MockerFixture,+):+ purge = mocker.patch.object(scheduler, "purge_expired_recordings", AsyncMock())+ generic = mocker.patch.object(+ scheduler, "cleanup_expired_files_async", AsyncMock(return_value=0)+ )+ mocker.patch.object(scheduler, "run_async", side_effect=_run)++ scheduler.cleanup_expired_files()++ generic.assert_awaited_once()+ purge.assert_not_called()+++def test_the_purge_job_clears_rows_through_the_database_manager(+ mocker: MockerFixture,+):+ db = MagicMock()+ mocker.patch.object(scheduler, "get_database_manager_async_client", return_value=db)+ purge = mocker.patch.object(+ scheduler,+ "purge_expired_recordings",+ AsyncMock(return_value=RecordingPurgeResult()),+ )+ mocker.patch.object(scheduler, "run_async", side_effect=_run)++ scheduler.purge_expired_recordings_job()++ purge.assert_awaited_once_with(db.clear_brain_dump_audio_paths)+++def test_a_failed_purge_is_reported_as_an_error(+ mocker: MockerFixture, caplog: pytest.LogCaptureFixture+):+ """The job raises rather than swallowing the failure, so APScheduler logs+ it at ERROR with the traceback, which Sentry's logging integration+ captures. A sweep that keeps failing has to be seen: nothing else deletes+ the recordings."""+ mocker.patch.object(scheduler, "get_database_manager_async_client")+ mocker.patch.object(+ scheduler,+ "purge_expired_recordings",+ AsyncMock(side_effect=RuntimeError("bucket down")),+ )+ mocker.patch.object(scheduler, "run_async", side_effect=_run)+ background = BackgroundScheduler(+ jobstores={"default": MemoryJobStore()}, timezone=timezone.utc+ )+ background.start(paused=True)+ job = background.add_job(+ scheduler.purge_expired_recordings_job, id=scheduler.RECORDING_PURGE_JOB_ID+ )+ try:+ with caplog.at_level(logging.ERROR, logger="apscheduler"):+ run_job(+ job,+ "default",+ [datetime.now(timezone.utc)],+ "apscheduler.executors.default",+ )+ finally:+ background.shutdown(wait=False)++ [record] = [r for r in caplog.records if r.levelno >= logging.ERROR]+ assert record.exc_info is not None+ assert isinstance(record.exc_info[1], RuntimeError)+++class TestRecordingPurgeRegistration:+ """ "Within 48 hours" holds only if the sweep really runs every+ ``RECORDING_PURGE_INTERVAL``. Re-adding an interval job on every start+ re-arms it at start + interval, so restarts would postpone the sweep;+ an unchanged job is left alone instead, and an overdue run fires on+ start."""++ @pytest.fixture+ def sched(self):+ background = BackgroundScheduler(+ jobstores={scheduler.Jobstores.EXECUTION.value: MemoryJobStore()},+ timezone=timezone.utc,+ )+ background.start(paused=True)+ sched = scheduler.Scheduler.__new__(scheduler.Scheduler)+ sched.scheduler = background+ yield sched+ background.shutdown(wait=False)++ def _job(self, sched):+ return sched.scheduler.get_job(+ scheduler.RECORDING_PURGE_JOB_ID,+ jobstore=scheduler.Jobstores.EXECUTION.value,+ )++ def test_registers_the_purge_at_its_interval(self, sched):+ sched._register_recording_purge()++ job = self._job(sched)+ assert job.func is scheduler.purge_expired_recordings_job+ assert isinstance(job.trigger, IntervalTrigger)+ assert job.trigger.interval == RECORDING_PURGE_INTERVAL++ def test_a_restart_does_not_postpone_an_overdue_purge(self, sched):+ sched._register_recording_purge()+ overdue = datetime.now(timezone.utc) - timedelta(minutes=30)+ sched.scheduler.modify_job(+ scheduler.RECORDING_PURGE_JOB_ID,+ jobstore=scheduler.Jobstores.EXECUTION.value,+ next_run_time=overdue,+ )++ sched._register_recording_purge()++ assert self._job(sched).next_run_time == overdue++ def test_a_job_at_another_interval_is_replaced(self, sched):+ sched.scheduler.add_job(+ scheduler.purge_expired_recordings_job,+ "interval",+ hours=6,+ id=scheduler.RECORDING_PURGE_JOB_ID,+ jobstore=scheduler.Jobstores.EXECUTION.value,+ )++ sched._register_recording_purge()++ assert self._job(sched).trigger.interval == RECORDING_PURGE_INTERVAL+++def test_the_purge_reaches_the_database_manager_endpoint_it_calls():+ """The server serves each exposed method at ``/{attribute name}`` and+ the client POSTs to ``/{function __name__}``; if they differ the purge+ 404s on every run."""+ served = {+ name+ for name, attr in vars(DatabaseManager).items()+ if getattr(attr, EXPOSED_FLAG, False)+ }+ called = DatabaseManagerAsyncClient.clear_brain_dump_audio_paths.__name__++ assert called == "clear_brain_dump_audio_paths"+ assert called in serveddiff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex dc588c5c9d..7866364653 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -21,6 +21,7 @@ from apscheduler.triggers.date import DateTriggerfrom apscheduler.triggers.interval import IntervalTriggerfrom backend.api.features.experts.models import ExpertRoutine+from backend.api.features.onboarding_dump.storage import RECORDING_PURGE_INTERVALfrom backend.copilot.credential_selection import CredentialPinfrom backend.copilot.dream.scope_reconcile import RECONCILE_INTERVAL_MINUTESfrom backend.copilot.executor.utils import ScheduledTurnOrigin@@ -1788,6 +1789,29 @@ def test_memory_cron_reconcile_registered_after_start_on_its_interval(assert kwargs["next_run_time"] <= datetime.now(timezone.utc)+def test_recording_purge_registered_after_start_on_its_interval(monkeypatch):+ """The brain-dump recording sweep keeps "deleted within 48 hours". It is+ registered once the scheduler has started, so an unchanged job keeps its+ next run; a start that stopped calling ``_register_recording_purge``+ would leave no sweep at all."""+ calls = _registered_jobs(monkeypatch, interval_hours=6).scheduler_calls++ names = [name for name, _, _ in calls]+ [at] = [+ i+ for i, (name, args, _) in enumerate(calls)+ if name == "add_job"+ and args+ and args[0] is scheduler_module.purge_expired_recordings_job+ ]+ _, args, kwargs = calls[at]+ assert names.index("start") < at+ assert kwargs["id"] == scheduler_module.RECORDING_PURGE_JOB_ID+ assert args[1].interval == RECORDING_PURGE_INTERVAL+ assert kwargs["max_instances"] == 1+ assert kwargs["jobstore"] == scheduler_module.Jobstores.EXECUTION.value++def test_a_system_job_due_at_start_reaches_the_scheduler(monkeypatch):"""An immediately due system job reaches the service only through``_scheduler_instance``, so that is published before ``start``; publisheddiff --git a/autogpt_platform/backend/backend/util/cloud_storage.py b/autogpt_platform/backend/backend/util/cloud_storage.pyindex aef6c47258..c68495e46b 100644--- a/autogpt_platform/backend/backend/util/cloud_storage.py+++ b/autogpt_platform/backend/backend/util/cloud_storage.py@@ -7,7 +7,7 @@ import loggingimport os.pathimport uuidfrom datetime import datetime, timedelta, timezone-from typing import Tuple+from typing import Any, Tupleimport aiohttpfrom gcloud.aio import storage as async_gcs_storage@@ -22,6 +22,10 @@ from backend.util.settings import Configlogger = logging.getLogger(__name__)+# What ``store_file`` writes for the expiry; the cleanup deletes on+# ``expires_at`` in custom metadata.+_EXPIRY_METADATA_KEYS = ("uploaded_at", "expires_at", "expiration_hours")+class CloudStorageConfig:"""Configuration for cloud storage providers."""@@ -143,6 +147,8 @@ class CloudStorageHandler:expiration_hours: int = 48,user_id: str | None = None,graph_exec_id: str | None = None,+ delete_when_expired: bool = False,+ custom_metadata: dict[str, str] | None = None,) -> str:"""Store file content in cloud storage.@@ -151,9 +157,16 @@ class CloudStorageHandler:content: File content as bytesfilename: Desired filenameprovider: Cloud storage provider ("gcs", "s3", "azure")- expiration_hours: Hours until expiration (1-48, default: 48)+ expiration_hours: Hours until expiration (default: 48)user_id: User ID for user-scoped files (optional)graph_exec_id: Graph execution ID for execution-scoped files (optional)+ delete_when_expired: Let the scheduled cleanup delete the file once+ ``expiration_hours`` have passed. Off by default: files from+ ``/files/upload`` carry an expiry but saved schedules and+ presets keep pointing at them, so they must not be deleted.+ custom_metadata: Extra custom metadata to keep on the object. Only+ the server sets it, so it can mark files a user's own upload+ must not be mistaken for.Note:Provide either user_id OR graph_exec_id, not both. If neither is provided,@@ -164,7 +177,13 @@ class CloudStorageHandler:"""if provider == "gcs":return await self._store_file_gcs(- content, filename, expiration_hours, user_id, graph_exec_id+ content,+ filename,+ expiration_hours,+ user_id,+ graph_exec_id,+ delete_when_expired,+ custom_metadata,)else:raise ValueError(f"Unsupported cloud storage provider: {provider}")@@ -176,6 +195,8 @@ class CloudStorageHandler:expiration_hours: int,user_id: str | None = None,graph_exec_id: str | None = None,+ delete_when_expired: bool = False,+ custom_metadata: dict[str, str] | None = None,) -> str:"""Store file in Google Cloud Storage."""if not self.config.gcs_bucket_name:@@ -185,6 +206,14 @@ class CloudStorageHandler:if user_id and graph_exec_id:raise ValueError("Provide either user_id OR graph_exec_id, not both")+ # Only ``delete_when_expired`` may make a file deletable, at its own+ # ``expiration_hours``.+ reserved = sorted(set(custom_metadata or {}) & set(_EXPIRY_METADATA_KEYS))+ if reserved:+ raise ValueError(+ f"custom_metadata must not set the expiry: {', '.join(reserved)}"+ )+async_client = await self._get_async_gcs_client()# Generate unique path with appropriate scope@@ -203,18 +232,23 @@ class CloudStorageHandler:upload_time = datetime.now(timezone.utc)expiration_time = upload_time + timedelta(hours=expiration_hours)+ expiry = {+ "uploaded_at": upload_time.isoformat(),+ "expires_at": expiration_time.isoformat(),+ "expiration_hours": str(expiration_hours),+ }+ # GCS keeps only what sits under "metadata" as custom metadata, which is+ # the one place the expiry cleanup reads. Top-level keys are dropped, so+ # a file stored without ``delete_when_expired`` is never deleted.+ custom = {**(expiry if delete_when_expired else {}), **(custom_metadata or {})}+ metadata: dict[str, Any] = {} if delete_when_expired else dict(expiry)+ if custom:+ metadata["metadata"] = customawait async_client.upload(self.config.gcs_bucket_name,blob_name,content,- # Top-level keys are not stored as custom metadata, so the expiry- # cleanup never fires. Nesting them under "metadata" would switch it- # on and delete files that saved schedules and presets still use.- metadata={- "uploaded_at": upload_time.isoformat(),- "expires_at": expiration_time.isoformat(),- "expiration_hours": str(expiration_hours),- },+ metadata=metadata,)return f"gcs://{self.config.gcs_bucket_name}/{blob_name}"diff --git a/autogpt_platform/backend/backend/util/cloud_storage_test.py b/autogpt_platform/backend/backend/util/cloud_storage_test.pyindex 3db4f87181..b954209b3e 100644--- a/autogpt_platform/backend/backend/util/cloud_storage_test.py+++ b/autogpt_platform/backend/backend/util/cloud_storage_test.py@@ -2,6 +2,7 @@Tests for cloud storage utilities."""+from datetime import datetime, timedelta, timezonefrom unittest.mock import AsyncMock, MagicMock, patchimport aiohttp@@ -80,11 +81,104 @@ class TestCloudStorageHandler:assert call_args[0][0] == "test-bucket" # bucket nameassert call_args[0][1].startswith("uploads/system/") # blob nameassert call_args[0][2] == content # file content- # Top level on purpose: see the comment in store_file.+ # Top level, which GCS drops: saved schedules and presets keep+ # pointing at /files/upload files, so they must never expire.upload_metadata = call_args.kwargs["metadata"]assert upload_metadata["expires_at"]assert "metadata" not in upload_metadata+ @patch.object(CloudStorageHandler, "_get_async_gcs_client")+ @pytest.mark.asyncio+ async def test_store_file_delete_when_expired_writes_custom_metadata(+ self, mock_get_async_client, handler+ ):+ """The expiry has to land where the cleanup reads it."""+ mock_async_client = AsyncMock()+ mock_get_async_client.return_value = mock_async_client++ before = datetime.now(timezone.utc)+ await handler.store_file(+ b"audio",+ "brain-dump-rec.webm",+ expiration_hours=168,+ user_id="user-1",+ delete_when_expired=True,+ )++ upload_metadata = mock_async_client.upload.call_args.kwargs["metadata"]+ assert set(upload_metadata) == {"metadata"}+ custom = upload_metadata["metadata"]+ expires_at = datetime.fromisoformat(custom["expires_at"])+ assert expires_at - before >= timedelta(hours=168)+ assert expires_at - before < timedelta(hours=168, minutes=1)+ assert custom["expiration_hours"] == "168"++ # And the cleanup deletes it once that time has passed.+ mock_async_client.list_objects = AsyncMock(+ return_value={"items": [{"name": "uploads/users/user-1/x/a.webm"}]}+ )+ mock_async_client.download_metadata = AsyncMock(+ return_value={"metadata": {**custom, "expires_at": before.isoformat()}}+ )+ mock_async_client.delete = AsyncMock()++ assert await handler.delete_expired_files() == 1+ mock_async_client.delete.assert_awaited_once_with(+ "test-bucket", "uploads/users/user-1/x/a.webm"+ )++ @patch.object(CloudStorageHandler, "_get_async_gcs_client")+ @pytest.mark.asyncio+ async def test_store_file_keeps_custom_metadata_whether_or_not_it_may_be_deleted(+ self, mock_get_async_client, handler+ ):+ """Every file carries an expiry; ``delete_when_expired`` only decides+ whether the cleanup can read it. Custom metadata is kept either way."""+ mock_async_client = AsyncMock()+ mock_get_async_client.return_value = mock_async_client+ marker = {"brain_dump_recording": "true"}++ await handler.store_file(+ b"audio",+ "a.webm",+ user_id="user-1",+ delete_when_expired=True,+ custom_metadata=marker,+ )+ custom = mock_async_client.upload.call_args.kwargs["metadata"]["metadata"]+ assert custom["brain_dump_recording"] == "true"+ assert custom["expires_at"]++ await handler.store_file(+ b"audio", "a.webm", user_id="user-1", custom_metadata=marker+ )+ upload_metadata = mock_async_client.upload.call_args.kwargs["metadata"]+ assert upload_metadata["metadata"] == marker++ @pytest.mark.parametrize("delete_when_expired", [False, True])+ @pytest.mark.parametrize("key", ["expires_at", "uploaded_at", "expiration_hours"])+ @patch.object(CloudStorageHandler, "_get_async_gcs_client")+ @pytest.mark.asyncio+ async def test_store_file_refuses_custom_metadata_that_sets_the_expiry(+ self, mock_get_async_client, handler, key, delete_when_expired+ ):+ """Only ``delete_when_expired`` may make a file deletable, and then+ only at its own ``expiration_hours``: an ``expires_at`` slipped in+ through custom metadata would switch the cleanup on for files saved+ schedules and presets still use."""+ mock_async_client = AsyncMock()+ mock_get_async_client.return_value = mock_async_client++ with pytest.raises(ValueError, match=key):+ await handler.store_file(+ b"audio",+ "a.webm",+ user_id="user-1",+ delete_when_expired=delete_when_expired,+ custom_metadata={key: "2020-01-01T00:00:00+00:00"},+ )+ mock_async_client.upload.assert_not_awaited()+@patch("backend.util.cloud_storage.async_gcs_storage.Storage")@pytest.mark.asyncioasync def test_retrieve_file_gcs(self, mock_storage_class, handler):@@ -197,8 +291,6 @@ class TestCloudStorageHandler:@pytest.mark.asyncioasync def test_delete_expired_files_gcs(self, mock_get_async_client, handler):"""Test deleting expired files from GCS."""- from datetime import datetime, timedelta, timezone-# Mock async GCS clientmock_async_client = AsyncMock()mock_get_async_client.return_value = mock_async_client@@ -237,8 +329,6 @@ class TestCloudStorageHandler:@pytest.mark.asyncioasync def test_check_file_expired_gcs(self, mock_get_async_client, handler):"""Test checking if a file has expired."""- from datetime import datetime, timedelta, timezone-# Mock async GCS clientmock_async_client = AsyncMock()mock_get_async_client.return_value = mock_async_clientdiff --git a/autogpt_platform/backend/backend/util/gcs_utils.py b/autogpt_platform/backend/backend/util/gcs_utils.pyindex 3b409be823..2addd72a1f 100644--- a/autogpt_platform/backend/backend/util/gcs_utils.py+++ b/autogpt_platform/backend/backend/util/gcs_utils.py@@ -7,7 +7,9 @@ including path parsing, client management, and signed URL generation.import asyncioimport logging+from contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezone+from typing import AsyncIteratorimport aiohttpfrom gcloud.aio import storage as async_gcs_storage@@ -139,6 +141,50 @@ async def download_range(bucket: str, blob: str, max_bytes: int) -> bytes:await session.close()+@asynccontextmanager+async def fresh_gcs_client() -> AsyncIterator[async_gcs_storage.Storage]:+ """An async client on its own session, closed on exit.++ Same reasoning as ``download_with_fresh_session``: a session created for+ the call can't be tied to an event loop that is gone by the next one.+ """+ session = aiohttp.ClientSession(+ connector=aiohttp.TCPConnector(limit=10, force_close=True)+ )+ client = async_gcs_storage.Storage(session=session)+ try:+ yield client+ finally:+ try:+ await client.close()+ except Exception:+ pass # Best-effort cleanup+ await session.close()+++def object_created_at(item: dict) -> datetime:+ """When GCS created the object, from its ``timeCreated`` field."""+ # RFC 3339 with a trailing "Z", which ``fromisoformat`` only reads from+ # Python 3.11 on.+ created = datetime.fromisoformat(item["timeCreated"].replace("Z", "+00:00"))+ return created if created.tzinfo else created.replace(tzinfo=timezone.utc)+++async def blob_created_at(bucket: str, blob: str) -> datetime | None:+ """When the object was created, or ``None`` if it doesn't exist.++ Errors other than a 404 propagate.+ """+ async with fresh_gcs_client() as client:+ try:+ metadata = await client.download_metadata(bucket, blob)+ except Exception as e:+ if is_not_found_error(e):+ return None+ raise+ return object_created_at(metadata)++async def generate_signed_url(sync_client: gcs_storage.Client,bucket_name: str,diff --git a/autogpt_platform/backend/backend/util/gcs_utils_test.py b/autogpt_platform/backend/backend/util/gcs_utils_test.pyindex 1990e0074d..19b7b67f86 100644--- a/autogpt_platform/backend/backend/util/gcs_utils_test.py+++ b/autogpt_platform/backend/backend/util/gcs_utils_test.py@@ -1,3 +1,4 @@+from datetime import datetime, timezonefrom unittest.mock import AsyncMock, MagicMockimport aiohttp@@ -7,6 +8,7 @@ from multidict import CIMultiDict, CIMultiDictProxyfrom yarl import URLfrom backend.util.gcs_utils import (+ blob_created_at,download_range,download_with_fresh_session,generate_iam_signed_url,@@ -171,3 +173,33 @@ async def test_download_keeps_other_errors_when_url_contains_404(mocker, status)with pytest.raises(aiohttp.ClientResponseError):await download_range("bucket", "blob", 16)+++async def test_blob_created_at_reads_metadata_only(mocker):+ client = _mock_client(mocker, download=AsyncMock())+ client.download_metadata = AsyncMock(+ return_value={"name": "blob", "timeCreated": "2026-10-13T12:00:00.123Z"}+ )++ assert await blob_created_at("bucket", "blob") == datetime(+ 2026, 10, 13, 12, 0, 0, 123000, tzinfo=timezone.utc+ )+ client.download_metadata.assert_awaited_once_with("bucket", "blob")+ client.download.assert_not_awaited()+ client.close.assert_awaited_once()+++async def test_blob_created_at_is_none_only_for_a_real_404(mocker):+ client = _mock_client(mocker, download=AsyncMock())+ client.download_metadata = AsyncMock(+ side_effect=_gcs_http_error(404, _URL_CONTAINING_404)+ )+ assert await blob_created_at("bucket", "missing") is None++ client.download_metadata = AsyncMock(+ side_effect=_gcs_http_error(503, _URL_CONTAINING_404)+ )+ with pytest.raises(aiohttp.ClientResponseError):+ await blob_created_at("bucket", "blob")diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/FailureState.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/FailureState.tsxindex fae3291047..39f7655de8 100644--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/FailureState.tsx+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/FailureState.tsx@@ -9,9 +9,12 @@ interface Props {}// Calm on purpose. The recording is safe in the browser and (once it got-// that far) on the server, so this is an inconvenience, not a loss — the-// copy and the download button both have to make that obvious. Retrying is-// the orb itself, so only the secondary exits live here.+// that far) on the server, which keeps a failed take for 46 hours 30 minutes+// (the hourly sweep then deletes it, within 48 hours), so this is an+// inconvenience, not a loss — the copy and the download button both have to+// make that obvious. Download and retry both read the+// browser's copy. Retrying is the orb itself, so only the secondary exits+// live here.export function FailureState({ onDownload, onSkip }: Props) {return (<div className="flex flex-col items-center gap-4 text-center">diff --git a/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/__tests__/finalize.test.tsx b/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/__tests__/finalize.test.tsxindex 1a42b32c1e..04a505b11c 100644--- a/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/__tests__/finalize.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/__tests__/finalize.test.tsx@@ -163,6 +163,18 @@ describe("server status panel", () => {"/api/proxy/api/onboarding/brain-dump/recording",);});++ it("offers no download once the server has deleted the recording", async () => {+ serveStatus({ status: "completed", input_mode: "voice", has_audio: false });++ render(<BrainDumpDebugPage />);++ const status = panel("Server status");+ expect(await status.findByText("completed")).toBeDefined();+ expect(+ screen.queryByRole("link", { name: /Download server-side recording/ }),+ ).toBeNull();+ });});describe("debug finalize", () => {diff --git a/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/components/ServerStatusPanel.tsx b/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/components/ServerStatusPanel.tsxindex 14c24423a4..ec63c5b045 100644--- a/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/components/ServerStatusPanel.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/dev/brain-dump-debug/components/ServerStatusPanel.tsx@@ -42,12 +42,14 @@ export function ServerStatusPanel({description={`GET /onboarding/brain-dump/status, polled every ${STATUS_POLL_MS}ms.`}icon={AudioWaveformIcon}action={- <Link href={recordingDownloadHref()} isExternal variant="secondary">- <span className="inline-flex items-center gap-2">- <Icon icon={Download01Icon} size={16} />- Download server-side recording- </span>- </Link>+ dump?.has_audio ? (+ <Link href={recordingDownloadHref()} isExternal variant="secondary">+ <span className="inline-flex items-center gap-2">+ <Icon icon={Download01Icon} size={16} />+ Download server-side recording+ </span>+ </Link>+ ) : null}>{isError ? (diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex e0589ce715..35dd794225 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -12518,7 +12518,7 @@"get": {"tags": ["v1", "onboarding", "onboarding", "brain-dump"],"summary": "Download Brain Dump Recording",- "description": "Hand back the user's own recording.\n\nKept working even when ``status`` is ``failed`` — \"your recording is\nsafe\" has to be true, and this is what makes it checkable.",+ "description": "Hand back the user's own recording, for as long as it is kept.\n\nMeant for a take that failed: \"your recording is safe\" has to be\ntrue, and this is what makes it checkable. A transcribed take has no\nrecording left, since only the transcript is kept. A failed take's\nrecording is deleted within 48 hours (see ``retention.py``), and one\nthat is due is refused even before the sweep reaches it; a signed URL\nfor one that is gone would only fail at the bucket, so say so here\ninstead.","operationId": "download_brain_dump_recording","responses": {"200": {