Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop copying the chrome profile template on CDP-connect paths and sweep stale temp artifacts (#7485)
Aaron Perezae23a3601f12
diff --git a/skyvern/config.py b/skyvern/config.pyindex 1d3b0a1dc..94847d884 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -759,6 +759,10 @@ class Settings(BaseSettings):CLEANUP_STALE_TASK_THRESHOLD_HOURS: int = 24"""Tasks/workflows not updated for this many hours are considered stale (stuck)."""+ TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS: float = 48.0+ """Age gate (hours) for the always-on sweep of per-run LOG_PATH/DOWNLOAD_PATH dirs left behind on+ crash paths. Non-positive disables the sweep."""+# Workflow Schedule SettingsENABLE_WORKFLOW_SCHEDULES: bool = True"""Enable recurring workflow schedules in the OSS/local server."""diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex 51f65b055..d4cecb404 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -51,7 +51,12 @@ from skyvern.forge.sdk.services.local_org_auth_token_service import (regenerate_local_api_key,)from skyvern.services.browser_recording.session_registry import interpretation_registry-from skyvern.services.cleanup_service import start_cleanup_scheduler, stop_cleanup_scheduler+from skyvern.services.cleanup_service import (+ start_cleanup_scheduler,+ start_temp_artifact_sweep,+ stop_cleanup_scheduler,+ stop_temp_artifact_sweep,+)from skyvern.services.workflow_schedule_service import (start_workflow_schedule_scheduler,stop_workflow_schedule_scheduler,@@ -307,6 +312,8 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:if cleanup_task:LOG.info("Cleanup scheduler started")+ start_temp_artifact_sweep()+workflow_schedule_task = start_workflow_schedule_scheduler()if workflow_schedule_task:LOG.info("Workflow schedule scheduler started")@@ -327,6 +334,7 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:# Stop cleanup schedulerawait stop_workflow_schedule_scheduler()await stop_cleanup_scheduler()+ await stop_temp_artifact_sweep()await interpretation_registry.stop_all()if forge_app.api_app_shutdown_event:diff --git a/skyvern/services/cleanup_service.py b/skyvern/services/cleanup_service.pyindex 91ed9ac28..ffb1d3f42 100644--- a/skyvern/services/cleanup_service.py+++ b/skyvern/services/cleanup_service.py@@ -8,6 +8,7 @@ This service is responsible for:import asyncioimport shutil+import timefrom datetime import datetime, timedeltafrom pathlib import Path@@ -16,6 +17,8 @@ import structlogfrom skyvern.config import settingsfrom skyvern.forge import app+from skyvern.forge.sdk.artifact.storage.factory import StorageFactory+from skyvern.forge.sdk.artifact.storage.local import LocalStorageLOG = structlog.get_logger()@@ -115,6 +118,107 @@ def cleanup_temp_directory() -> int:return removed_count+def sweep_stale_temp_artifacts(max_age_hours: float | None = None) -> int:+ """+ Remove stale per-run disk artifacts left behind on crash paths: per-day browser+ console-log dirs under LOG_PATH and per-run download dirs under DOWNLOAD_PATH.++ TEMP_PATH is intentionally excluded. Its dominant leak (the CDP-connect profile copy)+ is fixed at the source, and mtime is not a safe liveness signal for what remains there:+ reused generated-script caches (TEMP_PATH/<script_id>) are overwritten in place without+ bumping the dir mtime, and browser-session profile dirs are written only at open/close —+ both read as stale while actively in use. LOG_PATH/DOWNLOAD_PATH are single-tenant and+ keyed per run, so an aged top-level entry there is genuinely finished.++ DOWNLOAD_PATH is swept only when the active storage backend uploads run downloads+ elsewhere (S3/Azure/GCS), leaving the local copy as scratch. On the local backend+ LocalStorage.save_downloaded_files is a no-op and get_downloaded_files serves the files+ in place via file:// URIs, so DOWNLOAD_PATH/<run_id> is the run's permanent artifact+ record — sweeping it would silently delete user data — and it is left untouched.++ Returns:+ Number of entries removed.+ """+ if max_age_hours is None:+ max_age_hours = settings.TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS+ if max_age_hours <= 0:+ return 0++ cutoff = time.time() - max_age_hours * 3600+ removed_count = 0+ bases = [settings.LOG_PATH]+ if not isinstance(StorageFactory.get_storage(), LocalStorage):+ bases.append(settings.DOWNLOAD_PATH)+ for base in bases:+ base_path = Path(base)+ if not base_path.is_dir():+ continue+ try:+ entries = list(base_path.iterdir())+ except OSError:+ LOG.warning("Failed to list directory for temp-artifact sweep", base=str(base_path), exc_info=True)+ continue+ for entry in entries:+ try:+ # Accepted TOCTOU: a late write between this mtime check and rmtree is tolerated;+ # the age gate + hourly cadence mean a swept entry's run finished long ago.+ if entry.lstat().st_mtime >= cutoff:+ continue+ # is_symlink guard keeps rmtree from following a symlink out of the swept base.+ if entry.is_dir() and not entry.is_symlink():+ shutil.rmtree(entry, ignore_errors=True)+ else:+ entry.unlink(missing_ok=True)+ removed_count += 1+ except FileNotFoundError:+ continue+ except Exception:+ LOG.warning("Failed to sweep stale temp entry", entry=str(entry), exc_info=True)++ if removed_count:+ LOG.info("Swept stale temp artifacts", removed_count=removed_count, max_age_hours=max_age_hours)+ return removed_count+++async def _temp_artifact_sweep_loop() -> None:+ while True:+ try:+ await asyncio.to_thread(sweep_stale_temp_artifacts)+ except Exception:+ LOG.exception("Temp-artifact sweep failed")+ await asyncio.sleep(3600)+++_temp_sweep_task: asyncio.Task | None = None+++def start_temp_artifact_sweep() -> asyncio.Task | None:+ """Start the hourly stale temp-artifact sweep (idempotent; disabled by a non-positive age gate)."""+ global _temp_sweep_task++ if settings.TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS <= 0:+ LOG.info("Temp-artifact sweep disabled", max_age_hours=settings.TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS)+ return None+ if _temp_sweep_task is not None and not _temp_sweep_task.done():+ return _temp_sweep_task++ _temp_sweep_task = asyncio.create_task(_temp_artifact_sweep_loop())+ LOG.info("Started temp-artifact sweep", max_age_hours=settings.TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS)+ return _temp_sweep_task+++async def stop_temp_artifact_sweep() -> None:+ global _temp_sweep_task++ if _temp_sweep_task is not None and not _temp_sweep_task.done():+ _temp_sweep_task.cancel()+ try:+ await _temp_sweep_task+ except asyncio.CancelledError:+ pass+ _temp_sweep_task = None++def get_stale_browser_processes(max_age_minutes: int = 60) -> list[psutil.Process]:"""Get browser-related processes that have been running for too long.diff --git a/tests/unit/test_cleanup_service.py b/tests/unit/test_cleanup_service.pynew file mode 100644index 000000000..911017141--- /dev/null+++ b/tests/unit/test_cleanup_service.py@@ -0,0 +1,117 @@+import os+import time+from pathlib import Path++from skyvern.config import settings+from skyvern.forge.sdk.artifact.storage.local import LocalStorage+from skyvern.services.cleanup_service import start_temp_artifact_sweep, sweep_stale_temp_artifacts+++class _RemoteStorage:+ """Stand-in for any non-local backend (S3/Azure/GCS) that uploads downloads off-disk."""+++def _set_backend(monkeypatch, storage: object) -> None:+ monkeypatch.setattr("skyvern.services.cleanup_service.StorageFactory.get_storage", lambda: storage)+++def _make_entry(base: Path, name: str, *, age_hours: float, is_dir: bool = True) -> Path:+ entry = base / name+ if is_dir:+ entry.mkdir(parents=True)+ (entry / "payload.bin").write_bytes(b"x" * 16)+ else:+ entry.write_bytes(b"x" * 16)+ stamp = time.time() - age_hours * 3600+ os.utime(entry, (stamp, stamp))+ return entry+++def _patch_paths(monkeypatch, tmp_path: Path) -> tuple[Path, Path, Path]:+ temp_dir = tmp_path / "temp"+ log_dir = tmp_path / "log"+ download_dir = tmp_path / "downloads"+ for directory in (temp_dir, log_dir, download_dir):+ directory.mkdir()+ monkeypatch.setattr(settings, "TEMP_PATH", str(temp_dir))+ monkeypatch.setattr(settings, "LOG_PATH", str(log_dir))+ monkeypatch.setattr(settings, "DOWNLOAD_PATH", str(download_dir))+ return temp_dir, log_dir, download_dir+++def test_sweep_removes_aged_log_and_download_entries(tmp_path, monkeypatch):+ _, log_dir, download_dir = _patch_paths(monkeypatch, tmp_path)+ _set_backend(monkeypatch, _RemoteStorage())++ stale_log_day = _make_entry(log_dir, "2026-07-01", age_hours=72)+ fresh_log_day = _make_entry(log_dir, "2026-07-15", age_hours=1)+ stale_download = _make_entry(download_dir, "wr_123", age_hours=72)+ fresh_download = _make_entry(download_dir, "wr_456", age_hours=1)++ removed = sweep_stale_temp_artifacts(max_age_hours=48)++ assert removed == 2+ assert not stale_log_day.exists()+ assert not stale_download.exists()+ assert fresh_log_day.exists()+ assert fresh_download.exists()+++def test_sweep_spares_download_path_on_local_backend(tmp_path, monkeypatch):+ # On the local storage backend DOWNLOAD_PATH/<run_id> is the run's permanent artifact record+ # (served via file://, never uploaded), so the sweep must leave it alone while still clearing LOG_PATH.+ _, log_dir, download_dir = _patch_paths(monkeypatch, tmp_path)+ _set_backend(monkeypatch, LocalStorage())++ stale_log_day = _make_entry(log_dir, "2026-07-01", age_hours=72)+ stale_download = _make_entry(download_dir, "wr_123", age_hours=72)++ removed = sweep_stale_temp_artifacts(max_age_hours=48)++ assert removed == 1+ assert not stale_log_day.exists()+ assert stale_download.exists()+++def test_sweep_never_touches_temp_path(tmp_path, monkeypatch):+ # TEMP_PATH mixes reused script caches and browser-session profile dirs whose mtime is+ # frozen while in use, so the age-gated sweep must leave TEMP_PATH alone entirely.+ temp_dir, _, _ = _patch_paths(monkeypatch, tmp_path)++ script_cache = _make_entry(temp_dir, "s_abc123", age_hours=720)+ profile_dir = _make_entry(temp_dir, "skyvern_browser_profile_xyz", age_hours=720)+ loose_file = _make_entry(temp_dir, "orphan.zip", age_hours=720, is_dir=False)++ assert sweep_stale_temp_artifacts(max_age_hours=48) == 0+ assert script_cache.exists()+ assert profile_dir.exists()+ assert loose_file.exists()+++def test_sweep_disabled_when_gate_nonpositive(tmp_path, monkeypatch):+ _, log_dir, _ = _patch_paths(monkeypatch, tmp_path)+ stale = _make_entry(log_dir, "2026-07-01", age_hours=72)++ assert sweep_stale_temp_artifacts(max_age_hours=0) == 0+ assert stale.exists()+++def test_sweep_tolerates_missing_base_dirs(tmp_path, monkeypatch):+ monkeypatch.setattr(settings, "LOG_PATH", str(tmp_path / "nope_log"))+ monkeypatch.setattr(settings, "DOWNLOAD_PATH", str(tmp_path / "nope_dl"))++ assert sweep_stale_temp_artifacts(max_age_hours=48) == 0+++def test_sweep_default_gate_comes_from_settings(tmp_path, monkeypatch):+ _, log_dir, _ = _patch_paths(monkeypatch, tmp_path)+ stale = _make_entry(log_dir, "2026-07-01", age_hours=72)+ monkeypatch.setattr(settings, "TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS", 48.0)++ assert sweep_stale_temp_artifacts() == 1+ assert not stale.exists()+++def test_start_sweep_disabled_by_nonpositive_gate(monkeypatch):+ monkeypatch.setattr(settings, "TEMP_ARTIFACT_SWEEP_MAX_AGE_HOURS", 0.0)+ assert start_temp_artifact_sweep() is None