Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
use skyvern temp to save tempfiles (#1262)
LawyZhengaf25aeed774b
diff --git a/.gitignore b/.gitignoreindex 045020079..a88e841b2 100644--- a/.gitignore+++ b/.gitignore@@ -166,6 +166,7 @@ traces/har/postgres-datafiles/+temp/# Streamlit ignores**/secrets*.tomldiff --git a/skyvern/config.py b/skyvern/config.pyindex 72ade1452..bce59736f 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -14,6 +14,7 @@ class Settings(BaseSettings):VIDEO_PATH: str | None = NoneHAR_PATH: str | None = "./har"LOG_PATH: str = "./log"+ TEMP_PATH: str = "./temp"BROWSER_ACTION_TIMEOUT_MS: int = 5000BROWSER_SCREENSHOT_TIMEOUT_MS: int = 20000BROWSER_LOADING_TIMEOUT_MS: int = 120000@@ -74,9 +75,6 @@ class Settings(BaseSettings):WORKFLOW_DOWNLOAD_DIRECTORY_PARAMETER_KEY: str = "SKYVERN_DOWNLOAD_DIRECTORY"WORKFLOW_WAIT_BLOCK_MAX_SEC: int = 30 * 60- # streaming settings- STREAMING_FILE_BASE_PATH: str = "/tmp"-# Saved browser session settingsBROWSER_SESSION_BASE_PATH: str = f"{constants.REPO_ROOT_DIR}/browser_sessions"diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.pyindex 0434841f1..8fb378bb1 100644--- a/skyvern/forge/sdk/api/files.py+++ b/skyvern/forge/sdk/api/files.py@@ -14,13 +14,14 @@ from multidict import CIMultiDictProxyfrom skyvern.constants import REPO_ROOT_DIRfrom skyvern.exceptions import DownloadFileMaxSizeExceededfrom skyvern.forge.sdk.api.aws import AsyncAWSClient+from skyvern.forge.sdk.settings_manager import SettingsManagerLOG = structlog.get_logger()async def download_from_s3(client: AsyncAWSClient, s3_uri: str) -> str:downloaded_bytes = await client.download_file(uri=s3_uri)- file_path = tempfile.NamedTemporaryFile(delete=False)+ file_path = create_named_temporary_file(delete=False)file_path.write(downloaded_bytes)return file_path.name@@ -56,7 +57,7 @@ async def download_file(url: str, max_size_mb: int | None = None) -> str:a = urlparse(url)# Get the file name- temp_dir = tempfile.mkdtemp(prefix="skyvern_downloads_")+ temp_dir = make_temp_directory(prefix="skyvern_downloads_")file_name = os.path.basename(a.path)# if no suffix in the URL, we need to parse it from HTTP headers@@ -151,3 +152,31 @@ def calculate_sha256_for_file(file_path: str) -> str:for byte_block in iter(lambda: f.read(4096), b""):sha256_hash.update(byte_block)return sha256_hash.hexdigest()+++def create_folder_if_not_exist(dir: str) -> None:+ path = Path(dir)+ if path.exists():+ return+ path.mkdir(parents=True)+++def get_skyvern_temp_dir() -> str:+ temp_dir = SettingsManager.get_settings().TEMP_PATH+ create_folder_if_not_exist(temp_dir)+ return temp_dir+++def make_temp_directory(+ suffix: str | None = None,+ prefix: str | None = None,+) -> str:+ temp_dir = SettingsManager.get_settings().TEMP_PATH+ create_folder_if_not_exist(temp_dir)+ return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=temp_dir)+++def create_named_temporary_file(delete: bool = True) -> tempfile._TemporaryFileWrapper:+ temp_dir = SettingsManager.get_settings().TEMP_PATH+ create_folder_if_not_exist(temp_dir)+ return tempfile.NamedTemporaryFile(dir=temp_dir, delete=delete)diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.pyindex 7df2c54f1..54f0c54ed 100644--- a/skyvern/forge/sdk/artifact/storage/local.py+++ b/skyvern/forge/sdk/artifact/storage/local.py@@ -6,6 +6,7 @@ from urllib.parse import unquote, urlparseimport structlog+from skyvern.forge.sdk.api.files import get_skyvern_temp_dirfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.artifact.storage.base import FILE_EXTENTSION_MAP, BaseStoragefrom skyvern.forge.sdk.models import Step@@ -73,9 +74,9 @@ class LocalStorage(BaseStorage):returnasync def get_streaming_file(self, organization_id: str, file_name: str, use_default: bool = True) -> bytes | None:- file_path = Path(f"{SettingsManager.get_settings().STREAMING_FILE_BASE_PATH}/skyvern_screenshot.png")+ file_path = Path(f"{get_skyvern_temp_dir()}/skyvern_screenshot.png")if not use_default:- file_path = Path(f"{SettingsManager.get_settings().STREAMING_FILE_BASE_PATH}/{organization_id}/{file_name}")+ file_path = Path(f"{get_skyvern_temp_dir()}/{organization_id}/{file_name}")try:with open(file_path, "rb") as f:return f.read()diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex b7a1726ee..c3b7601c2 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -1,10 +1,14 @@import shutil-import tempfilefrom datetime import datetimefrom skyvern.config import settingsfrom skyvern.forge.sdk.api.aws import AsyncAWSClient-from skyvern.forge.sdk.api.files import unzip_files+from skyvern.forge.sdk.api.files import (+ create_named_temporary_file,+ get_skyvern_temp_dir,+ make_temp_directory,+ unzip_files,+)from skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.artifact.storage.base import FILE_EXTENTSION_MAP, BaseStoragefrom skyvern.forge.sdk.models import Step@@ -36,7 +40,7 @@ class S3Storage(BaseStorage):await self.async_client.upload_file_from_path(artifact.uri, path)async def save_streaming_file(self, organization_id: str, file_name: str) -> None:- from_path = f"{settings.STREAMING_FILE_BASE_PATH}/{organization_id}/{file_name}"+ from_path = f"{get_skyvern_temp_dir()}/{organization_id}/{file_name}"to_path = f"s3://{settings.AWS_S3_BUCKET_SCREENSHOTS}/{settings.ENV}/{organization_id}/{file_name}"await self.async_client.upload_file_from_path(to_path, from_path)@@ -46,7 +50,7 @@ class S3Storage(BaseStorage):async def store_browser_session(self, organization_id: str, workflow_permanent_id: str, directory: str) -> None:# Zip the directory to a temp file- temp_zip_file = tempfile.NamedTemporaryFile()+ temp_zip_file = create_named_temporary_file()zip_file_path = shutil.make_archive(temp_zip_file.name, "zip", directory)browser_session_uri = f"s3://{settings.AWS_S3_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/{workflow_permanent_id}.zip"await self.async_client.upload_file_from_path(browser_session_uri, zip_file_path)@@ -56,11 +60,11 @@ class S3Storage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(browser_session_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = tempfile.NamedTemporaryFile(delete=False)+ temp_zip_file = create_named_temporary_file(delete=False)temp_zip_file.write(downloaded_zip_bytes)temp_zip_file_path = temp_zip_file.name- temp_dir = tempfile.mkdtemp(prefix="skyvern_browser_session_")+ temp_dir = make_temp_directory(prefix="skyvern_browser_session_")unzip_files(temp_zip_file_path, temp_dir)temp_zip_file.close()return temp_dirdiff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 07dab2175..4386758d9 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -11,7 +11,6 @@ from dataclasses import dataclassfrom email.message import EmailMessagefrom enum import StrEnumfrom pathlib import Path-from tempfile import NamedTemporaryFilefrom typing import Annotated, Any, Literal, Unionimport filetype@@ -36,6 +35,7 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.aws import AsyncAWSClientfrom skyvern.forge.sdk.api.files import (calculate_sha256_for_file,+ create_named_temporary_file,download_file,download_from_s3,get_path_for_workflow_download_directory,@@ -1056,7 +1056,7 @@ class SendEmailBlock(Block):async def _download_from_s3(self, s3_uri: str) -> str:client = self.get_async_aws_client()downloaded_bytes = await client.download_file(uri=s3_uri)- file_path = NamedTemporaryFile(delete=False)+ file_path = create_named_temporary_file(delete=False)file_path.write(downloaded_bytes)return file_path.namediff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex 25b99fc60..da5889057 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport os-import tempfileimport timeimport uuidfrom datetime import datetime@@ -24,6 +23,7 @@ from skyvern.exceptions import (UnknownBrowserType,UnknownErrorWhileCreatingBrowserContext,)+from skyvern.forge.sdk.api.files import make_temp_directoryfrom skyvern.forge.sdk.core.skyvern_context import currentfrom skyvern.forge.sdk.schemas.tasks import ProxyLocationfrom skyvern.forge.sdk.settings_manager import SettingsManager@@ -153,7 +153,7 @@ class BrowserContextFactory:video_dir = f"{SettingsManager.get_settings().VIDEO_PATH}/{datetime.utcnow().strftime('%Y-%m-%d')}"har_dir = f"{SettingsManager.get_settings().HAR_PATH}/{datetime.utcnow().strftime('%Y-%m-%d')}/{BrowserContextFactory.get_subdir()}.har"return {- "user_data_dir": tempfile.mkdtemp(prefix="skyvern_browser_"),+ "user_data_dir": make_temp_directory(prefix="skyvern_browser_"),"locale": SettingsManager.get_settings().BROWSER_LOCALE,"timezone_id": SettingsManager.get_settings().BROWSER_TIMEZONE,"color_scheme": "no-preference",