Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add pyupgrade pre-commit hook + modernize python code (#2611)
Asher Foaeffd0c491106
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 24b08702c..5104e06fb 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -51,6 +51,14 @@ repos:- id: python-check-mock-methods- id: python-no-log-warn- id: python-use-type-annotations+ - repo: https://github.com/asottile/pyupgrade+ rev: v3.20.0+ hooks:+ - id: pyupgrade+ exclude: |+ (?x)(+ ^skyvern/client/.*+ )- repo: https://github.com/pre-commit/mirrors-mypyrev: v1.16.0hooks:@@ -61,7 +69,7 @@ repos:- types-requests- types-cachetools- alembic- - "sqlalchemy[mypy]"+ - 'sqlalchemy[mypy]'- types-PyYAML- types-aiofilesexclude: |@@ -91,7 +99,7 @@ repos:# pass_filenames: false# always_run: true- repo: https://github.com/pre-commit/mirrors-prettier- rev: "v4.0.0-alpha.8" # Use the sha or tag you want to point at+ rev: 'v4.0.0-alpha.8' # Use the sha or tag you want to point athooks:- id: prettiertypes: [javascript]diff --git a/evaluation/core/utils.py b/evaluation/core/utils.pyindex 4e1cd936b..9bc15fd81 100644--- a/evaluation/core/utils.py+++ b/evaluation/core/utils.py@@ -21,13 +21,13 @@ class WorkflowRunResultRequest(BaseModel):def load_webvoyager_case_from_json(file_path: str, group_id: str = "") -> Iterator[WebVoyagerTestCase]:- with open("evaluation/datasets/webvoyager_reference_answer.json", "r") as answer_file:+ with open("evaluation/datasets/webvoyager_reference_answer.json") as answer_file:webvoyager_answers: dict = json.load(answer_file)if not group_id:group_id = str(uuid4())- with open(file_path, "r", encoding="utf-8") as file:+ with open(file_path, encoding="utf-8") as file:for line in file:test_case: dict[str, str] = json.loads(line)web_name, id = test_case["id"].split("--")@@ -47,7 +47,7 @@ def load_webvoyager_case_from_json(file_path: str, group_id: str = "") -> Iteratdef load_records_from_json(file_path: str) -> Iterator[WorkflowRunResultRequest]:- with open(file_path, "r", encoding="utf-8") as f:+ with open(file_path, encoding="utf-8") as f:for line in f:item: dict[str, str] = json.loads(line)id = item["id"]diff --git a/evaluation/script/create_webvoyager_evaluation_result.py b/evaluation/script/create_webvoyager_evaluation_result.pyindex 506a52e01..d4b189cdc 100644--- a/evaluation/script/create_webvoyager_evaluation_result.py+++ b/evaluation/script/create_webvoyager_evaluation_result.py@@ -35,7 +35,7 @@ def main() -> None:client = SkyvernClient(base_url=base_url, credentials=cred)- with open(record_json_path, "r", encoding="utf-8") as file:+ with open(record_json_path, encoding="utf-8") as file:with open(output_csv_path, newline="", mode="w", encoding="utf-8") as csv_file:writer = csv.DictWriter(csv_file, fieldnames=csv_headers)writer.writeheader()diff --git a/evaluation/script/eval_webvoyager_task_v2.py b/evaluation/script/eval_webvoyager_task_v2.pyindex f9f67e339..8bca329cf 100644--- a/evaluation/script/eval_webvoyager_task_v2.py+++ b/evaluation/script/eval_webvoyager_task_v2.py@@ -79,7 +79,7 @@ async def run_eval() -> None:client = SkyvernClient(base_url=base_url, credentials=cred)- with open(record_json_path, "r", encoding="utf-8") as file:+ with open(record_json_path, encoding="utf-8") as file:with open(output_csv_path, newline="", mode="w", encoding="utf-8") as csv_file:writer = csv.DictWriter(csv_file, fieldnames=csv_headers)writer.writeheader()diff --git a/skyvern/analytics.py b/skyvern/analytics.pyindex 21d2bc0d8..2294c2365 100644--- a/skyvern/analytics.py+++ b/skyvern/analytics.py@@ -1,4 +1,3 @@-# -*- coding: utf-8 -*-import importlib.metadataimport platformfrom typing import Any, Dictdiff --git a/skyvern/cli/mcp.py b/skyvern/cli/mcp.pyindex 64030d46b..498e6e29d 100644--- a/skyvern/cli/mcp.py+++ b/skyvern/cli/mcp.py@@ -144,7 +144,7 @@ def setup_claude_desktop_config(host_system: str, path_to_env: str) -> bool:claude_config: dict = {"mcpServers": {}}if os.path.exists(path_claude_config):try:- with open(path_claude_config, "r") as f:+ with open(path_claude_config) as f:claude_config = json.load(f)claude_config["mcpServers"].pop("Skyvern", None)claude_config["mcpServers"]["Skyvern"] = {@@ -196,7 +196,7 @@ def setup_cursor_config(host_system: str, path_to_env: str) -> bool:cursor_config: dict = {"mcpServers": {}}if os.path.exists(path_cursor_config):try:- with open(path_cursor_config, "r") as f:+ with open(path_cursor_config) as f:cursor_config = json.load(f)cursor_config["mcpServers"].pop("Skyvern", None)cursor_config["mcpServers"]["Skyvern"] = {@@ -245,7 +245,7 @@ def setup_windsurf_config(host_system: str, path_to_env: str) -> bool:windsurf_config: dict = {"mcpServers": {}}if os.path.exists(path_windsurf_config):try:- with open(path_windsurf_config, "r") as f:+ with open(path_windsurf_config) as f:windsurf_config = json.load(f)windsurf_config["mcpServers"].pop("Skyvern", None)windsurf_config["mcpServers"]["Skyvern"] = {diff --git a/skyvern/cli/tasks.py b/skyvern/cli/tasks.pyindex 214cb134d..823eae86d 100644--- a/skyvern/cli/tasks.py+++ b/skyvern/cli/tasks.py@@ -4,7 +4,6 @@ from __future__ import annotationsimport jsonimport os-from typing import Optionalimport typerfrom dotenv import load_dotenv@@ -21,7 +20,7 @@ tasks_app = typer.Typer(help="Manage Skyvern tasks and operations.")@tasks_app.callback()def tasks_callback(ctx: typer.Context,- api_key: Optional[str] = typer.Option(+ api_key: str | None = typer.Option(None,"--api-key",help="Skyvern API key",@@ -32,7 +31,7 @@ def tasks_callback(ctx.obj = {"api_key": api_key}-def _get_client(api_key: Optional[str] = None) -> Skyvern:+def _get_client(api_key: str | None = None) -> Skyvern:"""Instantiate a Skyvern SDK client using environment variables."""load_dotenv()load_dotenv(".env")diff --git a/skyvern/cli/workflow.py b/skyvern/cli/workflow.pyindex bc9ffb8f4..f025b4aa7 100644--- a/skyvern/cli/workflow.py+++ b/skyvern/cli/workflow.py@@ -4,7 +4,6 @@ from __future__ import annotationsimport jsonimport os-from typing import Optionalimport typerfrom dotenv import load_dotenv@@ -22,7 +21,7 @@ workflow_app = typer.Typer(help="Manage Skyvern workflows.")@workflow_app.callback()def workflow_callback(ctx: typer.Context,- api_key: Optional[str] = typer.Option(+ api_key: str | None = typer.Option(None,"--api-key",help="Skyvern API key",@@ -33,7 +32,7 @@ def workflow_callback(ctx.obj = {"api_key": api_key}-def _get_client(api_key: Optional[str] = None) -> Skyvern:+def _get_client(api_key: str | None = None) -> Skyvern:"""Instantiate a Skyvern SDK client using environment variables."""load_dotenv()load_dotenv(".env")@@ -46,8 +45,8 @@ def start_workflow(ctx: typer.Context,workflow_id: str = typer.Argument(..., help="Workflow permanent ID"),parameters: str = typer.Option("{}", "--parameters", "-p", help="JSON parameters for the workflow"),- title: Optional[str] = typer.Option(None, "--title", help="Title for the workflow run"),- max_steps: Optional[int] = typer.Option(None, "--max-steps", help="Override the workflow max steps"),+ title: str | None = typer.Option(None, "--title", help="Title for the workflow run"),+ max_steps: int | None = typer.Option(None, "--max-steps", help="Override the workflow max steps"),) -> None:"""Dispatch a workflow run."""try:diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.pyindex 9087f6938..cab8708aa 100644--- a/skyvern/forge/sdk/api/aws.py+++ b/skyvern/forge/sdk/api/aws.py@@ -255,7 +255,7 @@ class AsyncAWSClient:return await client.deregister_task_definition(taskDefinition=task_definition)-class S3Uri(object):+class S3Uri:# From: https://stackoverflow.com/questions/42641315/s3-urls-get-bucket-name-and-path""">>> s = S3Uri("s3://bucket/hello/world")diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.pyindex cfd7f7648..ef94faf45 100644--- a/skyvern/forge/sdk/artifact/storage/local.py+++ b/skyvern/forge/sdk/artifact/storage/local.py@@ -38,7 +38,7 @@ class LocalStorage(BaseStorage):if not file_path.exists():return []try:- with open(file_path, "r") as f:+ with open(file_path) as f:return [line.strip() for line in f.readlines() if line.strip()]except Exception:return []diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex a43799d12..6761f750b 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -211,7 +211,7 @@ class Block(BaseModel, abc.ABC):return template.render(template_data)@classmethod- def get_subclasses(cls) -> tuple[type["Block"], ...]:+ def get_subclasses(cls) -> tuple[type[Block], ...]:return tuple(cls.__subclasses__())@staticmethod@@ -2123,7 +2123,7 @@ class FileParserBlock(Block):def validate_file_type(self, file_url_used: str, file_path: str) -> None:if self.file_type == FileType.CSV:try:- with open(file_path, "r") as file:+ with open(file_path) as file:csv.Sniffer().sniff(file.read(1024))except csv.Error as e:raise InvalidFileType(file_url=file_url_used, file_type=self.file_type, error=str(e))@@ -2172,7 +2172,7 @@ class FileParserBlock(Block):self.validate_file_type(self.file_url, file_path)# Parse the file into a list of dictionaries where each dictionary represents a row in the fileparsed_data = []- with open(file_path, "r") as file:+ with open(file_path) as file:if self.file_type == FileType.CSV:reader = csv.DictReader(file)for row in reader:diff --git a/skyvern/utils/__init__.py b/skyvern/utils/__init__.pyindex 72b55bad7..c615c57fb 100644--- a/skyvern/utils/__init__.py+++ b/skyvern/utils/__init__.py@@ -28,7 +28,7 @@ def detect_os() -> str:system = platform.system()if system == "Linux":try:- with open("/proc/version", "r") as f:+ with open("/proc/version") as f:version_info = f.read().lower()if "microsoft" in version_info:return "wsl"diff --git a/skyvern/utils/files.py b/skyvern/utils/files.pyindex ce614540e..5190236dc 100644--- a/skyvern/utils/files.py+++ b/skyvern/utils/files.py@@ -25,5 +25,5 @@ def get_json_from_file(file_path: str) -> dict[str, str]:if not os.path.exists(file_path):return {}- with open(file_path, "r") as json_file:+ with open(file_path) as json_file:return json.load(json_file)diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex bba67b13a..d5ea7d106 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -162,7 +162,7 @@ class BrowserContextFactory:preference_template = f"{SKYVERN_DIR}/webeye/chromium_preferences.json"preference_file_content = ""- with open(preference_template, "r") as f:+ with open(preference_template) as f:preference_file_content = f.read()preference_file_content = preference_file_content.replace("MASK_SAVEFILE_DEFAULT_DIRECTORY", download_dir)preference_file_content = preference_file_content.replace("MASK_DOWNLOAD_DEFAULT_DIRECTORY", download_dir)@@ -281,7 +281,7 @@ class BrowserContextFactory:class VideoArtifact(BaseModel):video_path: str | None = Nonevideo_artifact_id: str | None = None- video_data: bytes = bytes()+ video_data: bytes = b""class BrowserArtifacts(BaseModel):@@ -385,7 +385,7 @@ def _is_port_in_use(port: int) -> bool:try:s.bind(("localhost", port))return False- except socket.error:+ except OSError:return Truediff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.pyindex a85ba4ff9..29c84bef0 100644--- a/skyvern/webeye/persistent_sessions_manager.py+++ b/skyvern/webeye/persistent_sessions_manager.py@@ -1,7 +1,6 @@from __future__ import annotationsfrom dataclasses import dataclass-from typing import Dict, Optional, Tupleimport structlogfrom playwright._impl._errors import TargetClosedError@@ -22,7 +21,7 @@ class BrowserSession:class PersistentSessionsManager:instance: PersistentSessionsManager | None = None- _browser_sessions: Dict[str, BrowserSession] = dict()+ _browser_sessions: dict[str, BrowserSession] = dict()database: AgentDBdef __new__(cls, database: AgentDB) -> PersistentSessionsManager:@@ -82,7 +81,7 @@ class PersistentSessionsManager:organization_id=organization_id,)- async def get_network_info(self, session_id: str) -> Tuple[Optional[int], Optional[str]]:+ async def get_network_info(self, session_id: str) -> tuple[int | None, str | None]:"""Returns cdp port and ip address of the browser session"""browser_session = self._browser_sessions.get(session_id)if browser_session:diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 835bdd8dd..e24b88470 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -75,7 +75,7 @@ def load_js_script() -> str:try:# TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.# This will allow our code to be type safe.- with open(path, "r") as f:+ with open(path) as f:return f.read()except FileNotFoundError as e:LOG.exception("Failed to load the JS script", path=path)diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 73804f41a..31b9641a4 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -34,9 +34,7 @@ TEXT_INPUT_DELAY = 10 # 10ms between each character inputTEXT_PRESS_MAX_LENGTH = 20-async def resolve_locator(- scrape_page: ScrapedPage, page: Page, frame: str, css: str-) -> typing.Tuple[Locator, Page | Frame]:+async def resolve_locator(scrape_page: ScrapedPage, page: Page, frame: str, css: str) -> tuple[Locator, Page | Frame]:iframe_path: list[str] = []while frame != "main.frame":@@ -335,10 +333,10 @@ class SkyvernElement:def get_frame_id(self) -> str:return self._frame_id- def get_attributes(self) -> typing.Dict:+ def get_attributes(self) -> dict:return self._attributes- def get_options(self) -> typing.List[SkyvernOptionType]:+ def get_options(self) -> list[SkyvernOptionType]:options = self.__static_element.get("options", None)if options is None:return []diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex ab70d6db9..d1f9546db 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport asyncioimport time-from typing import Any, Dict, List+from typing import Anyimport structlogfrom playwright._impl._errors import TimeoutError@@ -21,7 +21,7 @@ def load_js_script() -> str:try:# TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.# This will allow our code to be type safe.- with open(path, "r") as f:+ with open(path) as f:return f.read()except FileNotFoundError as e:LOG.exception("Failed to load the JS script", path=path)@@ -43,7 +43,7 @@ async def _current_viewpoint_screenshot_helper(await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)LOG.debug("Page is fully loaded, agent is about to take screenshots")start_time = time.time()- screenshot: bytes = bytes()+ screenshot: bytes = b""if file_path:screenshot = await page.screenshot(path=file_path,@@ -77,14 +77,14 @@ async def _scrolling_screenshots_helper(url: str | None = None,draw_boxes: bool = False,max_number: int = settings.MAX_NUM_SCREENSHOTS,-) -> List[bytes]:+) -> list[bytes]:skyvern_page = await SkyvernFrame.create_instance(frame=page)# page is the main frame and the index must be 0assert isinstance(skyvern_page.frame, Page)frame = "main.frame"frame_index = 0- screenshots: List[bytes] = []+ screenshots: list[bytes] = []if await skyvern_page.is_window_scrollable():scroll_y_px_old = -30.0scroll_y_px = await skyvern_page.scroll_to_top(draw_boxes=draw_boxes, frame=frame, frame_index=frame_index)@@ -161,7 +161,7 @@ class SkyvernFrame:draw_boxes: bool = False,max_number: int = settings.MAX_NUM_SCREENSHOTS,scroll: bool = True,- ) -> List[bytes]:+ ) -> list[bytes]:if not scroll:return [await _current_viewpoint_screenshot_helper(page=page)]@@ -199,7 +199,7 @@ class SkyvernFrame:js_script = "(element) => scrollToElementTop(element)"return await self.evaluate(frame=self.frame, expression=js_script, arg=element)- async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> Dict:+ async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict:js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)"return await self.evaluate(frame=self.frame, expression=js_script, arg=[frame, element, interactable])