Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
A2A Agent: Implement A2UI cards (#9014)
Shuchang Zheng8b577991cfa7
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex ff55a8010..422732210 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -23,6 +23,7 @@ from skyvern.forge.log_redaction import (redact_sensitive_fields,strip_artifact_url_query,)+from skyvern.forge.request_privacy import private_logging_scope, private_request_audit, private_request_pathfrom skyvern.forge.sdk.forge_log import exception_log_fieldsif typing.TYPE_CHECKING: # pragma: no cover - import only for type hints@@ -241,7 +242,8 @@ def _is_fixture_endpoint(request: Request) -> bool:def _is_sensitive_endpoint(request: Request) -> bool:endpoint = f"{request.method.upper()} {request.url.path.rstrip('/')}"return (- _is_fixture_endpoint(request)+ private_request_path(request.scope) is not None+ or _is_fixture_endpoint(request)or endpoint in _SENSITIVE_ENDPOINTSor any(pattern.fullmatch(endpoint) for pattern in _SENSITIVE_ENDPOINT_PATTERNS)or (request.method.upper() == "POST" and _ACTION_LOG_ENDPOINT_RE.fullmatch(request.url.path) is not None)@@ -508,6 +510,15 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):await super().__call__(scope, receive, send)return+ if audit_path := private_request_path(scope):+ with private_logging_scope(audit_path):+ identity = _request_identity.set(_RequestIdentity())+ try:+ await self._private_request(scope, receive, send, audit_path=audit_path)+ finally:+ _request_identity.reset(identity)+ return+# BaseHTTPMiddleware raises a started stream's error only after dispatch returns.# Keep the client-visible status here so that error is still countable.response_status: int | None = None@@ -531,3 +542,70 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):_raw_request_exception_logger.set(None)finally:_raw_request_stream_success_logger.set(None)++ async def _private_request(self, scope: Scope, receive: Receive, send: Send, *, audit_path: str) -> None:+ start = time.monotonic()+ status: int | None = None+ complete = False+ outcome = "complete"++ async def opaque_send(message: Message) -> None:+ nonlocal status, complete+ await send(message)+ if message["type"] == "http.response.start":+ status = message["status"]+ elif message["type"] == "http.response.body" and not message.get("more_body", False):+ complete = True++ try:+ try:+ # Passing receive/send directly keeps payload bytes out of the logger's buffers.+ await self.app(scope, receive, opaque_send)+ except ClientDisconnect:+ outcome = "disconnected"+ if status is None:+ try:+ await Response(status_code=499)(scope, receive, opaque_send)+ except Exception:+ pass+ except Exception:+ outcome = "stream_failed" if status is not None else "failed"+ if status is None:+ try:+ await Response(+ b'{"detail":"private_request_failed"}', status_code=500, media_type="application/json"+ )(scope, receive, opaque_send)+ except Exception:+ pass+ elif not complete:+ try:+ await send({"type": "http.response.body", "body": b"", "more_body": False})+ except Exception:+ pass+ except asyncio.CancelledError:+ outcome = "cancelled"+ raise+ finally:+ if settings.LOG_RAW_API_REQUESTS:+ code = status+ log = (+ LOG.error+ if outcome != "complete" or (code is not None and code >= 500)+ else LOG.warning+ if code is not None and code >= 400+ else LOG.info+ )+ try:+ with private_request_audit():+ log(+ "api.raw_request",+ method=scope["method"]+ if scope["method"] in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}+ else "OTHER",+ path=audit_path,+ status_code=code,+ duration_seconds=time.monotonic() - start,+ outcome=outcome,+ )+ except Exception:+ passdiff --git a/skyvern/forge/request_privacy.py b/skyvern/forge/request_privacy.pynew file mode 100644index 000000000..f07ee2de2--- /dev/null+++ b/skyvern/forge/request_privacy.py@@ -0,0 +1,217 @@+from __future__ import annotations++import asyncio+import logging+import math+import re+from collections.abc import Callable, Coroutine, Generator, Iterator+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass+from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast++from structlog.typing import EventDict++if TYPE_CHECKING:+ from starlette.applications import Starlette+ from starlette.types import Scope++_private_logging: ContextVar[str | None] = ContextVar("private_request_logging", default=None)+_private_audit: ContextVar[bool] = ContextVar("private_request_audit", default=False)+_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "OTHER"})+_OUTCOMES = frozenset({"complete", "failed", "stream_failed", "disconnected", "cancelled"})+_STANDARD_RECORD_FIELDS = frozenset(logging.makeLogRecord({}).__dict__)+_Result = TypeVar("_Result")+++class _GeneratorAwaitable(Generic[_Result]):+ def __init__(self, coroutine: Generator[Any, None, _Result]) -> None:+ self.coroutine = coroutine++ def __await__(self) -> Generator[Any, None, _Result]:+ return (yield from self.coroutine)+++@dataclass(frozen=True)+class PrivateRequestPolicy:+ classifier: Callable[[str], bool]+ audit_path: str+++def register_private_request_classifier(app: Starlette, classifier: Callable[[str], bool], *, audit_path: str) -> None:+ if not callable(classifier) or re.fullmatch(r"/[a-z0-9_/-]{1,120}", audit_path) is None:+ raise ValueError("Private request classification requires a callable and a static audit path")+ app.state.private_request_policy = PrivateRequestPolicy(classifier, audit_path)+++def private_request_path(scope: Scope) -> str | None:+ application = scope.get("app")+ if application is None:+ return None+ try:+ policy = application.state.private_request_policy+ except AttributeError:+ return None+ if not isinstance(policy, PrivateRequestPolicy):+ return "/private"+ try:+ private = policy.classifier(scope["path"])+ except Exception:+ return policy.audit_path+ if type(private) is not bool:+ return policy.audit_path+ return policy.audit_path if private else None+++@contextmanager+def private_logging_scope(audit_path: str | None = None) -> Iterator[None]:+ try:+ install_private_task_guard(asyncio.get_running_loop())+ except RuntimeError:+ pass+ token = _private_logging.set(audit_path or _private_logging.get() or "/private")+ try:+ yield+ finally:+ _private_logging.reset(token)+++def _private_task_done(task: asyncio.Future[Any]) -> None:+ if not task.cancelled() and task.exception() is not None:+ with private_logging_scope():+ logging.getLogger(__name__).error("private.background_task_failed")+++def install_private_task_guard(loop: asyncio.AbstractEventLoop) -> None:+ previous = loop.get_task_factory()+ if previous is not None and getattr(previous, "_private_task_guard", False):+ return++ def factory(+ loop: asyncio.AbstractEventLoop,+ coroutine: Coroutine[Any, Any, _Result] | Generator[Any, None, _Result],+ **kwargs: Any,+ ) -> asyncio.Future[_Result]:+ context = kwargs.get("context")+ private = _private_logging.get() or (context.get(_private_logging) if context is not None else None)+ selected = coroutine+ entered = False+ if private:++ async def run_private() -> _Result:+ nonlocal entered+ entered = True+ with private_logging_scope(private):+ if isinstance(coroutine, Coroutine):+ return await coroutine+ return await _GeneratorAwaitable(coroutine)++ selected = run_private()+ try:+ # Python 3.11 accepts generator coroutines; newer stubs narrow the constructor annotation.+ selected_coroutine = cast(Coroutine[Any, Any, _Result], selected)+ task = (+ previous(loop, selected_coroutine, **kwargs)+ if previous is not None+ else asyncio.Task(selected_coroutine, loop=loop, **kwargs)+ )+ except BaseException:+ if private:+ selected.close()+ if not entered:+ coroutine.close()+ raise+ if private:++ def done(task: asyncio.Future[Any]) -> None:+ if not entered:+ coroutine.close()+ _private_task_done(task)++ task.add_done_callback(done)+ return task++ factory._private_task_guard = True # type: ignore[attr-defined]+ loop.set_task_factory(factory)+++@contextmanager+def private_request_audit() -> Iterator[None]:+ token = _private_audit.set(True)+ try:+ yield+ finally:+ _private_audit.reset(token)+++def redact_private_log_event(logger: object, method_name: str, event_dict: EventDict) -> EventDict:+ if not _private_logging.get():+ return event_dict+ key = "event" if "event" in event_dict else "msg"+ safe: EventDict = {key: "private.suppressed_log"}+ if _private_audit.get():+ safe[key] = "api.raw_request"+ safe["path"] = _private_logging.get()+ if type(event_dict.get("method")) is str and event_dict["method"] in _METHODS:+ safe["method"] = event_dict["method"]+ status = event_dict.get("status_code")+ if type(status) is int and 100 <= status <= 599:+ safe["status_code"] = status+ duration = event_dict.get("duration_seconds")+ if type(duration) is float and math.isfinite(duration) and duration >= 0:+ safe["duration_seconds"] = duration+ if type(event_dict.get("outcome")) is str and event_dict["outcome"] in _OUTCOMES:+ safe["outcome"] = event_dict["outcome"]+ if method_name in {+ "debug",+ "info",+ "warning",+ "error",+ "critical",+ }:+ safe["level"] = method_name+ return safe+++def redact_private_foreign_event(logger: object, method_name: str, event_dict: EventDict) -> EventDict:+ safe = redact_private_log_event(logger, method_name, event_dict)+ for plumbing in ("_record", "_from_structlog"):+ if plumbing in event_dict:+ safe[plumbing] = event_dict[plumbing]+ return safe+++def private_export_record(record: logging.LogRecord) -> logging.LogRecord:+ if not _private_logging.get():+ return record+ levels = {+ logging.DEBUG: "debug",+ logging.INFO: "info",+ logging.WARNING: "warning",+ logging.ERROR: "error",+ logging.CRITICAL: "critical",+ }+ level = record.levelno if type(record.levelno) is int and record.levelno in levels else logging.INFO+ event = record.msg if isinstance(record.msg, dict) else {}+ safe = redact_private_log_event(None, levels[level], event)+ exported = logging.LogRecord("private.request", level, "", 0, safe, (), None, func="")+ exported.levelname = levels[level].upper()+ for key, value in safe.items():+ if key not in {"event", "msg", "level"}:+ setattr(exported, key, value)+ return exported+++class PrivateRequestLogFilter(logging.Filter):+ def filter(self, record: logging.LogRecord) -> bool:+ if not _private_logging.get():+ return True+ if isinstance(record.msg, dict):+ record.msg = redact_private_log_event(None, record.levelname.lower(), record.msg)+ else:+ record.msg = "private.suppressed_log"+ record.args = ()+ record.exc_info = record.exc_text = record.stack_info = None+ for key in set(record.__dict__) - _STANDARD_RECORD_FIELDS - {"_logger", "_name"}:+ del record.__dict__[key]+ return Truediff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex b01a8940c..473137736 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -31,6 +31,11 @@ from skyvern.forge.log_redaction import (redact_proxy_observability_value,redact_sensitive_fields,)+from skyvern.forge.request_privacy import (+ PrivateRequestLogFilter,+ redact_private_foreign_event,+ redact_private_log_event,+)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.organization_age_cache import cached_org_age@@ -1694,6 +1699,7 @@ def setup_logger() -> None:wrapper_class=structlog.make_filtering_bound_logger(LOG_LEVEL_VAL),logger_factory=structlog.stdlib.LoggerFactory(),processors=[+ redact_private_log_event,_add_log_level,_LogTimeStamper(fmt="iso"),_add_entrypoint,@@ -1705,6 +1711,7 @@ def setup_logger() -> None:# Models and containers must be normalized before value scrubbing;# capture only copied, scrubbed values for later artifact serialization.redact_registered_secrets,+ redact_private_log_event,skyvern_logs_processor,sample_logs_processor,structlog.stdlib.ProcessorFormatter.wrap_for_formatter,@@ -1718,6 +1725,7 @@ def setup_logger() -> None:)handler = logging.StreamHandler()+ handler.addFilter(PrivateRequestLogFilter())handler.setFormatter(structlog.stdlib.ProcessorFormatter(foreign_pre_chain=[@@ -1725,6 +1733,7 @@ def setup_logger() -> None:# and never becomes a queryable attribute. Modules that run in images# without structlog installed have no other route to structured fields.structlog.stdlib.ExtraAdder(),+ redact_private_foreign_event,add_error_processor,structlog.processors.format_exc_info,redact_sensitive_event_fields,@@ -1747,6 +1756,7 @@ def setup_logger() -> None:redact_bearer_tokens,redact_registered_secrets,redact_codeblock_parameters,+ redact_private_log_event,*([escape_reserved_log_keys] if settings.JSON_LOGGING else []),renderer,],diff --git a/tests/unit/test_forge_log_redaction.py b/tests/unit/test_forge_log_redaction.pyindex 23e2cc4f7..a36b443b6 100644--- a/tests/unit/test_forge_log_redaction.py+++ b/tests/unit/test_forge_log_redaction.py@@ -1589,7 +1589,10 @@ def test_setup_logger_pins_redactor_processor_order() -> None:assert names.index("redact_bearer_tokens") < names.index("redact_sensitive_event_fields")assert names.index("compact_action_objects") < names.index("redact_sensitive_event_fields")assert names.index("redact_sensitive_event_fields") < names.index("redact_registered_secrets")- assert names.index("redact_registered_secrets") + 1 == names.index("skyvern_logs_processor")+ privacy_boundaries = [index for index, name in enumerate(names) if name == "redact_private_log_event"]+ assert privacy_boundaries[0] == 0+ assert privacy_boundaries[-1] == names.index("redact_registered_secrets") + 1+ assert privacy_boundaries[-1] + 1 == names.index("skyvern_logs_processor")finally:structlog.configure(**saved_config)root.handlers[:] = saved_handlersdiff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex f2d60ab9c..d4d8696d7 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -424,11 +424,8 @@ class TestIsLoggableContentType:# ----------------------------------------------------------------------------def _make_request(method: str = "GET", path: str = "/api/v1/test") -> MagicMock:- request = MagicMock()- request.method = method- request.url.path = path- return request+def _make_request(method: str = "GET", path: str = "/api/v1/test") -> Request:+ return Request({"type": "http", "method": method, "path": path, "headers": [], "query_string": b""})def test_sanitize_headers_removes_posthog_attribution_but_keeps_ordinary_header() -> None: