Skip to content

Automate browser based workflows with AI

Bind protected references to trusted consumers (SKY-12876) (#7970)

Aaron Perez06746c062b91
diff --git a/skyvern/forge/sdk/protected_reference.py b/skyvern/forge/sdk/protected_reference.py
index 2546bf76f..c4aa4f8e3 100644
--- a/skyvern/forge/sdk/protected_reference.py
+++ b/skyvern/forge/sdk/protected_reference.py
@@ -2,12 +2,153 @@
from __future__ import annotations
-from typing import Protocol
+import asyncio
+import secrets
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import NoReturn, Protocol
-from skyvern.forge.sdk.browser_action_policy import ProtectedReference
+from skyvern.forge.sdk.browser_action_policy import ProtectedReference, ProtectedReferenceKind
+
+ProtectedValueResolver = Callable[[], Awaitable[str]]
+
+
+class ProtectedReferenceErrorReason(StrEnum):
+ INCOMPLETE_BINDING = "incomplete_binding"
+ NOT_AUTHORIZED = "not_authorized"
+ RESOLUTION_FAILED = "resolution_failed"
+
+
+class ProtectedReferenceError(RuntimeError):
+ """A safe failure containing no protected identifiers or values."""
+
+ def __init__(self, reason: ProtectedReferenceErrorReason) -> None:
+ self.reason = reason
+ super().__init__(reason.value)
+
+
+def _raise_safe_error(reason: ProtectedReferenceErrorReason) -> NoReturn:
+ raise ProtectedReferenceError(reason)
class ProtectedReferenceResolver(Protocol):
"""Capability exposed only to final trusted sinks."""
async def resolve(self, ref: ProtectedReference, run_id: str, consumer_id: str) -> str: ...
+
+
+@dataclass(frozen=True, slots=True)
+class _BindingKey:
+ kind: ProtectedReferenceKind
+ reference_id: str
+ owner_id: str
+ run_id: str
+ consumer_id: str
+
+
+def _present(value: object) -> bool:
+ return isinstance(value, str) and bool(value.strip())
+
+
+def _sanitize_control_error(error: BaseException) -> BaseException:
+ if isinstance(error, BaseExceptionGroup):
+ children = [_sanitize_control_error(child) for child in error.exceptions]
+ return BaseExceptionGroup("protected resolver interrupted", children)
+ if isinstance(error, asyncio.CancelledError):
+ return asyncio.CancelledError()
+ if isinstance(error, KeyboardInterrupt):
+ return KeyboardInterrupt()
+ if isinstance(error, SystemExit):
+ code = error.code
+ return SystemExit(code if type(code) is int else int(code is not None))
+ if isinstance(error, GeneratorExit):
+ return GeneratorExit()
+ if isinstance(error, Exception):
+ return Exception()
+ return BaseException()
+
+
+async def _resolve_bound_value(resolver: ProtectedValueResolver) -> str:
+ control_error: BaseException | None = None
+ try:
+ resolved = await resolver()
+ except Exception:
+ pass
+ except BaseException as error:
+ control_error = _sanitize_control_error(error)
+ else:
+ if isinstance(resolved, str) and resolved:
+ return resolved
+ del resolved
+
+ del resolver
+ if control_error is not None:
+ safe_error = control_error
+ del control_error
+ raise safe_error
+ del control_error
+ # Raise after cleanup so provider exceptions and rejected values cannot survive in artifacts.
+ _raise_safe_error(ProtectedReferenceErrorReason.RESOLUTION_FAILED)
+
+
+class ProtectedReferenceStore:
+ """Binds opaque capabilities to deferred resolvers without retaining protected values."""
+
+ def __init__(self) -> None:
+ self._bindings: dict[_BindingKey, ProtectedValueResolver] = {}
+ self._issued_reference_ids: set[str] = set()
+
+ def bind(
+ self,
+ *,
+ kind: ProtectedReferenceKind,
+ owner_id: str,
+ run_id: str,
+ consumer_id: str,
+ resolver: ProtectedValueResolver,
+ ) -> ProtectedReference:
+ if (
+ not isinstance(kind, ProtectedReferenceKind)
+ or not _present(owner_id)
+ or not _present(run_id)
+ or not _present(consumer_id)
+ or not callable(resolver)
+ ):
+ del self, kind, owner_id, run_id, consumer_id, resolver
+ _raise_safe_error(ProtectedReferenceErrorReason.INCOMPLETE_BINDING)
+
+ reference_id = self._new_reference_id()
+ ref = ProtectedReference(kind=kind, reference_id=reference_id, owner_id=owner_id)
+ key = _BindingKey(kind, reference_id, owner_id, run_id, consumer_id)
+ self._bindings[key] = resolver
+ return ref
+
+ async def resolve(self, ref: ProtectedReference, run_id: str, consumer_id: str) -> str:
+ if (
+ not isinstance(ref, ProtectedReference)
+ or not ref.complete
+ or not _present(ref.reference_id)
+ or not _present(ref.owner_id)
+ or not _present(run_id)
+ or not _present(consumer_id)
+ ):
+ del self, ref, run_id, consumer_id
+ _raise_safe_error(ProtectedReferenceErrorReason.INCOMPLETE_BINDING)
+
+ key = _BindingKey(ref.kind, ref.reference_id, ref.owner_id, run_id, consumer_id)
+ resolver = self._bindings.get(key)
+ if resolver is None:
+ del self, ref, run_id, consumer_id, key, resolver
+ _raise_safe_error(ProtectedReferenceErrorReason.NOT_AUTHORIZED)
+
+ resolution = _resolve_bound_value(resolver)
+ del self, ref, run_id, consumer_id, key, resolver
+ return await resolution
+
+ def _new_reference_id(self) -> str:
+ while True:
+ reference_id = f"pref_{secrets.token_urlsafe(24)}"
+ if reference_id not in self._issued_reference_ids:
+ self._issued_reference_ids.add(reference_id)
+ return reference_id
diff --git a/tests/unit/test_protected_reference.py b/tests/unit/test_protected_reference.py
index f38c8bd95..30a2940b6 100644
--- a/tests/unit/test_protected_reference.py
+++ b/tests/unit/test_protected_reference.py
@@ -1,6 +1,69 @@
+import asyncio
+import dataclasses
import inspect
+import traceback
+from unittest.mock import AsyncMock
-from skyvern.forge.sdk.protected_reference import ProtectedReferenceResolver
+import pytest
+
+from skyvern.forge.sdk import protected_reference as protected_reference_module
+from skyvern.forge.sdk.browser_action_policy import ProtectedReference, ProtectedReferenceKind
+from skyvern.forge.sdk.protected_reference import (
+ ProtectedReferenceError,
+ ProtectedReferenceErrorReason,
+ ProtectedReferenceResolver,
+ ProtectedReferenceStore,
+ ProtectedValueResolver,
+)
+
+OWNER = "o_12876"
+RUN = "wr_12876"
+CONSUMER = "act_12876"
+SECRET_REFERENCE_ID = "cred_12876_password"
+SECRET_VALUE = "secret-value-that-must-not-leak"
+RAW_FILE_PATH = "/private/run-12876/protected-document.pdf"
+
+
+class RevealingResolver:
+ def __init__(self, *, raises: bool = False) -> None:
+ self.raises = raises
+ self.calls = 0
+
+ def __repr__(self) -> str:
+ return f"RevealingResolver({SECRET_VALUE!r}, {RAW_FILE_PATH!r})"
+
+ async def __call__(self) -> str:
+ self.calls += 1
+ if self.raises:
+ raise RuntimeError(f"lookup failed for {SECRET_VALUE} at {RAW_FILE_PATH}")
+ return SECRET_VALUE
+
+
+def assert_protected_data_absent_from_module_traceback(error: BaseException) -> None:
+ rendered = "".join(traceback.format_exception(error))
+ assert SECRET_VALUE not in rendered
+ assert RAW_FILE_PATH not in rendered
+ traceback_node = error.__traceback__
+ module_frames = 0
+ while traceback_node is not None:
+ if traceback_node.tb_frame.f_code.co_filename.endswith("/protected_reference.py"):
+ module_frames += 1
+ frame_locals = repr(traceback_node.tb_frame.f_locals)
+ assert SECRET_VALUE not in frame_locals
+ assert RAW_FILE_PATH not in frame_locals
+ traceback_node = traceback_node.tb_next
+
+ assert module_frames > 0
+
+
+def bind_secret(store: ProtectedReferenceStore, loader: ProtectedValueResolver) -> ProtectedReference:
+ return store.bind(
+ kind=ProtectedReferenceKind.SECRET,
+ owner_id=OWNER,
+ run_id=RUN,
+ consumer_id=CONSUMER,
+ resolver=loader,
+ )
def test_resolver_contract_is_consumer_bound() -> None:
@@ -12,3 +75,292 @@ def test_resolver_contract_is_consumer_bound() -> None:
assert signature.parameters["run_id"].annotation == "str"
assert signature.parameters["consumer_id"].annotation == "str"
assert signature.return_annotation == "str"
+
+
+def test_secret_binding_produces_an_opaque_reference_without_lookup() -> None:
+ loader = AsyncMock(return_value=SECRET_VALUE)
+ store = ProtectedReferenceStore()
+
+ ref = bind_secret(store, loader)
+
+ assert ref.kind is ProtectedReferenceKind.SECRET
+ assert ref.owner_id == OWNER
+ assert ref.reference_id.startswith("pref_")
+ assert ref.complete
+ loader.assert_not_awaited()
+ assert SECRET_VALUE not in repr(ref)
+ assert SECRET_VALUE not in repr(store)
+
+
+async def test_file_binding_uses_a_stable_opaque_capability_not_the_raw_path() -> None:
+ loader = AsyncMock(return_value=RAW_FILE_PATH)
+ store = ProtectedReferenceStore()
+
+ ref = store.bind(
+ kind=ProtectedReferenceKind.FILE,
+ owner_id=OWNER,
+ run_id=RUN,
+ consumer_id=CONSUMER,
+ resolver=loader,
+ )
+
+ assert ref.kind is ProtectedReferenceKind.FILE
+ assert ref.owner_id == OWNER
+ assert ref.reference_id.startswith("pref_")
+ loader.assert_not_awaited()
+ assert RAW_FILE_PATH not in repr(ref)
+ assert RAW_FILE_PATH not in repr(store)
+ reference_id = ref.reference_id
+ assert await store.resolve(ref, RUN, CONSUMER) == RAW_FILE_PATH
+ assert ref.reference_id == reference_id
+
+
+def test_binding_api_cannot_treat_a_raw_llm_path_or_uri_as_authority() -> None:
+ parameters = inspect.signature(ProtectedReferenceStore.bind).parameters
+
+ assert "reference_id" not in parameters
+ assert "path" not in parameters
+ assert "uri" not in parameters
+
+
+def test_each_binding_gets_a_distinct_opaque_capability() -> None:
+ loader = AsyncMock(return_value=SECRET_VALUE)
+ store = ProtectedReferenceStore()
+
+ first = bind_secret(store, loader)
+ second = bind_secret(store, loader)
+
+ assert first.reference_id != second.reference_id
+
+
+def test_reference_id_collision_retries_without_replacing_a_binding(monkeypatch: pytest.MonkeyPatch) -> None:
+ generated = iter(["collision", "collision", "unique"])
+ monkeypatch.setattr(protected_reference_module.secrets, "token_urlsafe", lambda _: next(generated))
+ loader = AsyncMock(return_value=SECRET_VALUE)
+ store = ProtectedReferenceStore()
+
+ first = bind_secret(store, loader)
+ second = bind_secret(store, loader)
+
+ assert first.reference_id == "pref_collision"
+ assert second.reference_id == "pref_unique"
+
+
[email protected](("field", "value"), [("kind", "secret"), ("resolver", object())])
+def test_binding_rejects_untyped_kinds_and_noncallable_resolvers(field: str, value: object) -> None:
+ arguments = {
+ "kind": ProtectedReferenceKind.SECRET,
+ "owner_id": OWNER,
+ "run_id": RUN,
+ "consumer_id": CONSUMER,
+ "resolver": AsyncMock(return_value=SECRET_VALUE),
+ field: value,
+ }
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ ProtectedReferenceStore().bind(**arguments)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.INCOMPLETE_BINDING
+
+
+ ("replacement", "resolve_run", "resolve_consumer"),
+ [
+ ({"reference_id": "cred_other"}, RUN, CONSUMER),
+ ({"owner_id": "o_other"}, RUN, CONSUMER),
+ ({"kind": ProtectedReferenceKind.FILE}, RUN, CONSUMER),
+ ({}, "wr_other", CONSUMER),
+ ({}, RUN, "act_other"),
+ ({"reference_id": RAW_FILE_PATH, "owner_id": SECRET_VALUE}, RUN, CONSUMER),
+ ],
+)
+async def test_every_capability_field_is_authorized_before_resolution(
+ replacement: dict[str, object], resolve_run: str, resolve_consumer: str
+) -> None:
+ loader = RevealingResolver()
+ store = ProtectedReferenceStore()
+ ref = dataclasses.replace(bind_secret(store, loader), **replacement)
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ await store.resolve(ref, resolve_run, resolve_consumer)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.NOT_AUTHORIZED
+ assert ref.reference_id not in str(caught.value)
+ assert OWNER not in str(caught.value)
+ assert RUN not in str(caught.value)
+ assert CONSUMER not in str(caught.value)
+ assert loader.calls == 0
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
+async def test_resolution_occurs_only_after_the_exact_binding_is_authorized() -> None:
+ loader = AsyncMock(return_value=SECRET_VALUE)
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+
+ resolved = await store.resolve(ref, RUN, CONSUMER)
+
+ assert resolved == SECRET_VALUE
+ loader.assert_awaited_once_with()
+ assert SECRET_VALUE not in repr(store)
+
+
+ "fields",
+ [
+ {"owner_id": ""},
+ {"owner_id": " "},
+ {"run_id": ""},
+ {"consumer_id": ""},
+ ],
+)
+def test_binding_fails_closed_on_incomplete_ownership_facts(fields: dict[str, str]) -> None:
+ loader = RevealingResolver()
+ arguments = {
+ "kind": ProtectedReferenceKind.SECRET,
+ "owner_id": OWNER,
+ "run_id": RUN,
+ "consumer_id": CONSUMER,
+ "resolver": loader,
+ **fields,
+ }
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ ProtectedReferenceStore().bind(**arguments)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.INCOMPLETE_BINDING
+ assert SECRET_VALUE not in str(caught.value)
+ assert loader.calls == 0
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
+ ("reference", "run_id", "consumer_id"),
+ [
+ (ProtectedReference(ProtectedReferenceKind.SECRET, RAW_FILE_PATH, ""), SECRET_VALUE, CONSUMER),
+ (ProtectedReference(ProtectedReferenceKind.SECRET, "", OWNER), RUN, CONSUMER),
+ (ProtectedReference(ProtectedReferenceKind.SECRET, SECRET_REFERENCE_ID, OWNER), "", CONSUMER),
+ (ProtectedReference(ProtectedReferenceKind.SECRET, SECRET_REFERENCE_ID, OWNER), RUN, ""),
+ ],
+)
+async def test_resolver_fails_closed_before_lookup_on_incomplete_ownership_facts(
+ reference: ProtectedReference, run_id: str, consumer_id: str
+) -> None:
+ loader = RevealingResolver()
+ store = ProtectedReferenceStore()
+ bind_secret(store, loader)
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ await store.resolve(reference, run_id, consumer_id)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.INCOMPLETE_BINDING
+ assert loader.calls == 0
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
+async def test_loader_failures_are_wrapped_without_sensitive_exception_context() -> None:
+ loader = RevealingResolver(raises=True)
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ await store.resolve(ref, RUN, CONSUMER)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.RESOLUTION_FAILED
+ assert caught.value.__context__ is None
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
[email protected]("resolved", [None, "", 7])
+async def test_loader_must_return_a_nonempty_string(resolved: object) -> None:
+ loader = AsyncMock(return_value=resolved)
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ await store.resolve(ref, RUN, CONSUMER)
+
+ assert caught.value.reason is ProtectedReferenceErrorReason.RESOLUTION_FAILED
+
+
+async def test_invalid_loader_value_is_removed_from_exception_traceback_locals() -> None:
+ loader = AsyncMock(return_value=f"{SECRET_VALUE} {RAW_FILE_PATH}".encode())
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+
+ with pytest.raises(ProtectedReferenceError) as caught:
+ await store.resolve(ref, RUN, CONSUMER)
+
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
+async def test_cancellation_preserves_semantics_without_protected_exception_state() -> None:
+ started = asyncio.Event()
+
+ async def loader() -> str:
+ protected_value = f"{SECRET_VALUE} {RAW_FILE_PATH}"
+ started.set()
+ await asyncio.sleep(60)
+ return protected_value
+
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+ task = asyncio.create_task(store.resolve(ref, RUN, CONSUMER))
+ await started.wait()
+ task.cancel(f"{SECRET_VALUE} {RAW_FILE_PATH}")
+
+ with pytest.raises(asyncio.CancelledError) as caught:
+ await task
+
+ assert task.cancelled()
+ assert caught.value.__context__ is None
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
[email protected]("error_type", [BaseException, KeyboardInterrupt, SystemExit, GeneratorExit])
+async def test_process_control_errors_preserve_type_without_protected_exception_state(
+ error_type: type[BaseException],
+) -> None:
+ message: object = 7 if error_type is SystemExit else f"{SECRET_VALUE} {RAW_FILE_PATH}"
+
+ async def loader() -> str:
+ raise error_type(message)
+
+ store = ProtectedReferenceStore()
+ ref = bind_secret(store, loader)
+
+ with pytest.raises(error_type) as caught:
+ await store.resolve(ref, RUN, CONSUMER)
+
+ assert caught.value.__context__ is None
+ if isinstance(caught.value, SystemExit):
+ assert caught.value.code == 7
+ assert_protected_data_absent_from_module_traceback(caught.value)
+
+
+async def test_base_exception_groups_are_recursively_sanitized() -> None:
+ group = BaseExceptionGroup(
+ f"group {SECRET_VALUE} {RAW_FILE_PATH}",
+ [SystemExit(SECRET_VALUE), BaseExceptionGroup(RAW_FILE_PATH, [KeyboardInterrupt(SECRET_VALUE)])],
+ )
+
+ async def loader() -> str:
+ raise group
+
+ store = ProtectedReferenceStore()
+ with pytest.raises(BaseExceptionGroup) as caught:
+ await store.resolve(bind_secret(store, loader), RUN, CONSUMER)
+ assert caught.value.__context__ is None
+ assert isinstance(caught.value.exceptions[0], SystemExit)
+ assert isinstance(caught.value.exceptions[1], BaseExceptionGroup)
+ assert_protected_data_absent_from_module_traceback(caught.value)