Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

45.7 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from typing import cast

import pytest

import dify_agent.layers.shell.layer as shell_layer_module
import dify_agent.runtime.command_runner as command_runner_module
from dify_agent.layers.shell import (
    DifyShellEnvVarConfig,
    DifyShellLayerConfig,
)
from dify_agent.layers.shell.layer import (
    CompleteRemoteCommandResult,
    DEFAULT_TERMINATE_GRACE_SECONDS,
    ShellSession,
    State,
)
from dify_agent.adapters.shell.protocols import (
    ShellCommandResult,
    ShellCommandStatus,
    ShellExecutionMode,
    ShellProviderError,
)
from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
from dify_agent.runtime_backend import (
    RuntimeLayout,
    RuntimeLease,
)


def _command_result(
    job_id: str,
    *,
    status: str = "running",
    done: bool = False,
    exit_code: int | None = None,
    output: str = "",
    offset: int = 0,
    truncated: bool = False,
    output_path: str | None = "/tmp/output.log",
) -> ShellCommandResult:
    return ShellCommandResult(
        job_id=job_id,
        status=status,
        done=done,
        exit_code=exit_code,
        output=output,
        offset=offset,
        truncated=truncated,
        output_path=output_path,
    )


def _command_status(
    job_id: str,
    *,
    status: str = "terminated",
    done: bool = True,
    exit_code: int | None = 130,
    offset: int = 0,
) -> ShellCommandStatus:
    return ShellCommandStatus(job_id=job_id, status=status, done=done, exit_code=exit_code, offset=offset)


def _parse_tagged_observation(result: object) -> tuple[dict[str, object], str]:
    assert isinstance(result, str)
    metadata_tag = "\n</metadata>\n\n<output>\n"
    assert result.startswith("<metadata>\n")
    assert result.endswith("\n</output>")
    metadata_block, output_block = result.split(metadata_tag, 1)
    metadata = json.loads(metadata_block.removeprefix("<metadata>\n"))
    assert isinstance(metadata, dict)
    output = output_block.removesuffix("\n</output>")
    return cast(dict[str, object], metadata), output


def _assert_error_observation(result: object, *, job_id: str | None = None, includes: str | None = None) -> None:
    assert isinstance(result, dict)
    assert isinstance(result.get("error"), str)
    if job_id is None:
        assert "job_id" not in result
    else:
        assert result["job_id"] == job_id
    if includes is not None:
        assert includes in result["error"]


def _capture_logged_exceptions(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, tuple[object, ...]]]:
    logged: list[tuple[str, tuple[object, ...]]] = []

    def fake_exception(message: str, *args: object, **kwargs: object) -> None:
        del kwargs
        logged.append((message, args))

    monkeypatch.setattr(shell_layer_module.logger, "exception", fake_exception)
    return logged


@dataclass(slots=True)
class RunCall:
    script: str
    cwd: str | None
    env: Mapping[str, str] | None
    timeout: float
    mode: ShellExecutionMode = "pty"


@dataclass(slots=True)
class WaitCall:
    job_id: str
    offset: int
    timeout: float


@dataclass(slots=True)
class InputCall:
    job_id: str
    text: str
    offset: int
    timeout: float


@dataclass(slots=True)
class TailCall:
    job_id: str


@dataclass(slots=True)
class InterruptCall:
    job_id: str
    grace_seconds: float


@dataclass(slots=True)
class DeleteCall:
    job_id: str
    force: bool
    grace_seconds: float | None


class _UnexpectedToolError(Exception):
    pass


@dataclass(slots=True)
class FakeCommands:
    run_handler: Callable[[str, str | None, Mapping[str, str] | None, float], ShellCommandResult] | None = None
    wait_handler: Callable[[str, int, float], ShellCommandResult] | None = None
    input_handler: Callable[[str, str, int, float], ShellCommandResult] | None = None
    tail_handler: Callable[[str], ShellCommandResult] | None = None
    interrupt_handler: Callable[[str, float], ShellCommandStatus] | None = None
    delete_handler: Callable[[str, bool, float | None], None] | None = None
    run_calls: list[RunCall] = field(default_factory=list)
    wait_calls: list[WaitCall] = field(default_factory=list)
    input_calls: list[InputCall] = field(default_factory=list)
    tail_calls: list[TailCall] = field(default_factory=list)
    interrupt_calls: list[InterruptCall] = field(default_factory=list)
    delete_calls: list[DeleteCall] = field(default_factory=list)

    async def run(
        self,
        script: str,
        *,
        cwd: str | None = None,
        env: dict[str, str] | None = None,
        timeout: float,
        mode: ShellExecutionMode = "pty",
    ):
        self.run_calls.append(RunCall(script=script, cwd=cwd, env=env, timeout=timeout, mode=mode))
        if self.run_handler is None:
            raise AssertionError("Unexpected run() call")
        return self.run_handler(script, cwd, env, timeout)

    async def wait(self, job_id: str, *, offset: int, timeout: float):
        self.wait_calls.append(WaitCall(job_id=job_id, offset=offset, timeout=timeout))
        if self.wait_handler is None:
            raise AssertionError("Unexpected wait() call")
        return self.wait_handler(job_id, offset, timeout)

    async def read_output(self, job_id: str, *, offset: int):
        self.wait_calls.append(WaitCall(job_id=job_id, offset=offset, timeout=0.0))
        if self.wait_handler is None:
            raise AssertionError("Unexpected read_output() call")
        return self.wait_handler(job_id, offset, 0.0)

    async def input(self, job_id: str, text: str, *, offset: int, timeout: float):
        self.input_calls.append(InputCall(job_id=job_id, text=text, offset=offset, timeout=timeout))
        if self.input_handler is None:
            raise AssertionError("Unexpected input() call")
        return self.input_handler(job_id, text, offset, timeout)

    async def interrupt(self, job_id: str, *, grace_seconds: float):
        self.interrupt_calls.append(InterruptCall(job_id=job_id, grace_seconds=grace_seconds))
        if self.interrupt_handler is None:
            raise AssertionError("Unexpected interrupt() call")
        return self.interrupt_handler(job_id, grace_seconds)

    async def tail(self, job_id: str):
        self.tail_calls.append(TailCall(job_id=job_id))
        if self.tail_handler is None:
            raise AssertionError("Unexpected tail() call")
        return self.tail_handler(job_id)

    async def delete(self, job_id: str, *, force: bool = False, grace_seconds: float | None = None) -> None:
        self.delete_calls.append(DeleteCall(job_id=job_id, force=force, grace_seconds=grace_seconds))
        if self.delete_handler is not None:
            self.delete_handler(job_id, force, grace_seconds)


@dataclass(slots=True)
class FakeResource:
    commands: FakeCommands
    handle: str = "sandbox-1"
    layout: RuntimeLayout = field(
        default_factory=lambda: RuntimeLayout(
            home_dir="/home/agent-1",
            workspace_dir="/home/agent-1/workspace/abc12ff",
        )
    )


@dataclass(slots=True)
class FakeLeaseFixture:
    """Command fixture for an acquired runtime lease."""

    resource: FakeResource


def _layer(
    *,
    commands: FakeCommands,
    config: DifyShellLayerConfig | None = None,
    shell_home_root: str = "/home",
) -> tuple[ShellSession, FakeLeaseFixture]:
    provider = FakeLeaseFixture(resource=FakeResource(commands=commands))
    root = shell_home_root.rstrip("/")
    provider.resource.layout = RuntimeLayout(
        home_dir=f"{root}/agent-1",
        workspace_dir=f"{root}/agent-1/workspace/abc12ff",
    )
    from tests.local.dify_agent.module_support import deps_for

    cfg = config or DifyShellLayerConfig()
    cfg = cfg.model_copy(update={"execution_context": None})
    deps = deps_for("shell", cfg, State().model_dump(mode="json"))
    deps.resources.leases["runtime"] = cast(RuntimeLease, provider.resource)
    layer = ShellSession("shell", deps)
    return layer, provider


def _bind_execution_context(layer: ShellSession, *, agent_id: str | None = "agent-1") -> None:
    layer.deps.layers["execution_context"] = {
        "config": _execution_context_config(agent_id=agent_id).model_dump(mode="json"),
        "state": {},
    }
    layer.deps.layers["shell"]["config"]["execution_context"] = "execution_context"


def _execution_context_config(*, agent_id: str | None = None) -> DifyExecutionContextLayerConfig:
    return DifyExecutionContextLayerConfig(
        tenant_id="tenant-1",
        user_id="user-1",
        user_from="account",
        agent_id=agent_id,
        agent_mode="agent_app",
        invoke_from="service-api",
    )


def _runtime_state(
    *,
    session_id: str = "abc12ff",
    workspace_cwd: str = "~/workspace/abc12ff",
    sandbox_id: str | None = None,
    job_ids: list[str] | None = None,
    job_offsets: dict[str, int] | None = None,
) -> State:
    del session_id, workspace_cwd, sandbox_id
    return State(
        job_ids=[] if job_ids is None else job_ids,
        job_offsets={} if job_offsets is None else job_offsets,
    )


def test_shell_prefix_prompt_describes_workspace_as_temp_space() -> None:
    prompt = shell_layer_module._SHELL_LAYER_PREFIX_PROMPT

    assert "`cwd`) is the active Workspace and temporary working space" in prompt
    assert "`TMPDIR`, `TMP`, and `TEMP`) resolve directly to `cwd`" in prompt
    assert "`$HOME` is the system space for reusable tools and state" in prompt
    assert "<cwd>/.tmp" not in prompt


def test_shell_layer_uses_sandbox_layout_for_home_and_workspace_cwd() -> None:
    expected_home = "/home/agent-1"
    expected_workspace_cwd = "/home/agent-1/workspace/abc12ff"

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del timeout
        if script == "pwd":
            assert cwd == expected_workspace_cwd
            assert env == {"HOME": expected_home}
            return _command_result("user-job", status="exited", done=True, exit_code=0, output=expected_home, offset=13)
        raise AssertionError(f"Unexpected script: {script!r}")

    commands = FakeCommands(run_handler=run_handler)
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)

    async def scenario() -> None:
        run_result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        metadata, output = _parse_tagged_observation(run_result)
        assert metadata["job_id"] == "user-job"
        assert output == expected_home

    asyncio.run(scenario())

    assert layer.runtime_state.job_ids == ["user-job"]
    assert commands.delete_calls == []


def test_shell_layer_tools_map_inputs_and_maintain_offsets_with_tail_end() -> None:
    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        assert script == "pwd"
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1"}
        return _command_result(
            "user-job",
            status="running",
            done=False,
            output="head-output\n",
            offset=10,
            truncated=True,
            output_path="/tmp/initial.log",
        )

    def wait_handler(job_id: str, offset: int, timeout: float) -> ShellCommandResult:
        assert job_id == "user-job"
        if timeout == 4.0:
            assert offset == 22
            return _command_result("user-job", status="running", done=False, output="more\n", offset=30)
        raise AssertionError(f"Unexpected wait/read_output: offset={offset} timeout={timeout}")

    def input_handler(job_id: str, text: str, offset: int, timeout: float) -> ShellCommandResult:
        assert (job_id, text, offset, timeout) == ("user-job", "ls\n", 30, 5.0)
        return _command_result("user-job", status="exited", done=True, exit_code=0, output="file.txt\n", offset=34)

    def tail_handler(job_id: str) -> ShellCommandResult:
        assert job_id == "user-job"
        return _command_result(
            job_id,
            status="exited",
            done=True,
            exit_code=0,
            output="tail-output\n",
            offset=22,
            output_path="/tmp/resolved.log",
        )

    def interrupt_handler(job_id: str, grace_seconds: float) -> ShellCommandStatus:
        assert (job_id, grace_seconds) == ("user-job", 1.5)
        return _command_status("user-job", status="terminated", done=True, exit_code=130, offset=34)

    commands = FakeCommands(
        run_handler=run_handler,
        wait_handler=wait_handler,
        input_handler=input_handler,
        tail_handler=tail_handler,
        interrupt_handler=interrupt_handler,
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        run_result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        wait_result = await layer._tool_wait(**{"job_id": "user-job", "timeout": 4.0})
        input_result = await layer._tool_input(**{"job_id": "user-job", "text": "ls\n", "timeout": 5.0})
        interrupt_result = await layer._tool_interrupt(**{"job_id": "user-job", "grace_seconds": 1.5})

        run_metadata, run_output = _parse_tagged_observation(run_result)
        wait_metadata, wait_output = _parse_tagged_observation(wait_result)
        input_metadata, input_output = _parse_tagged_observation(input_result)
        interrupt_metadata, interrupt_output = _parse_tagged_observation(interrupt_result)

        assert run_metadata == {
            "job_id": "user-job",
            "status": "running",
            "done": False,
            "exit_code": None,
            "output_path": "/tmp/resolved.log",
        }
        assert "head-output" in run_output
        assert "tail-output" in run_output
        assert wait_metadata["job_id"] == "user-job"
        assert wait_output == "more\n"
        assert input_metadata["exit_code"] == 0
        assert input_output == "file.txt\n"
        assert interrupt_metadata == {
            "job_id": "user-job",
            "status": "terminated",
            "done": True,
            "exit_code": 130,
            "output_path": "/tmp/resolved.log",
        }
        assert interrupt_output == "Job was interrupted."

    asyncio.run(scenario())

    assert layer.runtime_state.job_offsets == {"user-job": 34}
    assert commands.run_calls[0].mode == "pty"
    assert commands.tail_calls == [TailCall(job_id="user-job"), TailCall(job_id="user-job")]


def test_shell_run_keeps_original_offset_when_tail_lookup_fails_for_truncated_output() -> None:
    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        assert script == "pwd"
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1"}
        return _command_result(
            "user-job",
            status="running",
            done=False,
            output="head-output\n",
            offset=10,
            truncated=True,
            output_path="/tmp/current.log",
        )

    def tail_handler(job_id: str) -> ShellCommandResult:
        raise RuntimeError(f"tail unavailable for {job_id}")

    commands = FakeCommands(run_handler=run_handler, tail_handler=tail_handler)
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        run_result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        metadata, output = _parse_tagged_observation(run_result)
        assert metadata == {
            "job_id": "user-job",
            "status": "running",
            "done": False,
            "exit_code": None,
            "output_path": "/tmp/current.log",
        }
        assert "head-output" in output

    asyncio.run(scenario())

    assert layer.runtime_state.job_offsets == {"user-job": 10}
    assert commands.tail_calls == [TailCall(job_id="user-job")]


def test_shell_run_formats_large_non_truncated_output_without_tail_lookup() -> None:
    large_output = ("head-" + ("x" * 9000) + "-tail").replace("head-x", "head-y", 1)
    commands = FakeCommands(
        run_handler=lambda script, cwd, env, timeout: _command_result(
            "user-job",
            status="exited",
            done=True,
            exit_code=0,
            output=large_output,
            offset=len(large_output),
            output_path="/tmp/large.log",
        )
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "cat large.log"})  # pyright: ignore[reportArgumentType]
        metadata, output = _parse_tagged_observation(result)
        assert metadata["output_path"] == "/tmp/large.log"
        assert output.startswith("head-y")
        assert "max output size is limited to 8192 bytes" in output
        assert output.endswith("(check the /tmp/large.log for full output)")
        assert "-tail" in output

    asyncio.run(scenario())
    assert commands.tail_calls == []


def test_shell_interrupt_succeeds_when_tail_lookup_fails() -> None:
    commands = FakeCommands(
        interrupt_handler=lambda job_id, grace_seconds: _command_status(job_id, offset=22),
        tail_handler=lambda job_id: (_ for _ in ()).throw(RuntimeError("tail unavailable")),
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 22}))
    layer._job_agent_stub_tokens["user-job"] = "actual-long-jwe-token"

    async def scenario() -> None:
        result = await layer._tool_interrupt(**{"job_id": "user-job"})  # pyright: ignore[reportArgumentType]
        metadata, output = _parse_tagged_observation(result)
        assert metadata == {
            "job_id": "user-job",
            "status": "terminated",
            "done": True,
            "exit_code": 130,
        }
        assert output == "Job was interrupted."

    asyncio.run(scenario())
    assert layer._job_agent_stub_tokens == {}


def test_shell_run_returns_provider_timeout_error_observation_without_unexpected_logging(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        run_handler=lambda script, cwd, env, timeout: (_ for _ in ()).throw(
            ShellProviderError("provider timed out", code="timeout")
        )
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        _assert_error_observation(result, includes="timeout")

    asyncio.run(scenario())
    assert logged == []


def test_shell_wait_returns_provider_request_error_observation_with_job_id_and_no_unexpected_logging(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        wait_handler=lambda job_id, offset, timeout: (_ for _ in ()).throw(
            ShellProviderError("provider unavailable", code="request_error")
        )
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 3}))

    async def scenario() -> None:
        result = await layer._tool_wait(**{"job_id": "user-job", "timeout": 4.0})
        _assert_error_observation(result, job_id="user-job", includes="request_error")

    asyncio.run(scenario())
    assert logged == []


def test_shell_input_returns_provider_timeout_error_observation_with_job_id_and_no_unexpected_logging(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        input_handler=lambda job_id, text, offset, timeout: (_ for _ in ()).throw(
            ShellProviderError("provider timed out", code="timeout")
        )
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 6}))

    async def scenario() -> None:
        result = await layer._tool_input(**{"job_id": "user-job", "text": "ls\n", "timeout": 5.0})
        _assert_error_observation(result, job_id="user-job", includes="timeout")

    asyncio.run(scenario())
    assert logged == []


def test_shell_interrupt_returns_provider_request_error_observation_with_job_id_and_no_unexpected_logging(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        interrupt_handler=lambda job_id, grace_seconds: (_ for _ in ()).throw(
            ShellProviderError("provider unavailable", code="request_error")
        )
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 22}))

    async def scenario() -> None:
        result = await layer._tool_interrupt(**{"job_id": "user-job", "grace_seconds": 1.5})
        _assert_error_observation(result, job_id="user-job", includes="request_error")

    asyncio.run(scenario())
    assert logged == []


def test_shell_run_returns_error_observation_and_logs_unexpected_exception(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        run_handler=lambda script, cwd, env, timeout: (_ for _ in ()).throw(_UnexpectedToolError("boom"))
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        _assert_error_observation(result, includes="boom")

    asyncio.run(scenario())
    assert logged == [
        (
            "Unexpected shell tool failure: tool=%s job_id=%s",
            ("shell_run", None),
        )
    ]


def test_shell_wait_returns_error_observation_and_logs_unexpected_exception(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        wait_handler=lambda job_id, offset, timeout: (_ for _ in ()).throw(_UnexpectedToolError("wait exploded"))
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 3}))

    async def scenario() -> None:
        result = await layer._tool_wait(**{"job_id": "user-job", "timeout": 4.0})
        _assert_error_observation(result, job_id="user-job", includes="wait exploded")

    asyncio.run(scenario())
    assert logged == [
        (
            "Unexpected shell tool failure: tool=%s job_id=%s",
            ("shell_wait", "user-job"),
        )
    ]


def test_shell_input_returns_error_observation_and_logs_unexpected_exception(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        input_handler=lambda job_id, text, offset, timeout: (_ for _ in ()).throw(
            _UnexpectedToolError("stdin exploded")
        )
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 6}))

    async def scenario() -> None:
        result = await layer._tool_input(**{"job_id": "user-job", "text": "ls\n", "timeout": 5.0})
        _assert_error_observation(result, job_id="user-job", includes="stdin exploded")

    asyncio.run(scenario())
    assert logged == [
        (
            "Unexpected shell tool failure: tool=%s job_id=%s",
            ("shell_input", "user-job"),
        )
    ]


def test_shell_interrupt_returns_error_observation_and_logs_unexpected_exception(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        interrupt_handler=lambda job_id, grace_seconds: (_ for _ in ()).throw(
            _UnexpectedToolError("interrupt exploded")
        )
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 22}))

    async def scenario() -> None:
        result = await layer._tool_interrupt(**{"job_id": "user-job", "grace_seconds": 1.5})
        _assert_error_observation(result, job_id="user-job", includes="interrupt exploded")

    asyncio.run(scenario())
    assert logged == [
        (
            "Unexpected shell tool failure: tool=%s job_id=%s",
            ("shell_interrupt", "user-job"),
        )
    ]


def test_shell_interrupt_logs_unexpected_tail_failure_but_still_succeeds(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logged = _capture_logged_exceptions(monkeypatch)
    commands = FakeCommands(
        interrupt_handler=lambda job_id, grace_seconds: _command_status(job_id, offset=22),
        tail_handler=lambda job_id: (_ for _ in ()).throw(_UnexpectedToolError("tail exploded")),
    )
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state(job_ids=["user-job"], job_offsets={"user-job": 22}))

    async def scenario() -> None:
        result = await layer._tool_interrupt(**{"job_id": "user-job"})  # pyright: ignore[reportArgumentType]
        metadata, output = _parse_tagged_observation(result)
        assert metadata == {
            "job_id": "user-job",
            "status": "terminated",
            "done": True,
            "exit_code": 130,
        }
        assert output == "Job was interrupted."

    asyncio.run(scenario())
    assert logged == [
        (
            "Failed to fetch output path for interrupted shell job %s",
            ("user-job",),
        )
    ]


def test_agent_stub_token_omits_workspace_session_identity() -> None:
    seen_session_ids: list[str | None] = []

    def token_factory(
        execution_context: DifyExecutionContextLayerConfig,
        *,
        session_id: str | None,
    ) -> str:
        del execution_context
        seen_session_ids.append(session_id)
        return "stub-token"

    def run_handler(
        _script: str,
        _cwd: str | None,
        env: Mapping[str, str] | None,
        _timeout: float,
    ) -> ShellCommandResult:
        assert env is not None
        assert env["DIFY_AGENT_STUB_AUTH_JWE"] == "stub-token"
        return _command_result("remote-job", status="exited", done=True, exit_code=0)

    layer, _provider = _layer(commands=FakeCommands(run_handler=run_handler))
    _bind_execution_context(layer)
    layer.deps.services.agent_stub_api_base_url = "http://localhost:5050/agent-stub"
    layer.deps.services.agent_stub_token_factory = token_factory

    async def scenario() -> None:
        _ = await layer.run_remote_script_complete("true", inject_agent_stub_env=True)

    asyncio.run(scenario())

    assert seen_session_ids == [None]


def test_shell_run_propagates_cancelled_error() -> None:
    commands = FakeCommands(
        run_handler=lambda script, cwd, env, timeout: (_ for _ in ()).throw(asyncio.CancelledError())
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        with pytest.raises(asyncio.CancelledError):
            await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]

    asyncio.run(scenario())


def test_run_remote_script_complete_uses_read_output_before_wait_and_deletes_job() -> None:
    events: list[str] = []

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        events.append("run")
        assert script == "printf 'abcdefghi'"
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1"}
        return _command_result("remote-job", status="running", done=False, output="abc", offset=3, truncated=True)

    def wait_handler(job_id: str, offset: int, timeout: float) -> ShellCommandResult:
        if timeout == 0.0:
            events.append("read_output")
            assert offset == 3
            return _command_result(job_id, status="running", done=False, output="def", offset=6)
        events.append("wait")
        assert offset == 6
        return _command_result(job_id, status="exited", done=True, exit_code=0, output="ghi", offset=9)

    commands = FakeCommands(run_handler=run_handler, wait_handler=wait_handler)
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script_complete("printf 'abcdefghi'")
        assert isinstance(result, CompleteRemoteCommandResult)
        assert result.output == "abcdefghi"
        assert result.output_complete is True
        assert result.incomplete_reason is None

    asyncio.run(scenario())
    assert events == ["run", "read_output", "wait"]
    assert commands.run_calls[0].mode == "stdio"
    assert [call.job_id for call in commands.delete_calls] == ["remote-job"]


def test_run_remote_script_complete_uses_agent_specific_home_and_workspace_cwd() -> None:
    expected_home = "/home/agent-1"
    expected_workspace_cwd = "/home/agent-1/workspace/abc12ff"

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del timeout
        assert script == "pwd"
        assert cwd == expected_workspace_cwd
        assert env == {"HOME": expected_home}
        return _command_result(
            "remote-job",
            status="exited",
            done=True,
            exit_code=0,
            output=expected_home,
            offset=len(expected_home),
        )

    commands = FakeCommands(run_handler=run_handler)
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script_complete("pwd")
        assert isinstance(result, CompleteRemoteCommandResult)
        assert result.output == expected_home

    asyncio.run(scenario())
    assert [call.job_id for call in commands.delete_calls] == ["remote-job"]


def test_run_remote_script_complete_passes_config_env_values_to_shellctl_env() -> None:
    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del timeout
        assert script == "printenv API_TOKEN"
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1", "API_TOKEN": "inline-secret-value"}
        return _command_result(
            "remote-job",
            status="exited",
            done=True,
            exit_code=0,
            output="inline-secret-value\n",
            offset=len("inline-secret-value\n"),
        )

    commands = FakeCommands(run_handler=run_handler)
    layer, _provider = _layer(
        commands=commands,
        config=DifyShellLayerConfig(env=[DifyShellEnvVarConfig(name="API_TOKEN", value="inline-secret-value")]),
    )
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script_complete("printenv API_TOKEN")
        assert isinstance(result, CompleteRemoteCommandResult)
        assert result.output == "inline-secret-value\n"

    asyncio.run(scenario())
    assert [call.job_id for call in commands.delete_calls] == ["remote-job"]


def test_run_remote_script_uses_agent_specific_home_and_workspace_cwd() -> None:
    expected_home = "/home/agent-1"
    expected_workspace_cwd = "/home/agent-1/workspace/abc12ff"

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del timeout
        assert script == "pwd"
        assert cwd == expected_workspace_cwd
        assert env == {"HOME": expected_home}
        return _command_result(
            "remote-job",
            status="exited",
            done=True,
            exit_code=0,
            output=expected_home,
            offset=len(expected_home),
        )

    commands = FakeCommands(run_handler=run_handler)
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script("pwd")
        assert isinstance(result, CompleteRemoteCommandResult)
        assert result.output == expected_home

    asyncio.run(scenario())
    assert [call.job_id for call in commands.delete_calls] == ["remote-job"]


def test_run_remote_script_complete_returns_incomplete_reason_for_output_limit() -> None:
    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del script, timeout
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1"}
        return _command_result(
            "remote-job",
            status="running",
            done=False,
            output="hello world",
            offset=11,
            truncated=True,
        )

    commands = FakeCommands(
        run_handler=run_handler,
        interrupt_handler=lambda job_id, grace_seconds: _command_status(job_id, status="terminated", offset=11),
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script_complete("printf 'hello world'", max_output_bytes=5)
        assert result.output == "hello"
        assert result.output_complete is False
        assert result.incomplete_reason == "output_limit"
        assert result.status == "terminated"

    asyncio.run(scenario())
    assert commands.wait_calls == []
    assert commands.interrupt_calls == [
        InterruptCall(job_id="remote-job", grace_seconds=DEFAULT_TERMINATE_GRACE_SECONDS)
    ]


def test_run_remote_script_complete_returns_incomplete_reason_for_timeout(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    now = 100.0

    def fake_monotonic() -> float:
        return now

    monkeypatch.setattr(command_runner_module.time, "monotonic", fake_monotonic)

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        nonlocal now
        assert script == "sleep 10"
        assert cwd == "/home/agent-1/workspace/abc12ff"
        assert env == {"HOME": "/home/agent-1"}
        assert timeout == pytest.approx(60.0, rel=0, abs=0.01)
        now = 161.0
        return _command_result("remote-job", status="running", done=False, output="hello", offset=5)

    commands = FakeCommands(
        run_handler=run_handler,
        interrupt_handler=lambda job_id, grace_seconds: _command_status(
            job_id,
            status="terminated",
            done=True,
            exit_code=130,
            offset=5,
        ),
    )
    layer, _provider = _layer(commands=commands)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer.run_remote_script_complete("sleep 10", timeout=60.0)
        assert result.output == "hello"
        assert result.output_complete is False
        assert result.incomplete_reason == "timeout"
        assert result.status == "terminated"
        assert result.exit_code == 130

    asyncio.run(scenario())
    assert commands.wait_calls == []
    assert commands.interrupt_calls == [
        InterruptCall(job_id="remote-job", grace_seconds=DEFAULT_TERMINATE_GRACE_SECONDS)
    ]
    assert [call.job_id for call in commands.delete_calls] == ["remote-job"]


def test_shell_layer_rejects_untracked_job_ids_without_provider_calls() -> None:
    commands = FakeCommands()
    layer, _provider = _layer(commands=commands)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        wait_result = await layer._tool_wait(**{"job_id": "missing"})  # pyright: ignore[reportArgumentType]
        input_result = await layer._tool_input(**{"job_id": "missing", "text": "hello"})
        interrupt_result = await layer._tool_interrupt(**{"job_id": "missing"})  # pyright: ignore[reportArgumentType]
        _assert_error_observation(wait_result, job_id="missing")
        _assert_error_observation(input_result, job_id="missing")
        _assert_error_observation(interrupt_result, job_id="missing")

    asyncio.run(scenario())
    assert commands.wait_calls == []
    assert commands.input_calls == []
    assert commands.interrupt_calls == []


def test_shell_layer_hooks_and_tools_fail_clearly_outside_active_resource_context() -> None:
    layer, _provider = _layer(commands=FakeCommands())
    layer.deps.resources.leases.clear()
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "pwd"})  # pyright: ignore[reportArgumentType]
        _assert_error_observation(result, includes="active Runtime lease")

    asyncio.run(scenario())


def test_shell_runtime_state_validates_offset_keys() -> None:
    state = State.model_validate(
        {
            "job_ids": ['job"bad with spaces'],
            "job_offsets": {'job"bad with spaces': 0},
        }
    )
    assert state.job_ids == ['job"bad with spaces']
    with pytest.raises(ValueError, match="unknown job ids"):
        _ = State.model_validate(
            {
                "job_ids": ["job-1"],
                "job_offsets": {"job-2": 3},
            }
        )


# ---------------------------------------------------------------------------
# Output redaction tests
# ---------------------------------------------------------------------------


def _layer_with_redaction(
    *,
    commands: FakeCommands,
    config: DifyShellLayerConfig | None = None,
    shell_redact_patterns: list[str] | None = None,
    token_value: str = "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0.fake-long-jwe-token-value",
) -> tuple[ShellSession, FakeLeaseFixture]:
    """Create a layer with agent_stub env injection and optional redaction patterns."""
    layer, provider = _layer(commands=commands, config=config)
    layer.deps.services.shell_redact_patterns = shell_redact_patterns or []
    layer.deps.services.agent_stub_api_base_url = "http://localhost:5050/agent-stub"
    layer.deps.services.agent_stub_token_factory = lambda execution_context, session_id: token_value
    return layer, provider


def test_shell_run_issues_one_jwe_and_redaction_does_not_issue_another() -> None:
    """The output filter must use the exact JWE injected into the shell job."""
    issued_tokens: list[str] = []

    def token_factory(execution_context: DifyExecutionContextLayerConfig, session_id: str | None) -> str:
        del execution_context, session_id
        token = f"actual-long-jwe-token-{len(issued_tokens) + 1}"
        issued_tokens.append(token)
        return token

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        del script, cwd, timeout
        assert env is not None
        token = env["DIFY_AGENT_STUB_AUTH_JWE"]
        return _command_result(
            "job-1",
            status="exited",
            done=True,
            exit_code=0,
            output=f"DIFY_AGENT_STUB_AUTH_JWE={token}\n",
            offset=100,
        )

    commands = FakeCommands(run_handler=run_handler)
    layer, _provider = _layer_with_redaction(commands=commands)
    layer.deps.services.agent_stub_token_factory = token_factory
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "env"})  # pyright: ignore[reportArgumentType]
        _, output = _parse_tagged_observation(result)
        assert issued_tokens[0] not in output
        assert "DIFY_AGENT_STUB_AUTH_JWE=***" in output

    asyncio.run(scenario())
    assert issued_tokens == ["actual-long-jwe-token-1"]
    assert layer._job_agent_stub_tokens == {}


def test_two_running_jobs_redact_and_clean_up_their_own_jwes_independently() -> None:
    tokens = ["actual-long-jwe-token-for-job-1", "actual-long-jwe-token-for-job-2"]
    issued_tokens: list[str] = []

    def token_factory(execution_context: DifyExecutionContextLayerConfig, session_id: str | None) -> str:
        del execution_context, session_id
        token = tokens[len(issued_tokens)]
        issued_tokens.append(token)
        return token

    def run_handler(
        script: str,
        cwd: str | None,
        env: Mapping[str, str] | None,
        timeout: float,
    ) -> ShellCommandResult:
        del script, cwd, timeout
        assert env is not None
        token = env["DIFY_AGENT_STUB_AUTH_JWE"]
        job_id = "job-1" if token == tokens[0] else "job-2"
        return _command_result(job_id, status="running", done=False, output="started\n", offset=8)

    commands = FakeCommands(
        run_handler=run_handler,
        wait_handler=lambda job_id, offset, timeout: _command_result(
            job_id,
            status="exited",
            done=True,
            exit_code=0,
            output=f"wait={tokens[0]}\n",
            offset=20,
        ),
        input_handler=lambda job_id, text, offset, timeout: _command_result(
            job_id,
            status="exited",
            done=True,
            exit_code=0,
            output=f"input={tokens[1]}\n",
            offset=40,
        ),
    )
    layer, _provider = _layer_with_redaction(commands=commands)
    layer.deps.services.agent_stub_token_factory = token_factory
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        await layer._tool_run(**{"script": "start first"})  # pyright: ignore[reportArgumentType]
        await layer._tool_run(**{"script": "start second"})  # pyright: ignore[reportArgumentType]
        assert issued_tokens == tokens

        wait_result = await layer._tool_wait(**{"job_id": "job-1"})  # pyright: ignore[reportArgumentType]
        _, wait_output = _parse_tagged_observation(wait_result)
        assert wait_output == "wait=***\n"

        input_result = await layer._tool_input(**{"job_id": "job-2", "text": "finish\n"})
        _, input_output = _parse_tagged_observation(input_result)
        assert input_output == "input=***\n"
        assert layer._job_agent_stub_tokens == {}

    asyncio.run(scenario())


def test_redact_output_applies_server_level_patterns() -> None:
    """Server-level regex patterns from env var should redact matching content."""

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        return _command_result(
            "job-1",
            status="exited",
            done=True,
            exit_code=0,
            output="api_key=sk-proj-abc123xyz\n",
            offset=30,
        )

    commands = FakeCommands(
        run_handler=run_handler,
        tail_handler=lambda _: _command_result("job-1", done=True, exit_code=0, status="exited", offset=30),
    )
    layer, _provider = _layer_with_redaction(
        commands=commands,
        shell_redact_patterns=[r"sk-proj-[A-Za-z0-9]+"],
    )
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "cat .env"})  # pyright: ignore[reportArgumentType]
        _, output = _parse_tagged_observation(result)
        assert "sk-proj-abc123xyz" not in output
        assert "api_key=***" in output

    asyncio.run(scenario())


def test_redact_output_applies_per_agent_config_patterns() -> None:
    """Per-agent redact_patterns from DifyShellLayerConfig should also apply."""

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        return _command_result(
            "job-1",
            status="exited",
            done=True,
            exit_code=0,
            output="token: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890\n",
            offset=50,
        )

    commands = FakeCommands(
        run_handler=run_handler,
        tail_handler=lambda _: _command_result("job-1", done=True, exit_code=0, status="exited", offset=50),
    )
    config = DifyShellLayerConfig(redact_patterns=[r"ghp_[A-Za-z0-9]{36}"])
    layer, _provider = _layer_with_redaction(commands=commands, config=config)
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "echo $TOKEN"})  # pyright: ignore[reportArgumentType]
        _, output = _parse_tagged_observation(result)
        assert "ghp_" not in output
        assert "token: ***" in output

    asyncio.run(scenario())


def test_redact_output_skips_short_jwe_values() -> None:
    """JWE values ≤8 chars should not be redacted to avoid false positives."""

    def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult:
        return _command_result(
            "job-1",
            status="exited",
            done=True,
            exit_code=0,
            output="short\n",
            offset=6,
        )

    commands = FakeCommands(
        run_handler=run_handler,
        tail_handler=lambda _: _command_result("job-1", done=True, exit_code=0, status="exited", offset=6),
    )
    # Token value is short — should NOT be redacted even if it appears in output.
    layer, _provider = _layer_with_redaction(commands=commands, token_value="short")
    _bind_execution_context(layer)
    layer.save_state(_runtime_state())

    async def scenario() -> None:
        result = await layer._tool_run(**{"script": "echo hi"})  # pyright: ignore[reportArgumentType]
        _, output = _parse_tagged_observation(result)
        assert "short" in output

    asyncio.run(scenario())