Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dify-agent): stabilize E2B config materialization (#39873)

Ingram Z66a69c30f51a
diff --git a/dify-agent/src/dify_agent/layers/config/layer.py b/dify-agent/src/dify_agent/layers/config/layer.py
index 7088179996f..7d68202d2a0 100644
--- a/dify-agent/src/dify_agent/layers/config/layer.py
+++ b/dify-agent/src/dify_agent/layers/config/layer.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-import asyncio
+import json
import shlex
from dataclasses import dataclass
from typing import ClassVar
@@ -194,71 +194,101 @@ class DifyConfigLayer(PlainLayer[DifyConfigDeps, DifyConfigLayerConfig, DifyConf
if not self.config.mentioned_skill_names and not self.config.mentioned_file_names:
return
- tasks = [
- *(self._pull_mentioned_skill(name) for name in self.config.mentioned_skill_names),
- *(self._pull_mentioned_file(name) for name in self.config.mentioned_file_names),
- ]
- await asyncio.gather(*tasks)
-
- async def _pull_mentioned_skill(self, name: str) -> None:
- result = await self.deps.shell.run_remote_script(
- self._build_shell_skill_pull_script(name),
- inject_agent_stub_env=True,
- )
- if result.exit_code != 0:
- raise DifyConfigLayerError(
- "config mentioned skill pull failed in shell: "
- f"{result.status} exit_code={result.exit_code}\n{result.output}"
+ if names := self.config.mentioned_skill_names:
+ output = await self._run_mentioned_pull(
+ script=self._build_shell_skill_pull_script(names),
+ target_kind="skill",
)
- if not result.output_complete:
- reason = result.incomplete_reason or "unknown"
- raise DifyConfigLayerError(
- f"config mentioned skill pull output was incomplete before the payload finished: {reason}"
+ self.runtime_state.pulled_skill_outputs = _parse_skill_pull_outputs(output, names)
+
+ if names := self.config.mentioned_file_names:
+ output = await self._run_mentioned_pull(
+ script=self._build_shell_file_pull_script(names),
+ target_kind="file",
)
- output = result.output.strip()
- if not output:
- raise DifyConfigLayerError(f"missing pull output for mentioned config skill {name}")
- self.runtime_state.pulled_skill_outputs = {
- **self.runtime_state.pulled_skill_outputs,
- name: output,
- }
+ self.runtime_state.pulled_file_outputs = _parse_file_pull_outputs(output, names)
- async def _pull_mentioned_file(self, name: str) -> None:
+ async def _run_mentioned_pull(self, *, script: str, target_kind: str) -> str:
result = await self.deps.shell.run_remote_script(
- self._build_shell_file_pull_script(name),
+ script,
inject_agent_stub_env=True,
)
if result.exit_code != 0:
raise DifyConfigLayerError(
- "config mentioned file pull failed in shell: "
+ f"config mentioned {target_kind} pull failed in shell: "
f"{result.status} exit_code={result.exit_code}\n{result.output}"
)
if not result.output_complete:
reason = result.incomplete_reason or "unknown"
raise DifyConfigLayerError(
- f"config mentioned file pull output was incomplete before the payload finished: {reason}"
+ f"config mentioned {target_kind} pull output was incomplete before the payload finished: {reason}"
)
output = result.output.strip()
if not output:
- raise DifyConfigLayerError(f"missing pull output for mentioned config file {name}")
- self.runtime_state.pulled_file_outputs = {
- **self.runtime_state.pulled_file_outputs,
- name: output,
- }
+ raise DifyConfigLayerError(f"missing pull output for mentioned config {target_kind}s")
+ return output
- def _build_shell_skill_pull_script(self, name: str) -> str:
- lines = [
- "set -eu",
- f"dify-agent config skills pull {shlex.quote(name)}",
- ]
- return "\n".join(lines)
+ def _build_shell_skill_pull_script(self, names: list[str]) -> str:
+ targets = " ".join(shlex.quote(name) for name in names)
+ return f"set -eu\ndify-agent config skills pull --json {targets}"
- def _build_shell_file_pull_script(self, name: str) -> str:
- lines = [
- "set -eu",
- f"dify-agent config files pull {shlex.quote(name)}",
- ]
- return "\n".join(lines)
+ def _build_shell_file_pull_script(self, names: list[str]) -> str:
+ targets = " ".join(shlex.quote(name) for name in names)
+ return f"set -eu\ndify-agent config files pull --json {targets}"
+
+
+def _parse_skill_pull_outputs(output: str, expected_names: list[str]) -> dict[str, str]:
+ items = _parse_pull_items(output, target_kind="skill")
+ parsed: dict[str, str] = {}
+ for name in expected_names:
+ item = items.get(name)
+ if item is None:
+ raise DifyConfigLayerError(f"missing pull output for mentioned config skill {name}")
+ directory_path = item.get("directory_path")
+ skill_md = item.get("skill_md")
+ if not isinstance(directory_path, str) or not directory_path:
+ raise DifyConfigLayerError(f"invalid directory path in pull output for mentioned config skill {name}")
+ if not isinstance(skill_md, str):
+ raise DifyConfigLayerError(f"invalid skill content in pull output for mentioned config skill {name}")
+ parsed[name] = f"{directory_path}\n{skill_md}".strip()
+ return parsed
+
+
+def _parse_file_pull_outputs(output: str, expected_names: list[str]) -> dict[str, str]:
+ items = _parse_pull_items(output, target_kind="file")
+ parsed: dict[str, str] = {}
+ for name in expected_names:
+ item = items.get(name)
+ if item is None:
+ raise DifyConfigLayerError(f"missing pull output for mentioned config file {name}")
+ path = item.get("path")
+ if not isinstance(path, str) or not path:
+ raise DifyConfigLayerError(f"invalid path in pull output for mentioned config file {name}")
+ parsed[name] = path
+ return parsed
+
+
+def _parse_pull_items(output: str, *, target_kind: str) -> dict[str, dict[str, object]]:
+ try:
+ payload: object = json.loads(output)
+ except json.JSONDecodeError as exc:
+ raise DifyConfigLayerError(f"invalid JSON pull output for mentioned config {target_kind}s") from exc
+ if not isinstance(payload, dict):
+ raise DifyConfigLayerError(f"invalid pull output for mentioned config {target_kind}s")
+ raw_items = payload.get("items")
+ if not isinstance(raw_items, list):
+ raise DifyConfigLayerError(f"missing items in pull output for mentioned config {target_kind}s")
+
+ items: dict[str, dict[str, object]] = {}
+ for raw_item in raw_items:
+ if not isinstance(raw_item, dict):
+ raise DifyConfigLayerError(f"invalid item in pull output for mentioned config {target_kind}s")
+ item = {str(key): value for key, value in raw_item.items()}
+ name = item.get("name")
+ if not isinstance(name, str) or not name:
+ raise DifyConfigLayerError(f"missing item name in pull output for mentioned config {target_kind}s")
+ items[name] = item
+ return items
def _format_command_output(command: str, output: str) -> str:
diff --git a/dify-agent/tests/local/dify_agent/layers/config/test_layer.py b/dify-agent/tests/local/dify_agent/layers/config/test_layer.py
index c222ddaab3e..990a19eb477 100644
--- a/dify-agent/tests/local/dify_agent/layers/config/test_layer.py
+++ b/dify-agent/tests/local/dify_agent/layers/config/test_layer.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-import asyncio
+import json
from typing import Literal
import pytest
@@ -62,32 +62,40 @@ def _remote_result(
)
-def _skill_pull_output(*, include_skill: bool = True) -> str:
- if not include_skill:
- return ""
- return "/workspace/.dify_conf/skills/alpha\n# Alpha\nUse it.\n"
+def _skill_pull_output(*names: str, include_skill: bool = True) -> str:
+ items = []
+ if include_skill:
+ items = [
+ {
+ "name": name,
+ "archive_path": f"/workspace/.dify_conf/skills/{name}.zip",
+ "directory_path": f"/workspace/.dify_conf/skills/{name}",
+ "skill_md": "# Alpha\nUse it.\n",
+ }
+ for name in names or ("alpha",)
+ ]
+ return json.dumps({"items": items})
-def _file_pull_output(*, include_file: bool = True) -> str:
- if not include_file:
- return ""
- return "/workspace/.dify_conf/files/guide.txt\n"
+def _file_pull_output(*names: str, include_file: bool = True) -> str:
+ items = []
+ if include_file:
+ items = [{"name": name, "path": f"/workspace/.dify_conf/files/{name}"} for name in names or ("guide.txt",)]
+ return json.dumps({"items": items})
def test_build_shell_pull_scripts_include_targets() -> None:
layer = _build_layer()
- skill_script = layer._build_shell_skill_pull_script("alpha")
- file_script = layer._build_shell_file_pull_script("guide.txt")
+ skill_script = layer._build_shell_skill_pull_script(["alpha", "skill with space"])
+ file_script = layer._build_shell_file_pull_script(["guide.txt", "file with space.txt"])
- assert skill_script == "set -eu\ndify-agent config skills pull alpha"
- assert "__DIFY_CONFIG_SKILLS_BEGIN__" not in skill_script
- assert file_script == "set -eu\ndify-agent config files pull guide.txt"
- assert "__DIFY_CONFIG_FILES_BEGIN__" not in file_script
+ assert skill_script == "set -eu\ndify-agent config skills pull --json alpha 'skill with space'"
+ assert file_script == "set -eu\ndify-agent config files pull --json guide.txt 'file with space.txt'"
@pytest.mark.anyio
-async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assets_in_parallel(
+async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assets_in_batches(
monkeypatch: pytest.MonkeyPatch,
) -> None:
layer = _build_layer()
@@ -104,7 +112,6 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_ass
captured_scripts.append(script)
active_commands += 1
max_active_commands = max(max_active_commands, active_commands)
- await asyncio.sleep(0)
active_commands -= 1
if "skills pull" in script:
return _remote_result(_skill_pull_output())
@@ -116,11 +123,11 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_ass
await layer.on_context_create()
- assert max_active_commands > 1
+ assert max_active_commands == 1
assert len(captured_scripts) == 2
- assert sorted(captured_scripts) == [
- "set -eu\ndify-agent config files pull guide.txt",
- "set -eu\ndify-agent config skills pull alpha",
+ assert captured_scripts == [
+ "set -eu\ndify-agent config skills pull --json alpha",
+ "set -eu\ndify-agent config files pull --json guide.txt",
]
assert layer.runtime_state.pulled_skill_outputs == {"alpha": "/workspace/.dify_conf/skills/alpha\n# Alpha\nUse it."}
assert layer.runtime_state.pulled_file_outputs == {"guide.txt": "/workspace/.dify_conf/files/guide.txt"}
@@ -140,6 +147,39 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_ass
assert _AGENT_FILE_UPLOAD_REPLY_HINT in suffix_prompt
+async def test_on_context_create_batches_all_mentioned_assets_into_two_serial_jobs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ layer = _build_layer()
+ layer.config = layer.config.model_copy(
+ update={
+ "mentioned_skill_names": [f"skill-{index}" for index in range(4)],
+ "mentioned_file_names": [f"file-{index}.txt" for index in range(4)],
+ }
+ )
+ captured_scripts: list[str] = []
+
+ async def fake_run_remote_script(self, script: str, *, inject_agent_stub_env: bool = False, timeout: float = 10.0):
+ del self, timeout
+ assert inject_agent_stub_env is True
+ captured_scripts.append(script)
+ if "skills pull" in script:
+ return _remote_result(_skill_pull_output(*(f"skill-{index}" for index in range(4))))
+ return _remote_result(_file_pull_output(*(f"file-{index}.txt" for index in range(4))))
+
+ monkeypatch.setattr(DifyShellLayer, "run_remote_script", fake_run_remote_script)
+
+ await layer.on_context_create()
+
+ assert captured_scripts == [
+ "set -eu\ndify-agent config skills pull --json skill-0 skill-1 skill-2 skill-3",
+ "set -eu\ndify-agent config files pull --json file-0.txt file-1.txt file-2.txt file-3.txt",
+ ]
+ assert set(layer.runtime_state.pulled_skill_outputs) == {f"skill-{index}" for index in range(4)}
+ assert set(layer.runtime_state.pulled_file_outputs) == {f"file-{index}.txt" for index in range(4)}
+
+
@pytest.mark.anyio
async def test_on_context_resume_does_not_recompute_or_pull(monkeypatch: pytest.MonkeyPatch) -> None:
layer = _build_layer()