Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(dify-agent): stabilize E2B config materialization (#39873)
Ingram Z66a69c30f51a
diff --git a/dify-agent/src/dify_agent/layers/config/layer.py b/dify-agent/src/dify_agent/layers/config/layer.pyindex 7088179996f..7d68202d2a0 100644--- a/dify-agent/src/dify_agent/layers/config/layer.py+++ b/dify-agent/src/dify_agent/layers/config/layer.py@@ -2,7 +2,7 @@from __future__ import annotations-import asyncio+import jsonimport shlexfrom dataclasses import dataclassfrom typing import ClassVar@@ -194,71 +194,101 @@ class DifyConfigLayer(PlainLayer[DifyConfigDeps, DifyConfigLayerConfig, DifyConfif not self.config.mentioned_skill_names and not self.config.mentioned_file_names:return- tasks = [- *(self._pull_mentioned_skill(name) for name in self.config.mentioned_skill_names),- *(self._pull_mentioned_file(name) for name in self.config.mentioned_file_names),- ]- await asyncio.gather(*tasks)-- async def _pull_mentioned_skill(self, name: str) -> None:- result = await self.deps.shell.run_remote_script(- self._build_shell_skill_pull_script(name),- inject_agent_stub_env=True,- )- if result.exit_code != 0:- raise DifyConfigLayerError(- "config mentioned skill pull failed in shell: "- f"{result.status} exit_code={result.exit_code}\n{result.output}"+ if names := self.config.mentioned_skill_names:+ output = await self._run_mentioned_pull(+ script=self._build_shell_skill_pull_script(names),+ target_kind="skill",)- if not result.output_complete:- reason = result.incomplete_reason or "unknown"- raise DifyConfigLayerError(- f"config mentioned skill pull output was incomplete before the payload finished: {reason}"+ self.runtime_state.pulled_skill_outputs = _parse_skill_pull_outputs(output, names)++ if names := self.config.mentioned_file_names:+ output = await self._run_mentioned_pull(+ script=self._build_shell_file_pull_script(names),+ target_kind="file",)- output = result.output.strip()- if not output:- raise DifyConfigLayerError(f"missing pull output for mentioned config skill {name}")- self.runtime_state.pulled_skill_outputs = {- **self.runtime_state.pulled_skill_outputs,- name: output,- }+ self.runtime_state.pulled_file_outputs = _parse_file_pull_outputs(output, names)- async def _pull_mentioned_file(self, name: str) -> None:+ async def _run_mentioned_pull(self, *, script: str, target_kind: str) -> str:result = await self.deps.shell.run_remote_script(- self._build_shell_file_pull_script(name),+ script,inject_agent_stub_env=True,)if result.exit_code != 0:raise DifyConfigLayerError(- "config mentioned file pull failed in shell: "+ f"config mentioned {target_kind} pull failed in shell: "f"{result.status} exit_code={result.exit_code}\n{result.output}")if not result.output_complete:reason = result.incomplete_reason or "unknown"raise DifyConfigLayerError(- f"config mentioned file pull output was incomplete before the payload finished: {reason}"+ f"config mentioned {target_kind} pull output was incomplete before the payload finished: {reason}")output = result.output.strip()if not output:- raise DifyConfigLayerError(f"missing pull output for mentioned config file {name}")- self.runtime_state.pulled_file_outputs = {- **self.runtime_state.pulled_file_outputs,- name: output,- }+ raise DifyConfigLayerError(f"missing pull output for mentioned config {target_kind}s")+ return output- def _build_shell_skill_pull_script(self, name: str) -> str:- lines = [- "set -eu",- f"dify-agent config skills pull {shlex.quote(name)}",- ]- return "\n".join(lines)+ def _build_shell_skill_pull_script(self, names: list[str]) -> str:+ targets = " ".join(shlex.quote(name) for name in names)+ return f"set -eu\ndify-agent config skills pull --json {targets}"- def _build_shell_file_pull_script(self, name: str) -> str:- lines = [- "set -eu",- f"dify-agent config files pull {shlex.quote(name)}",- ]- return "\n".join(lines)+ def _build_shell_file_pull_script(self, names: list[str]) -> str:+ targets = " ".join(shlex.quote(name) for name in names)+ return f"set -eu\ndify-agent config files pull --json {targets}"+++def _parse_skill_pull_outputs(output: str, expected_names: list[str]) -> dict[str, str]:+ items = _parse_pull_items(output, target_kind="skill")+ parsed: dict[str, str] = {}+ for name in expected_names:+ item = items.get(name)+ if item is None:+ raise DifyConfigLayerError(f"missing pull output for mentioned config skill {name}")+ directory_path = item.get("directory_path")+ skill_md = item.get("skill_md")+ if not isinstance(directory_path, str) or not directory_path:+ raise DifyConfigLayerError(f"invalid directory path in pull output for mentioned config skill {name}")+ if not isinstance(skill_md, str):+ raise DifyConfigLayerError(f"invalid skill content in pull output for mentioned config skill {name}")+ parsed[name] = f"{directory_path}\n{skill_md}".strip()+ return parsed+++def _parse_file_pull_outputs(output: str, expected_names: list[str]) -> dict[str, str]:+ items = _parse_pull_items(output, target_kind="file")+ parsed: dict[str, str] = {}+ for name in expected_names:+ item = items.get(name)+ if item is None:+ raise DifyConfigLayerError(f"missing pull output for mentioned config file {name}")+ path = item.get("path")+ if not isinstance(path, str) or not path:+ raise DifyConfigLayerError(f"invalid path in pull output for mentioned config file {name}")+ parsed[name] = path+ return parsed+++def _parse_pull_items(output: str, *, target_kind: str) -> dict[str, dict[str, object]]:+ try:+ payload: object = json.loads(output)+ except json.JSONDecodeError as exc:+ raise DifyConfigLayerError(f"invalid JSON pull output for mentioned config {target_kind}s") from exc+ if not isinstance(payload, dict):+ raise DifyConfigLayerError(f"invalid pull output for mentioned config {target_kind}s")+ raw_items = payload.get("items")+ if not isinstance(raw_items, list):+ raise DifyConfigLayerError(f"missing items in pull output for mentioned config {target_kind}s")++ items: dict[str, dict[str, object]] = {}+ for raw_item in raw_items:+ if not isinstance(raw_item, dict):+ raise DifyConfigLayerError(f"invalid item in pull output for mentioned config {target_kind}s")+ item = {str(key): value for key, value in raw_item.items()}+ name = item.get("name")+ if not isinstance(name, str) or not name:+ raise DifyConfigLayerError(f"missing item name in pull output for mentioned config {target_kind}s")+ items[name] = item+ return itemsdef _format_command_output(command: str, output: str) -> str:diff --git a/dify-agent/tests/local/dify_agent/layers/config/test_layer.py b/dify-agent/tests/local/dify_agent/layers/config/test_layer.pyindex c222ddaab3e..990a19eb477 100644--- a/dify-agent/tests/local/dify_agent/layers/config/test_layer.py+++ b/dify-agent/tests/local/dify_agent/layers/config/test_layer.py@@ -2,7 +2,7 @@from __future__ import annotations-import asyncio+import jsonfrom typing import Literalimport pytest@@ -62,32 +62,40 @@ def _remote_result()-def _skill_pull_output(*, include_skill: bool = True) -> str:- if not include_skill:- return ""- return "/workspace/.dify_conf/skills/alpha\n# Alpha\nUse it.\n"+def _skill_pull_output(*names: str, include_skill: bool = True) -> str:+ items = []+ if include_skill:+ items = [+ {+ "name": name,+ "archive_path": f"/workspace/.dify_conf/skills/{name}.zip",+ "directory_path": f"/workspace/.dify_conf/skills/{name}",+ "skill_md": "# Alpha\nUse it.\n",+ }+ for name in names or ("alpha",)+ ]+ return json.dumps({"items": items})-def _file_pull_output(*, include_file: bool = True) -> str:- if not include_file:- return ""- return "/workspace/.dify_conf/files/guide.txt\n"+def _file_pull_output(*names: str, include_file: bool = True) -> str:+ items = []+ if include_file:+ items = [{"name": name, "path": f"/workspace/.dify_conf/files/{name}"} for name in names or ("guide.txt",)]+ return json.dumps({"items": items})def test_build_shell_pull_scripts_include_targets() -> None:layer = _build_layer()- skill_script = layer._build_shell_skill_pull_script("alpha")- file_script = layer._build_shell_file_pull_script("guide.txt")+ skill_script = layer._build_shell_skill_pull_script(["alpha", "skill with space"])+ file_script = layer._build_shell_file_pull_script(["guide.txt", "file with space.txt"])- assert skill_script == "set -eu\ndify-agent config skills pull alpha"- assert "__DIFY_CONFIG_SKILLS_BEGIN__" not in skill_script- assert file_script == "set -eu\ndify-agent config files pull guide.txt"- assert "__DIFY_CONFIG_FILES_BEGIN__" not in file_script+ assert skill_script == "set -eu\ndify-agent config skills pull --json alpha 'skill with space'"+ assert file_script == "set -eu\ndify-agent config files pull --json guide.txt 'file with space.txt'"@pytest.mark.anyio-async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assets_in_parallel(+async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assets_in_batches(monkeypatch: pytest.MonkeyPatch,) -> None:layer = _build_layer()@@ -104,7 +112,6 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_asscaptured_scripts.append(script)active_commands += 1max_active_commands = max(max_active_commands, active_commands)- await asyncio.sleep(0)active_commands -= 1if "skills pull" in script:return _remote_result(_skill_pull_output())@@ -116,11 +123,11 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assawait layer.on_context_create()- assert max_active_commands > 1+ assert max_active_commands == 1assert len(captured_scripts) == 2- assert sorted(captured_scripts) == [- "set -eu\ndify-agent config files pull guide.txt",- "set -eu\ndify-agent config skills pull alpha",+ assert captured_scripts == [+ "set -eu\ndify-agent config skills pull --json alpha",+ "set -eu\ndify-agent config files pull --json guide.txt",]assert layer.runtime_state.pulled_skill_outputs == {"alpha": "/workspace/.dify_conf/skills/alpha\n# Alpha\nUse it."}assert layer.runtime_state.pulled_file_outputs == {"guide.txt": "/workspace/.dify_conf/files/guide.txt"}@@ -140,6 +147,39 @@ async def test_on_context_create_computes_runtime_fields_and_pulls_mentioned_assassert _AGENT_FILE_UPLOAD_REPLY_HINT in suffix_prompt+async def test_on_context_create_batches_all_mentioned_assets_into_two_serial_jobs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ layer = _build_layer()+ layer.config = layer.config.model_copy(+ update={+ "mentioned_skill_names": [f"skill-{index}" for index in range(4)],+ "mentioned_file_names": [f"file-{index}.txt" for index in range(4)],+ }+ )+ captured_scripts: list[str] = []++ async def fake_run_remote_script(self, script: str, *, inject_agent_stub_env: bool = False, timeout: float = 10.0):+ del self, timeout+ assert inject_agent_stub_env is True+ captured_scripts.append(script)+ if "skills pull" in script:+ return _remote_result(_skill_pull_output(*(f"skill-{index}" for index in range(4))))+ return _remote_result(_file_pull_output(*(f"file-{index}.txt" for index in range(4))))++ monkeypatch.setattr(DifyShellLayer, "run_remote_script", fake_run_remote_script)++ await layer.on_context_create()++ assert captured_scripts == [+ "set -eu\ndify-agent config skills pull --json skill-0 skill-1 skill-2 skill-3",+ "set -eu\ndify-agent config files pull --json file-0.txt file-1.txt file-2.txt file-3.txt",+ ]+ assert set(layer.runtime_state.pulled_skill_outputs) == {f"skill-{index}" for index in range(4)}+ assert set(layer.runtime_state.pulled_file_outputs) == {f"file-{index}.txt" for index in range(4)}++@pytest.mark.anyioasync def test_on_context_resume_does_not_recompute_or_pull(monkeypatch: pytest.MonkeyPatch) -> None:layer = _build_layer()