Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

test(e2e): cover all routes/modes, render dashboard, fix issues found (#42)

Yanami-2K96e38f946b51
diff --git a/.claude/skills/agent-observability/scripts/build_dashboard_data.py b/.claude/skills/agent-observability/scripts/build_dashboard_data.py
index e33f816a..4c639f1a 100644
--- a/.claude/skills/agent-observability/scripts/build_dashboard_data.py
+++ b/.claude/skills/agent-observability/scripts/build_dashboard_data.py
@@ -522,6 +522,12 @@ def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None
"exec": info.get("executor") or "-", "status": status,
"retry_count": retry_count, "duration": duration,
})
+ # 只有所有阶段都已完成或跳过时才算“已完成”;停在两个阶段之间(等待审批、
+ # 下一阶段尚未派发)的运行仍是进行中,不能因为没有 in_progress 就显示完成。
+ if overall == "completed" and any(
+ stage["status"] not in {"completed", "skipped"} for stage in stages_out
+ ):
+ overall = "running"
# 按真实起止时间跨度算总时长(而不是逐阶段 duration 相加)——阶段之间可能
# 有重叠(比如 TASK-05 的 started_at 早于 SOLO 的 completed_at),相加会
# 把重叠部分重复计入,虚高于源数据本身反映的运行时长。
diff --git a/.claude/skills/agent-observability/scripts/core/agent_identity.py b/.claude/skills/agent-observability/scripts/core/agent_identity.py
index 270ff904..1255f2e0 100644
--- a/.claude/skills/agent-observability/scripts/core/agent_identity.py
+++ b/.claude/skills/agent-observability/scripts/core/agent_identity.py
@@ -162,7 +162,8 @@ def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Option
next_target = _extract_next_target(payload)
if from_role == "main" and mailbox_role and mailbox_role != "main":
- if not msg.get("is_shutdown"):
+ # A standby assignment only creates the member; it is not active yet.
+ if not msg.get("is_shutdown") and not msg.get("is_standby"):
current = mailbox_role
dispatched = mailbox_role
evidence = f"dispatch:{mailbox_name}"
@@ -521,6 +522,7 @@ def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, i
"ts": ts,
"index": index,
"is_shutdown": _is_shutdown_message(payload, summary, text),
+ "is_standby": _is_standby_message(payload, text),
}
@@ -561,6 +563,14 @@ def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any
return "shutdown" in summary_low or "shutdown request" in text_low
+def _is_standby_message(payload: dict[str, Any] | None, text: Any) -> bool:
+ """识别“先创建成员、待命等待唤醒”的消息;这类消息不代表成员已开始工作。"""
+ if isinstance(payload, dict) and (payload.get("event") or payload.get("dispatch_prompt")):
+ return False
+ text_low = text.lower() if isinstance(text, str) else ""
+ return "待命" in text_low or "standby" in text_low or "stand by" in text_low
+
+
def _parse_timestamp(value: Any) -> float:
"""把消息时间字段解析成 Unix 时间戳。"""
if not isinstance(value, str) or not value.strip():
diff --git a/.claude/skills/agent-observability/scripts/core/cls_sink.py b/.claude/skills/agent-observability/scripts/core/cls_sink.py
index 6525f6ed..e2c72391 100644
--- a/.claude/skills/agent-observability/scripts/core/cls_sink.py
+++ b/.claude/skills/agent-observability/scripts/core/cls_sink.py
@@ -338,6 +338,10 @@ def load_config(cwd: str | None = None) -> CLSConfig:
def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool:
config = load_config(cwd)
+ if not (config.endpoint or config.topic_id):
+ # CLS mirroring was never configured: stay silent instead of writing a
+ # debug line into the skill directory for every local event.
+ return False
if not config.ready:
_debug_write(
{
diff --git a/.claude/skills/agent-observability/tests/test_build_dashboard_data.py b/.claude/skills/agent-observability/tests/test_build_dashboard_data.py
index 53b031c1..6ee93f75 100644
--- a/.claude/skills/agent-observability/tests/test_build_dashboard_data.py
+++ b/.claude/skills/agent-observability/tests/test_build_dashboard_data.py
@@ -395,6 +395,36 @@ class BuildDevflowRunsClassicSoloTests(unittest.TestCase):
self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段
+class BuildDevflowRunsOverallStatusTests(unittest.TestCase):
+ """E2E 发现:刚开始或停在两个阶段之间的运行(没有 in_progress、也没有 failed)
+ 曾被显示为“已完成”。"""
+
+ def _overall(self, stages):
+ with tempfile.TemporaryDirectory() as td:
+ project_root = Path(td)
+ run_dir = project_root / "artifacts" / "between-stages"
+ run_dir.mkdir(parents=True)
+ (run_dir / "workflow-state.json").write_text(json.dumps({
+ "version": "1.3", "task_slug": "between-stages", "size_class": "medium",
+ "current_stage": "TASK-02", "stages": stages,
+ }), encoding="utf-8")
+ return bdd.build_devflow_runs(project_root)[0]["overall"]
+
+ def test_run_waiting_between_stages_is_not_completed(self):
+ stages = {
+ "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0},
+ "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0},
+ "TASK-03": {"status": "pending", "executor": "developer", "retry_count": 0},
+ }
+ self.assertEqual(self._overall(stages), "running")
+
+ def test_fully_finished_run_is_completed(self):
+ names = ("TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05")
+ stages = {name: {"status": "completed", "executor": "x", "retry_count": 0} for name in names}
+ stages["TASK-05"]["status"] = "skipped"
+ self.assertEqual(self._overall(stages), "completed")
+
+
class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase):
"""用户反馈复现:workflow-state.json 模板固定给每次运行都写一份 SOLO stage
(见 assets/workflow-state-template.json 里 SOLO.description 的说明:
diff --git a/.claude/skills/agent-observability/tests/test_cls_sink.py b/.claude/skills/agent-observability/tests/test_cls_sink.py
index 2da9d6a1..7fdee02a 100644
--- a/.claude/skills/agent-observability/tests/test_cls_sink.py
+++ b/.claude/skills/agent-observability/tests/test_cls_sink.py
@@ -17,6 +17,22 @@ from core import cls_sink, emitter # type: ignore
class CLSSinkTests(unittest.TestCase):
+ def setUp(self):
+ # Keep the suite hermetic: never write debug logs into the source tree and
+ # do not depend on the optional Node SDK being checked out next to the repo.
+ sandbox = tempfile.TemporaryDirectory()
+ self.addCleanup(sandbox.cleanup)
+ sdk_entry = Path(sandbox.name) / "tencentcloud-cls-sdk-js" / "dist" / "index.js"
+ sdk_entry.parent.mkdir(parents=True)
+ sdk_entry.write_text("module.exports = {};\n", encoding="utf-8")
+ for name, value in (
+ ("sdk_entry_path", sdk_entry),
+ ("debug_log_path", Path(sandbox.name) / "cls-push-debug.ndjson"),
+ ):
+ patcher = mock.patch.object(cls_sink, name, return_value=value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
def _write_cls_env(self, root: Path, *extra_lines: str) -> None:
lines = [
"CLS_TOPIC_ID=test-topic-id",
@@ -171,6 +187,18 @@ class CLSSinkTests(unittest.TestCase):
self.assertEqual(entries[-1]["stage"], "uploader_ok")
self.assertEqual(entries[-1]["method"], "python_api_v3")
+ def test_unconfigured_cls_stays_silent(self):
+ with tempfile.TemporaryDirectory() as td:
+ debug_path = Path(td) / "debug.ndjson"
+ with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \
+ mock.patch.object(cls_sink.subprocess, "run") as run, \
+ mock.patch.dict(os.environ, {}, clear=True):
+ ok = cls_sink.mirror_record({"event": "start", "sid": "s-off"}, cwd=td)
+
+ self.assertFalse(ok)
+ run.assert_not_called()
+ self.assertFalse(debug_path.exists())
+
def test_emit_keeps_local_log_when_cls_push_raises(self):
with tempfile.TemporaryDirectory() as td:
log_path = Path(td) / "metrics.ndjson"
diff --git a/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py b/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py
index e33f816a..4c639f1a 100644
--- a/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py
+++ b/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py
@@ -522,6 +522,12 @@ def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None
"exec": info.get("executor") or "-", "status": status,
"retry_count": retry_count, "duration": duration,
})
+ # 只有所有阶段都已完成或跳过时才算“已完成”;停在两个阶段之间(等待审批、
+ # 下一阶段尚未派发)的运行仍是进行中,不能因为没有 in_progress 就显示完成。
+ if overall == "completed" and any(
+ stage["status"] not in {"completed", "skipped"} for stage in stages_out
+ ):
+ overall = "running"
# 按真实起止时间跨度算总时长(而不是逐阶段 duration 相加)——阶段之间可能
# 有重叠(比如 TASK-05 的 started_at 早于 SOLO 的 completed_at),相加会
# 把重叠部分重复计入,虚高于源数据本身反映的运行时长。
diff --git a/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py b/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py
index 270ff904..1255f2e0 100644
--- a/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py
+++ b/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py
@@ -162,7 +162,8 @@ def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Option
next_target = _extract_next_target(payload)
if from_role == "main" and mailbox_role and mailbox_role != "main":
- if not msg.get("is_shutdown"):
+ # A standby assignment only creates the member; it is not active yet.
+ if not msg.get("is_shutdown") and not msg.get("is_standby"):
current = mailbox_role
dispatched = mailbox_role
evidence = f"dispatch:{mailbox_name}"
@@ -521,6 +522,7 @@ def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, i
"ts": ts,
"index": index,
"is_shutdown": _is_shutdown_message(payload, summary, text),
+ "is_standby": _is_standby_message(payload, text),
}
@@ -561,6 +563,14 @@ def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any
return "shutdown" in summary_low or "shutdown request" in text_low
+def _is_standby_message(payload: dict[str, Any] | None, text: Any) -> bool:
+ """识别“先创建成员、待命等待唤醒”的消息;这类消息不代表成员已开始工作。"""
+ if isinstance(payload, dict) and (payload.get("event") or payload.get("dispatch_prompt")):
+ return False
+ text_low = text.lower() if isinstance(text, str) else ""
+ return "待命" in text_low or "standby" in text_low or "stand by" in text_low
+
+
def _parse_timestamp(value: Any) -> float:
"""把消息时间字段解析成 Unix 时间戳。"""
if not isinstance(value, str) or not value.strip():
diff --git a/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py b/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py
index 6525f6ed..e2c72391 100644
--- a/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py
+++ b/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py
@@ -338,6 +338,10 @@ def load_config(cwd: str | None = None) -> CLSConfig:
def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool:
config = load_config(cwd)
+ if not (config.endpoint or config.topic_id):
+ # CLS mirroring was never configured: stay silent instead of writing a
+ # debug line into the skill directory for every local event.
+ return False
if not config.ready:
_debug_write(
{
diff --git a/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py b/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py
index 53b031c1..6ee93f75 100644
--- a/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py
+++ b/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py
@@ -395,6 +395,36 @@ class BuildDevflowRunsClassicSoloTests(unittest.TestCase):
self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段
+class BuildDevflowRunsOverallStatusTests(unittest.TestCase):
+ """E2E 发现:刚开始或停在两个阶段之间的运行(没有 in_progress、也没有 failed)
+ 曾被显示为“已完成”。"""
+
+ def _overall(self, stages):
+ with tempfile.TemporaryDirectory() as td:
+ project_root = Path(td)
+ run_dir = project_root / "artifacts" / "between-stages"
+ run_dir.mkdir(parents=True)
+ (run_dir / "workflow-state.json").write_text(json.dumps({
+ "version": "1.3", "task_slug": "between-stages", "size_class": "medium",
+ "current_stage": "TASK-02", "stages": stages,
+ }), encoding="utf-8")
+ return bdd.build_devflow_runs(project_root)[0]["overall"]
+
+ def test_run_waiting_between_stages_is_not_completed(self):
+ stages = {
+ "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0},
+ "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0},
+ "TASK-03": {"status": "pending", "executor": "developer", "retry_count": 0},
+ }
+ self.assertEqual(self._overall(stages), "running")
+
+ def test_fully_finished_run_is_completed(self):
+ names = ("TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05")
+ stages = {name: {"status": "completed", "executor": "x", "retry_count": 0} for name in names}
+ stages["TASK-05"]["status"] = "skipped"
+ self.assertEqual(self._overall(stages), "completed")
+
+
class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase):
"""用户反馈复现:workflow-state.json 模板固定给每次运行都写一份 SOLO stage
(见 assets/workflow-state-template.json 里 SOLO.description 的说明:
diff --git a/.codebuddy/skills/agent-observability/tests/test_cls_sink.py b/.codebuddy/skills/agent-observability/tests/test_cls_sink.py
index 2da9d6a1..7fdee02a 100644
--- a/.codebuddy/skills/agent-observability/tests/test_cls_sink.py
+++ b/.codebuddy/skills/agent-observability/tests/test_cls_sink.py
@@ -17,6 +17,22 @@ from core import cls_sink, emitter # type: ignore
class CLSSinkTests(unittest.TestCase):
+ def setUp(self):
+ # Keep the suite hermetic: never write debug logs into the source tree and
+ # do not depend on the optional Node SDK being checked out next to the repo.
+ sandbox = tempfile.TemporaryDirectory()
+ self.addCleanup(sandbox.cleanup)
+ sdk_entry = Path(sandbox.name) / "tencentcloud-cls-sdk-js" / "dist" / "index.js"
+ sdk_entry.parent.mkdir(parents=True)
+ sdk_entry.write_text("module.exports = {};\n", encoding="utf-8")
+ for name, value in (
+ ("sdk_entry_path", sdk_entry),
+ ("debug_log_path", Path(sandbox.name) / "cls-push-debug.ndjson"),
+ ):
+ patcher = mock.patch.object(cls_sink, name, return_value=value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
def _write_cls_env(self, root: Path, *extra_lines: str) -> None:
lines = [
"CLS_TOPIC_ID=test-topic-id",
@@ -171,6 +187,18 @@ class CLSSinkTests(unittest.TestCase):
self.assertEqual(entries[-1]["stage"], "uploader_ok")
self.assertEqual(entries[-1]["method"], "python_api_v3")
+ def test_unconfigured_cls_stays_silent(self):
+ with tempfile.TemporaryDirectory() as td:
+ debug_path = Path(td) / "debug.ndjson"
+ with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \
+ mock.patch.object(cls_sink.subprocess, "run") as run, \
+ mock.patch.dict(os.environ, {}, clear=True):
+ ok = cls_sink.mirror_record({"event": "start", "sid": "s-off"}, cwd=td)
+
+ self.assertFalse(ok)
+ run.assert_not_called()
+ self.assertFalse(debug_path.exists())
+
def test_emit_keeps_local_log_when_cls_push_raises(self):
with tempfile.TemporaryDirectory() as td:
log_path = Path(td) / "metrics.ndjson"
diff --git a/.cursor/skills/agent-observability/scripts/build_dashboard_data.py b/.cursor/skills/agent-observability/scripts/build_dashboard_data.py
index e33f816a..4c639f1a 100644
--- a/.cursor/skills/agent-observability/scripts/build_dashboard_data.py
+++ b/.cursor/skills/agent-observability/scripts/build_dashboard_data.py
@@ -522,6 +522,12 @@ def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None
"exec": info.get("executor") or "-", "status": status,
"retry_count": retry_count, "duration": duration,
})
+ # 只有所有阶段都已完成或跳过时才算“已完成”;停在两个阶段之间(等待审批、
+ # 下一阶段尚未派发)的运行仍是进行中,不能因为没有 in_progress 就显示完成。
+ if overall == "completed" and any(
+ stage["status"] not in {"completed", "skipped"} for stage in stages_out
+ ):
+ overall = "running"
# 按真实起止时间跨度算总时长(而不是逐阶段 duration 相加)——阶段之间可能
# 有重叠(比如 TASK-05 的 started_at 早于 SOLO 的 completed_at),相加会
# 把重叠部分重复计入,虚高于源数据本身反映的运行时长。
diff --git a/.cursor/skills/agent-observability/scripts/core/agent_identity.py b/.cursor/skills/agent-observability/scripts/core/agent_identity.py
index 270ff904..1255f2e0 100644
--- a/.cursor/skills/agent-observability/scripts/core/agent_identity.py
+++ b/.cursor/skills/agent-observability/scripts/core/agent_identity.py
@@ -162,7 +162,8 @@ def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Option
next_target = _extract_next_target(payload)
if from_role == "main" and mailbox_role and mailbox_role != "main":
- if not msg.get("is_shutdown"):
+ # A standby assignment only creates the member; it is not active yet.
+ if not msg.get("is_shutdown") and not msg.get("is_standby"):
current = mailbox_role
dispatched = mailbox_role
evidence = f"dispatch:{mailbox_name}"
@@ -521,6 +522,7 @@ def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, i
"ts": ts,
"index": index,
"is_shutdown": _is_shutdown_message(payload, summary, text),
+ "is_standby": _is_standby_message(payload, text),
}
@@ -561,6 +563,14 @@ def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any
return "shutdown" in summary_low or "shutdown request" in text_low
+def _is_standby_message(payload: dict[str, Any] | None, text: Any) -> bool:
+ """识别“先创建成员、待命等待唤醒”的消息;这类消息不代表成员已开始工作。"""
+ if isinstance(payload, dict) and (payload.get("event") or payload.get("dispatch_prompt")):
+ return False
+ text_low = text.lower() if isinstance(text, str) else ""
+ return "待命" in text_low or "standby" in text_low or "stand by" in text_low
+
+
def _parse_timestamp(value: Any) -> float:
"""把消息时间字段解析成 Unix 时间戳。"""
if not isinstance(value, str) or not value.strip():
diff --git a/.cursor/skills/agent-observability/scripts/core/cls_sink.py b/.cursor/skills/agent-observability/scripts/core/cls_sink.py
index 6525f6ed..e2c72391 100644
--- a/.cursor/skills/agent-observability/scripts/core/cls_sink.py
+++ b/.cursor/skills/agent-observability/scripts/core/cls_sink.py
@@ -338,6 +338,10 @@ def load_config(cwd: str | None = None) -> CLSConfig:
def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool:
config = load_config(cwd)
+ if not (config.endpoint or config.topic_id):
+ # CLS mirroring was never configured: stay silent instead of writing a
+ # debug line into the skill directory for every local event.
+ return False
if not config.ready:
_debug_write(
{
diff --git a/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py b/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py
index 53b031c1..6ee93f75 100644
--- a/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py
+++ b/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py
@@ -395,6 +395,36 @@ class BuildDevflowRunsClassicSoloTests(unittest.TestCase):
self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段
+class BuildDevflowRunsOverallStatusTests(unittest.TestCase):
+ """E2E 发现:刚开始或停在两个阶段之间的运行(没有 in_progress、也没有 failed)
+ 曾被显示为“已完成”。"""
+
+ def _overall(self, stages):
+ with tempfile.TemporaryDirectory() as td:
+ project_root = Path(td)
+ run_dir = project_root / "artifacts" / "between-stages"
+ run_dir.mkdir(parents=True)
+ (run_dir / "workflow-state.json").write_text(json.dumps({
+ "version": "1.3", "task_slug": "between-stages", "size_class": "medium",
+ "current_stage": "TASK-02", "stages": stages,
+ }), encoding="utf-8")
+ return bdd.build_devflow_runs(project_root)[0]["overall"]
+
+ def test_run_waiting_between_stages_is_not_completed(self):
+ stages = {
+ "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0},
+ "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0},
+ "TASK-03": {"status": "pending", "executor": "developer", "retry_count": 0},
+ }
+ self.assertEqual(self._overall(stages), "running")
+
+ def test_fully_finished_run_is_completed(self):
+ names = ("TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05")
+ stages = {name: {"status": "completed", "executor": "x", "retry_count": 0} for name in names}
+ stages["TASK-05"]["status"] = "skipped"
+ self.assertEqual(self._overall(stages), "completed")
+
+
class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase):
"""用户反馈复现:workflow-state.json 模板固定给每次运行都写一份 SOLO stage
(见 assets/workflow-state-template.json 里 SOLO.description 的说明:
diff --git a/.cursor/skills/agent-observability/tests/test_cls_sink.py b/.cursor/skills/agent-observability/tests/test_cls_sink.py
index 2da9d6a1..7fdee02a 100644
--- a/.cursor/skills/agent-observability/tests/test_cls_sink.py
+++ b/.cursor/skills/agent-observability/tests/test_cls_sink.py
@@ -17,6 +17,22 @@ from core import cls_sink, emitter # type: ignore
class CLSSinkTests(unittest.TestCase):
+ def setUp(self):
+ # Keep the suite hermetic: never write debug logs into the source tree and
+ # do not depend on the optional Node SDK being checked out next to the repo.
+ sandbox = tempfile.TemporaryDirectory()
+ self.addCleanup(sandbox.cleanup)
+ sdk_entry = Path(sandbox.name) / "tencentcloud-cls-sdk-js" / "dist" / "index.js"
+ sdk_entry.parent.mkdir(parents=True)
+ sdk_entry.write_text("module.exports = {};\n", encoding="utf-8")
+ for name, value in (
+ ("sdk_entry_path", sdk_entry),
+ ("debug_log_path", Path(sandbox.name) / "cls-push-debug.ndjson"),
+ ):
+ patcher = mock.patch.object(cls_sink, name, return_value=value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
def _write_cls_env(self, root: Path, *extra_lines: str) -> None:
lines = [
"CLS_TOPIC_ID=test-topic-id",
@@ -171,6 +187,18 @@ class CLSSinkTests(unittest.TestCase):
self.assertEqual(entries[-1]["stage"], "uploader_ok")
self.assertEqual(entries[-1]["method"], "python_api_v3")
+ def test_unconfigured_cls_stays_silent(self):
+ with tempfile.TemporaryDirectory() as td:
+ debug_path = Path(td) / "debug.ndjson"
+ with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \
+ mock.patch.object(cls_sink.subprocess, "run") as run, \
+ mock.patch.dict(os.environ, {}, clear=True):
+ ok = cls_sink.mirror_record({"event": "start", "sid": "s-off"}, cwd=td)
+
+ self.assertFalse(ok)
+ run.assert_not_called()
+ self.assertFalse(debug_path.exists())
+
def test_emit_keeps_local_log_when_cls_push_raises(self):
with tempfile.TemporaryDirectory() as td:
log_path = Path(td) / "metrics.ndjson"
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index ab15b727..c677346a 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -14,13 +14,17 @@ concurrency:
jobs:
e2e:
- name: e2e (python ${{ matrix.python-version }})
- runs-on: ubuntu-latest
- timeout-minutes: 15
+ name: e2e (${{ matrix.os }}, python ${{ matrix.python-version }}, node ${{ matrix.node-version }})
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
strategy:
fail-fast: false
matrix:
- python-version: ["3.8", "3.12"]
+ include:
+ - {os: ubuntu-latest, python-version: "3.8", node-version: "22"}
+ - {os: ubuntu-latest, python-version: "3.12", node-version: "22"}
+ - {os: ubuntu-latest, python-version: "3.12", node-version: "20"}
+ - {os: macos-latest, python-version: "3.12", node-version: "22"}
steps:
- uses: actions/[email protected]
- uses: actions/[email protected]
@@ -28,21 +32,50 @@ jobs:
python-version: ${{ matrix.python-version }}
- uses: actions/[email protected]
with:
- node-version: "22"
+ node-version: ${{ matrix.node-version }}
- run: python3 -m pip install pyyaml==6.0.3
- name: Run E2E suite (packaged CLI, fake host Agent, no API keys)
run: bash scripts/e2e.sh
+ e2e-browser:
+ name: e2e-browser (dashboard in Chromium)
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/[email protected]
+ - uses: actions/[email protected]
+ with:
+ python-version: "3.12"
+ - uses: actions/[email protected]
+ with:
+ node-version: "22"
+ - name: Install Playwright
+ run: |
+ python3 -m pip install pyyaml==6.0.3 playwright==1.63.0
+ python3 -m playwright install --with-deps chromium
+ - name: Render the observability dashboard
+ env:
+ LOOPFORGE_E2E_BROWSER: "1"
+ LOOPFORGE_E2E_ARTIFACTS: ${{ runner.temp }}/dashboard-screenshots
+ run: bash scripts/e2e.sh -k DashboardBrowserE2E
+ - name: Upload dashboard screenshots
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: dashboard-screenshots
+ path: ${{ runner.temp }}/dashboard-screenshots
+ if-no-files-found: ignore
+
e2e-gate:
# Single stable check name to mark as required in branch protection.
name: e2e-required
if: always()
- needs: [e2e]
+ needs: [e2e, e2e-browser]
runs-on: ubuntu-latest
steps:
- name: Require every E2E job to pass
run: |
- if [[ "${{ needs.e2e.result }}" != "success" ]]; then
- echo "E2E result: ${{ needs.e2e.result }}" >&2
+ if [[ "${{ needs.e2e.result }}" != "success" || "${{ needs.e2e-browser.result }}" != "success" ]]; then
+ echo "e2e=${{ needs.e2e.result }} e2e-browser=${{ needs.e2e-browser.result }}" >&2
exit 1
fi
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 64ddd17b..b0331a1e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -77,6 +77,7 @@ jobs:
run: |
python3 scripts/build-classic-hosts.py --check
bash scripts/validate.sh
+ bash scripts/test-observability.sh
bash scripts/smoke-install.sh
bash scripts/smoke-npm.sh
bash scripts/smoke-curl-install.sh
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 8fc60ec2..f78eeafe 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -25,6 +25,7 @@ jobs:
node-version: "22"
- run: python3 -m pip install pyyaml==6.0.3
- run: bash scripts/validate.sh
+ - run: bash scripts/test-observability.sh
- run: bash scripts/smoke-install.sh
- run: bash scripts/smoke-npm.sh
diff --git a/AGENTS.md b/AGENTS.md
index e05dedad..8b416ab9 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -19,6 +19,7 @@ python3 scripts/build-classic-hosts.py --check
bash scripts/validate.sh
bash scripts/smoke-install.sh
bash scripts/e2e.sh
+bash scripts/test-observability.sh
bash scripts/scan-secrets.sh
```
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f7abeaf8..132ecdc6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -11,7 +11,7 @@ Thanks for contributing to LoopForge.
3. Do not commit credentials, local permission configurations, or production data.
4. First indicate whether a change belongs to Classic, Portable, or shared behaviour contracts. Portable only modifies `skills/`; Classic `.cursor/.claude` files are not edited directly — run `build-classic-hosts.py --write`.
5. When modifying shared workflow contracts, describe the impact on both editions separately; do not copy runtime files between them.
-6. Run `bash scripts/validate.sh`, `bash scripts/smoke-install.sh` and `bash scripts/e2e.sh` before submitting. The E2E suite drives the installed workflow with a fake host Agent and recorded stage outputs, so it needs no API key; see [tests/e2e/README.md](tests/e2e/README.md).
+6. Run `bash scripts/validate.sh`, `bash scripts/smoke-install.sh` and `bash scripts/e2e.sh` before submitting, plus `bash scripts/test-observability.sh` when you change agent-observability. The E2E suite drives the installed workflow with a fake host Agent and recorded stage outputs, so it needs no API key; see [tests/e2e/README.md](tests/e2e/README.md).
## Pull request requirements
diff --git a/CONTRIBUTING.zh-CN.md b/CONTRIBUTING.zh-CN.md
index cf316cc5..d77d859f 100644
--- a/CONTRIBUTING.zh-CN.md
+++ b/CONTRIBUTING.zh-CN.md
@@ -11,7 +11,7 @@
3. 不提交任何凭据、本地权限配置或生产数据。
4. 先标明改动属于 Classic、Portable 或共享行为合同。Portable 只修改 `skills/`;Classic 的 `.cursor/.claude` 不直接修改,运行 `build-classic-hosts.py --write`。
5. 修改共享工作流契约时分别说明两个 edition 的影响;不要在二者之间复制运行文件。
-6. 提交前运行 `bash scripts/validate.sh`、`bash scripts/smoke-install.sh` 和 `bash scripts/e2e.sh`。E2E 用假宿主 Agent 和录制的阶段产物驱动已安装的工作流,不需要 API Key,说明见 [tests/e2e/README.md](tests/e2e/README.md)。
+6. 提交前运行 `bash scripts/validate.sh`、`bash scripts/smoke-install.sh`、`bash scripts/e2e.sh`;修改 agent-observability 时再运行 `bash scripts/test-observability.sh`。E2E 用假宿主 Agent 和录制的阶段产物驱动已安装的工作流,不需要 API Key,说明见 [tests/e2e/README.md](tests/e2e/README.md)。
## Pull Request 要求
diff --git a/package.json b/package.json
index 75fce41f..fa655f93 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,13 @@
"!skills/**/tests/**",
"!**/__pycache__/**",
"!**/*.pyc",
- "!**/*.pyo"
+ "!**/*.pyo",
+ "!.codebuddy/skills/agent-observability/logs/**",
+ ".codebuddy/skills/agent-observability/logs/.gitkeep",
+ "!.cursor/skills/agent-observability/logs/**",
+ ".cursor/skills/agent-observability/logs/.gitkeep",
+ "!.claude/skills/agent-observability/logs/**",
+ ".claude/skills/agent-observability/logs/.gitkeep"
],
"engines": {
"node": ">=20"
diff --git a/scripts/build-classic-hosts.py b/scripts/build-classic-hosts.py
index a8cfcbb5..e76744b7 100644
--- a/scripts/build-classic-hosts.py
+++ b/scripts/build-classic-hosts.py
@@ -17,6 +17,17 @@ ROLES = (
HOSTS = ("cursor", "claude")
+def is_runtime_artifact(relative: Path) -> bool:
+ """Same rule as devflow_cli.core.is_runtime_artifact (kept stdlib-only here)."""
+ parts = relative.parts
+ if "__pycache__" in parts or relative.suffix in {".pyc", ".pyo"}:
+ return True
+ for index in range(len(parts) - 2):
+ if parts[index:index + 2] == ("agent-observability", "logs"):
+ return parts[-1] != ".gitkeep"
+ return False
+
+
def strip_frontmatter(text: str) -> str:
if text.startswith("---\n"):
end = text.find("\n---\n", 4)
@@ -133,7 +144,7 @@ def render_agent(role: str, host: str) -> str:
def add_tree(files: Dict[str, bytes], source: Path, target_prefix: str, host: str) -> None:
for path in sorted(source.rglob("*")):
- if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc":
+ if not path.is_file() or is_runtime_artifact(path.relative_to(source)):
continue
relative = path.relative_to(source)
target = str(Path(target_prefix) / relative)
@@ -214,7 +225,7 @@ def check_host(host: str) -> list:
actual = {
str(path.relative_to(target)): path.read_bytes()
for path in target.rglob("*")
- if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc"
+ if path.is_file() and not is_runtime_artifact(path.relative_to(target))
} if target.is_dir() else {}
errors = []
for name in sorted(set(wanted) | set(actual)):
diff --git a/scripts/build-release-archive.sh b/scripts/build-release-archive.sh
index e9f839a2..0f8c3ff0 100755
--- a/scripts/build-release-archive.sh
+++ b/scripts/build-release-archive.sh
@@ -18,6 +18,8 @@ cp "$ROOT_DIR/package.json" "$ROOT_DIR/LICENSE" \
find "$PAYLOAD_DIR/skills" -type d -name tests -prune -exec rm -rf {} +
find "$PAYLOAD_DIR" -type d -name __pycache__ -prune -exec rm -rf {} +
find "$PAYLOAD_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
+# agent-observability run output (metrics, state, debug logs) is local-only.
+find "$PAYLOAD_DIR" -path '*/agent-observability/logs/*' -type f ! -name .gitkeep -delete
tar -czf "$OUTPUT_DIR/$ASSET_NAME" -C "$TMP_DIR" loopforge-workflow
if command -v sha256sum >/dev/null 2>&1; then
diff --git a/scripts/e2e.sh b/scripts/e2e.sh
index 01220d49..05d8fcb5 100755
--- a/scripts/e2e.sh
+++ b/scripts/e2e.sh
@@ -8,5 +8,5 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
export PYTHONDONTWRITEBYTECODE=1
-python3 -m unittest discover -s "$ROOT_DIR/tests/e2e" -t "$ROOT_DIR/tests/e2e" -p 'test_e2e_*.py' -v "$@"
+python3 -m unittest discover -s "$ROOT_DIR/tests/e2e" -t "$ROOT_DIR/tests/e2e" -p 'test_e2e_*.py' -v ${@+"$@"}
echo "E2E tests passed."
diff --git a/scripts/smoke-npm.sh b/scripts/smoke-npm.sh
index 1ed68b58..026e7d4f 100755
--- a/scripts/smoke-npm.sh
+++ b/scripts/smoke-npm.sh
@@ -34,7 +34,8 @@ for (const item of files) {
item.includes("__pycache__") ||
item.endsWith(".pyc") ||
item.startsWith("build/") ||
- /^skills\/.+\/tests\//.test(item)
+ /^skills\/.+\/tests\//.test(item) ||
+ (/\/agent-observability\/logs\//.test(item) && !item.endsWith("/.gitkeep"))
) {
throw new Error(`npm package contains generated file ${item}`)
}
diff --git a/scripts/test-observability.sh b/scripts/test-observability.sh
new file mode 100755
index 00000000..8ad8d768
--- /dev/null
+++ b/scripts/test-observability.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Unit tests for the Classic agent-observability skill (dashboard data, hooks,
+# CLS sink) in every Classic host bundle, then a check that running them left no
+# run output behind: logs written into the source tree would end up in installs.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+export PYTHONDONTWRITEBYTECODE=1
+
+for host in codebuddy cursor claude; do
+ tests="$ROOT_DIR/.$host/skills/agent-observability/tests"
+ echo "== agent-observability unit tests (.$host)"
+ python3 -m unittest discover -s "$tests" -t "$tests"
+done
+
+leftovers="$(find "$ROOT_DIR"/.codebuddy "$ROOT_DIR"/.cursor "$ROOT_DIR"/.claude \
+ -path '*/agent-observability/logs/*' -type f ! -name .gitkeep -print)"
+if [[ -n "$leftovers" ]]; then
+ echo "Tests wrote run output into the source tree:" >&2
+ echo "$leftovers" >&2
+ exit 1
+fi
+echo "Observability tests passed and left the source tree clean."
diff --git a/setup.py b/setup.py
index 82bed8cf..7428fc89 100644
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,11 @@ def data_files():
or "tests" in relative.parts
or "__pycache__" in relative.parts
or path.suffix == ".pyc"
+ or (
+ "logs" in relative.parts
+ and "agent-observability" in relative.parts
+ and path.name != ".gitkeep"
+ )
):
continue
target = Path("share/devflow") / path.parent.relative_to(ROOT)
diff --git a/skills/devflow/scripts/workflow_state.py b/skills/devflow/scripts/workflow_state.py
index 8a5733dc..8f1cc80f 100755
--- a/skills/devflow/scripts/workflow_state.py
+++ b/skills/devflow/scripts/workflow_state.py
@@ -698,7 +698,9 @@ def command_finish(args: argparse.Namespace) -> int:
executor_topology=None, team_name=None, dispatch_tool=None,
)
data["stages"][stage_name].update(reset)
- data.update(status="in_progress", next_stage="REQUIREMENT", current_stage="SOLO")
+ # SOLO is no longer part of the medium route; park current_stage on the
+ # PHASE-0 sentinel so the escalated state stays valid for resume/prepare.
+ data.update(status="in_progress", next_stage="REQUIREMENT", current_stage="PHASE-0")
elif args.result == "completed":
artifact_root = Path(data["artifacts_dir"])
required = required_artifacts(
diff --git a/skills/devflow/tests/test_regressions.py b/skills/devflow/tests/test_regressions.py
index dee927a8..7794cc14 100644
--- a/skills/devflow/tests/test_regressions.py
+++ b/skills/devflow/tests/test_regressions.py
@@ -505,6 +505,35 @@ class LifecycleTests(unittest.TestCase):
self.assertEqual("main-1", state["stages"]["SUMMARY"]["executor_id"])
self.assertEqual("coordinator", state["stages"]["SUMMARY"]["executor_type"])
+ def test_solo_overflow_leaves_a_resumable_medium_state(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ state_path = Path(run_script(
+ "workflow_state.py", "init", "--project-root", root,
+ "--slug", "overflow", "--size", "small", "--mode", "auto",
+ "--execution-mode", "isolated", "--host-adapter", "claude",
+ "--coordinator-id", "main",
+ ).stdout.strip())
+ run_script("workflow_state.py", "prepare", "--state", state_path, "--stage", "SOLO")
+ run_script(
+ "workflow_state.py", "assign", "--state", state_path, "--stage", "SOLO",
+ "--role", STAGE_ROLES["SOLO"], "--executor-type", "devflow-stage-executor",
+ "--executor-id", "solo-1", "--dispatch-tool", "agent",
+ )
+ run_script("workflow_state.py", "start", "--state", state_path, "--stage", "SOLO")
+ run_script(
+ "workflow_state.py", "finish", "--state", state_path, "--stage", "SOLO",
+ "--result", "overflow",
+ )
+ run_script("workflow_state.py", "validate", "--state", state_path)
+ resume = json.loads(run_script(
+ "workflow_state.py", "resume", "--state", state_path
+ ).stdout)
+ self.assertEqual(("prepare", "REQUIREMENT"), (resume["action"], resume["stage"]))
+ state = json.loads(state_path.read_text(encoding="utf-8"))
+ self.assertEqual("medium", state["size_class"])
+ self.assertEqual("skipped", state["stages"]["SOLO"]["status"])
+
def test_large_keeps_knowledge_and_legacy_medium_route_remains_valid(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
diff --git a/src/devflow_cli/core.py b/src/devflow_cli/core.py
index d40097b7..13a3a984 100644
--- a/src/devflow_cli/core.py
+++ b/src/devflow_cli/core.py
@@ -21,6 +21,21 @@ class DevFlowError(RuntimeError):
pass
+def is_runtime_artifact(relative: Path) -> bool:
+ """Local run output (logs, caches) that must never be installed or packaged.
+
+ agent-observability writes metrics, state and debug logs under its ``logs/``
+ directory; only the ``.gitkeep`` placeholder belongs to the asset bundle.
+ """
+ parts = Path(relative).parts
+ if "__pycache__" in parts or Path(relative).suffix in {".pyc", ".pyo"}:
+ return True
+ for index in range(len(parts) - 2):
+ if parts[index:index + 2] == ("agent-observability", "logs"):
+ return parts[-1] != ".gitkeep"
+ return False
+
+
def digest(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
@@ -82,7 +97,7 @@ def save_state(project: Path, state: dict) -> None:
def snapshot(root: Path) -> Dict[str, Tuple[bytes, int]]:
result = {}
for path in sorted(root.rglob("*")):
- if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc":
+ if not path.is_file() or is_runtime_artifact(path.relative_to(root)):
continue
relative = str(path.relative_to(root))
if relative.endswith(".devflow-managed-skills.json"):
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
index a0656e29..a6938b09 100644
--- a/tests/e2e/README.md
+++ b/tests/e2e/README.md
@@ -6,8 +6,11 @@ What it does:
1. Builds the npm tarball once (`npm pack`), installs it into a temporary prefix, and uses that `loopforge` binary, which is the same path users take with `npx loopforge-cli`.
2. Installs every supported host and edition into throw-away git projects, then checks the entrypoint files, `status`, `doctor`, `update` and a clean `uninstall` (`test_e2e_install.py`).
-3. Drives the installed Portable workflow end to end with [`fake_agent.py`](fake_agent.py), a deterministic stand-in for the host Agent. It follows the Skill's `resume`-driven protocol (`init`, `prepare --emit-prompt`, `assign`, `start`, `finish`, `approve`, `decide-knowledge`), but writes the stage outputs recorded in [`fixtures/scenario-medium.json`](fixtures/scenario-medium.json) instead of calling an LLM. The developer stage really changes [`fixtures/sample_project`](fixtures/sample_project/slugify.py), and the test stage really runs its tests. The suite covers every host, resume after a crash and after an interruption, the requirement confirmation gate, review rejection, unfilled artifacts and repeated failures (`test_e2e_portable_workflow.py`).
-4. Runs the Classic CodeBuddy hooks from an installed project: auto-dispatch routing between roles, the fallbacks to the main Agent, and a recorded session flowing through the observability hooks into the local dashboard, which is served over HTTP (`test_e2e_classic_runtime.py`).
+3. Drives the installed Portable workflow end to end with [`fake_agent.py`](fake_agent.py), a deterministic stand-in for the host Agent. It follows the Skill's `resume`-driven protocol (`init`, `prepare --emit-prompt`, `assign`, `start`, `finish`, `approve`, `decide-knowledge`), but writes the stage outputs recorded in [`fixtures/scenario-slugify.json`](fixtures/scenario-slugify.json) instead of calling an LLM. The developer stage really changes [`fixtures/sample_project`](fixtures/sample_project/slugify.py), and the test stage really runs its tests. The suite covers every host, resume after a crash and after an interruption, the requirement confirmation gate, review rejection, unfilled artifacts and repeated failures (`test_e2e_portable_workflow.py`).
+4. Covers the other Portable routes and modes (`test_e2e_workflow_modes.py`): small/SOLO, SOLO overflow escalating to medium, large with a knowledge stage, `manual` approval gates, and design-implementation tasks that need design context and visual evidence.
+5. Runs the Classic CodeBuddy hooks from an installed project: auto-dispatch routing between roles, the fallbacks to the main Agent, and a recorded session flowing through the observability hooks into the local dashboard, which is served over HTTP (`test_e2e_classic_runtime.py`).
+
+6. Renders the dashboard in Chromium with Playwright (`test_e2e_dashboard_browser.py`). It checks the data badge, KPIs, session table, Devflow run card and auto-dispatch health, plus the empty and load-failure states, and blocks every request outside 127.0.0.1. The test is skipped when `playwright` is not installed. CI sets `LOOPFORGE_E2E_BROWSER=1`, so there a missing browser fails the job instead, and the screenshots are uploaded as the `dashboard-screenshots` artifact.
Useful options:
@@ -15,6 +18,8 @@ Useful options:
LOOPFORGE_E2E_SOURCE=1 bash scripts/e2e.sh # run the CLI from src/ and skip npm pack
bash scripts/e2e.sh -k resume # run only tests matching a pattern
python3 tests/e2e/fake_agent.py --help # drive a project by hand
+pip install playwright && python3 -m playwright install chromium
+LOOPFORGE_E2E_BROWSER=1 bash scripts/e2e.sh -k DashboardBrowserE2E
```
When a test fails, the assertion message includes the fake Agent transcript (`.fake-agent/transcript.jsonl` in the project) that shows each action and the command that was rejected.
diff --git a/tests/e2e/fake_agent.py b/tests/e2e/fake_agent.py
index 29f9d905..13946000 100644
--- a/tests/e2e/fake_agent.py
+++ b/tests/e2e/fake_agent.py
@@ -23,7 +23,9 @@ import re
import shutil
import subprocess
import sys
+import struct
import uuid
+import zlib
from pathlib import Path
HOST_SKILL_DIRS = {
@@ -56,6 +58,7 @@ class FakeAgent:
self.review_rejections_left = args.review_reject
self.failures_left = dict(args.fail_stage)
self.fresh_team = bool(args.resume and args.host == "codebuddy")
+ self.overflowed = False
self.rounds = {}
# -- helpers ---------------------------------------------------------
@@ -107,8 +110,9 @@ class FakeAgent:
def init(self):
output = self.run(
"init", "--project-root", self.root, "--slug", self.args.slug,
- "--size", "medium", "--mode", "auto", "--execution-mode", "isolated",
+ "--size", self.args.size, "--mode", self.args.mode, "--execution-mode", "isolated",
"--host-adapter", self.host, "--coordinator-id", "fake-main",
+ *(["--require-design-artifacts"] if self.args.require_design_artifacts else []),
).stdout.strip()
self.log("init", state=output)
@@ -130,20 +134,25 @@ class FakeAgent:
if kind == "blocked":
return EXIT_BLOCKED
if kind == "approve":
- if action.get("requires_user_confirmation") and not self.args.user_confirms:
+ held = action["stage"] in self.args.hold_gate
+ if held or (action.get("requires_user_confirmation") and not self.args.user_confirms):
self.log("waiting-for-user", stage=action["stage"])
return EXIT_WAITING
extra = ["--user-confirmed"] if action.get("requires_user_confirmation") else []
self.run("approve", "--state", self.state_path(), "--stage", action["stage"], *extra)
+ self.log("approved", stage=action["stage"], user_confirmed=bool(extra))
elif kind == "rotate-team":
rotated = json.loads(self.run("rotate-team", "--state", self.state_path()).stdout)
self.log("team-rotated", old=rotated["old_team_name"], new=rotated["team_name"])
elif kind == "assess-knowledge":
- decision = self.scenario["knowledge_decision"]
+ decision = dict(self.scenario["knowledge_decision"])
+ if self.args.knowledge == "run":
+ decision = dict(self.scenario["knowledge_run"])
self.run(
"decide-knowledge", "--state", self.state_path(),
"--decision", decision["decision"], "--reason", decision["reason"],
)
+ self.log("knowledge-decided", decision=decision["decision"])
elif kind == "prepare":
prompt = self.run(
"prepare", "--state", self.state_path(), "--stage", action["stage"],
@@ -196,9 +205,18 @@ class FakeAgent:
if self.failures_left.get(stage, 0) > 0:
self.failures_left[stage] -= 1
result = "failed"
+ if stage == "SOLO" and self.args.solo_overflow and not self.overflowed:
+ # The single agent discovers the task is bigger than "small" and escalates.
+ self.overflowed = True
+ self.run("finish", "--state", self.state_path(), "--stage", "SOLO",
+ "--result", "overflow", "--host-adapter", self.host)
+ self.log("finish", stage=stage, result="overflow", rc=0, error=None)
+ return
for relative, source in self.scenario.get("code_changes", {}).get(stage, {}).items():
shutil.copyfile(str(FIXTURES / source), str(self.root / relative))
- if stage == "TEST":
+ if stage == "DESIGN" and self.state().get("requires_design_artifacts"):
+ self.write_design_context(evidence)
+ if stage in ("TEST", "SOLO"):
verify = subprocess.run(
[sys.executable] + self.scenario["verify_command"], cwd=str(self.root),
text=True, capture_output=True,
@@ -235,6 +253,16 @@ class FakeAgent:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
+ def write_design_context(self, evidence):
+ """Design-implementation tasks also need sections and visual evidence."""
+ root = self.state_path().parent / "02-design" / "design-context"
+ template = (self.skill_root / "templates" / "design-context-section.md").read_text(encoding="utf-8")
+ section = re.sub(r"\{\{[^{}]+\}\}", evidence, template)
+ (root / "sections").mkdir(parents=True, exist_ok=True)
+ (root / "sections" / "result-list.md").write_text(section, encoding="utf-8")
+ (root / "evidence").mkdir(parents=True, exist_ok=True)
+ (root / "evidence" / "mockup.png").write_bytes(tiny_png())
+
def required_artifacts(self, stage):
sys.path.insert(0, str(self.skill_root / "scripts"))
try:
@@ -257,14 +285,31 @@ class FakeAgent:
return code
+def tiny_png():
+ """A valid 1x1 PNG, generated so the fixture set has no binary files."""
+ def chunk(kind, data):
+ body = kind + data
+ return struct.pack(">I", len(data)) + body + struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF)
+ header = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)
+ return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header)
+ + chunk(b"IDAT", zlib.compress(b"\x00\xff\xff\xff")) + chunk(b"IEND", b""))
+
+
def parse_args(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--host", choices=sorted(HOST_SKILL_DIRS), required=True)
parser.add_argument("--project-root", type=Path, required=True)
parser.add_argument("--slug", default="e2e-task")
- parser.add_argument("--scenario", type=Path, default=FIXTURES / "scenario-medium.json")
+ parser.add_argument("--scenario", type=Path, default=FIXTURES / "scenario-slugify.json")
parser.add_argument("--resume", action="store_true", help="new session; do not init")
+ parser.add_argument("--size", choices=("small", "medium", "large"), default="medium")
+ parser.add_argument("--mode", choices=("auto", "manual"), default="auto")
+ parser.add_argument("--require-design-artifacts", action="store_true")
+ parser.add_argument("--solo-overflow", action="store_true", help="SOLO escalates to medium")
+ parser.add_argument("--knowledge", choices=("skip", "run"), default="skip")
parser.add_argument("--no-user-confirmation", dest="user_confirms", action="store_false")
+ parser.add_argument("--hold-gate", action="append", default=[],
+ help="the user does not answer this approval gate (session ends)")
parser.add_argument("--stop-after", type=int, default=0, help="interrupt after N actions")
parser.add_argument("--crash-in", help="die right after starting this stage")
parser.add_argument("--review-reject", type=int, default=0, help="reject REVIEW N times")
diff --git a/tests/e2e/fixtures/scenario-medium.json b/tests/e2e/fixtures/scenario-slugify.json
similarity index 70%
rename from tests/e2e/fixtures/scenario-medium.json
rename to tests/e2e/fixtures/scenario-slugify.json
index 2364e33c..81f17fb6 100644
--- a/tests/e2e/fixtures/scenario-medium.json
+++ b/tests/e2e/fixtures/scenario-slugify.json
@@ -1,7 +1,10 @@
{
- "description": "Recorded stage outputs for a medium task: fix slugify() in the sample project. Each stage executor fills the real templates produced by `workflow_state.py prepare` with this evidence.",
+ "description": "Recorded stage outputs for a task that fixes slugify() in the sample project. Each stage executor fills the real templates produced by `workflow_state.py prepare` with this evidence.",
"request": "修复 slugify:空格转连字符、去掉符号,并补充验收测试",
- "knowledge_decision": {"decision": "skip", "reason": "只有常规变更和测试结果,没有新增可复用约束"},
+ "knowledge_decision": {
+ "decision": "skip",
+ "reason": "只有常规变更和测试结果,没有新增可复用约束"
+ },
"overrides": {
"{{说明当前实现、关键模块、仓库相对路径、约束和非目标}}": "当前实现位于 `slugify.py`,仅修改该文件;验收用例在 `check_slugify.py`。",
"{{动作导向的任务名}}": "修复 slugify 规范化逻辑",
@@ -22,11 +25,25 @@
"REVIEW": "审查 `slugify.py` 的 diff,未发现阻断问题。",
"REVIEW_REJECTED": "不通过:`slugify.py` 未处理符号,阻断级别,建议改为按字母数字切分。",
"TEST": "运行 `python3 -m unittest check_slugify`,2 个用例全部通过。",
- "KNOWLEDGE": "无新增经验。",
- "SUMMARY": "交付完成;`python3 -m unittest check_slugify` 已通过,无剩余风险。"
+ "KNOWLEDGE": "可复用约束:slug 只保留 `[a-z0-9]+` 片段,见 `slugify.py`。",
+ "SUMMARY": "交付完成;`python3 -m unittest check_slugify` 已通过,无剩余风险。",
+ "SOLO": "已修改 `slugify.py` 并运行 `python3 -m unittest check_slugify`,结果 passed;任务仍符合 small 标准。"
},
"code_changes": {
- "IMPLEMENT": {"slugify.py": "patches/slugify.py"}
+ "IMPLEMENT": {
+ "slugify.py": "patches/slugify.py"
+ },
+ "SOLO": {
+ "slugify.py": "patches/slugify.py"
+ }
},
- "verify_command": ["-m", "unittest", "check_slugify"]
+ "verify_command": [
+ "-m",
+ "unittest",
+ "check_slugify"
+ ],
+ "knowledge_run": {
+ "decision": "run",
+ "reason": "审查发现可复用约束:`slugify.py` 只保留 `[a-z0-9]+` 片段,调用方不得依赖原始大小写"
+ }
}
diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py
index 592482cc..09213b5d 100644
--- a/tests/e2e/harness.py
+++ b/tests/e2e/harness.py
@@ -141,3 +141,87 @@ def loopforge_validate(project, slug="e2e-task", host_skills=None):
state = project / "artifacts" / slug / "workflow-state.json"
return run([sys.executable, script, "validate", "--state", state]).stdout
raise AssertionError("no installed workflow_state.py found")
+
+
+def classic_hook_env(project):
+ """Environment the CodeBuddy host gives its hooks; CLS mirroring stays off."""
+ env = dict(os.environ, CODEBUDDY_PROJECT_DIR=str(project), PYTHONDONTWRITEBYTECODE="1")
+ for key in list(env):
+ if key.startswith(("CLS_", "TENCENTCLOUD_", "TC_")):
+ del env[key]
+ return env
+
+
+def run_hook(project, command, payload):
+ result = subprocess.run(
+ [str(item) for item in command], input=json.dumps(payload), cwd=str(project),
+ env=classic_hook_env(project), capture_output=True, text=True, timeout=60,
+ )
+ if result.returncode != 0:
+ raise AssertionError("hook {} failed: {}".format(command, result.stderr))
+ return result.stdout
+
+
+def send_dispatch(project, sender, event, target, state_path):
+ payload = {"event": event, "dispatch_prompt": "continue as " + target,
+ "next_target": {"subagent_name": target},
+ "workflow_state_path": str(state_path)}
+ output = run_hook(
+ project, [sys.executable, Path(project) / ".codebuddy/hooks/devflow-auto-dispatch.py"],
+ {"tool_name": "SendMessage", "tool_input": {
+ "type": "message", "from": sender, "recipient": "main",
+ "content": json.dumps(payload)}},
+ )
+ return json.loads(output)
+
+
+def seed_classic_run(project, slug="e2e-classic"):
+ """A Classic workflow-state.json as the leader writes it at Phase 0."""
+ state_path = Path(project) / "artifacts" / slug / "workflow-state.json"
+ state_path.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(str(Path(project) / ".codebuddy/assets/workflow-state-template.json"), str(state_path))
+ return state_path
+
+
+def record_classic_session(project, state_path, session_id="e2e-session"):
+ """Replay one recorded CodeBuddy session through the observability hooks."""
+ project = Path(project)
+ skill = project / ".codebuddy/skills/agent-observability"
+ transcript = project / "session.jsonl"
+ shutil.copy(str(REPO / ".codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl"),
+ str(transcript))
+ base = {"session_id": session_id, "cwd": str(project), "transcript_path": str(transcript)}
+ tool = dict(base, tool_name="Bash", tool_input={"command": "ls"}, tool_use_id="call-002")
+ hook = [skill / "scripts/run_hook.sh"]
+ run_hook(project, hook + ["session-start"], base)
+ run_hook(project, hook + ["user-prompt-submit"], dict(base, prompt="/start-devflow fix slugify"))
+ run_hook(project, hook + ["pre"], tool)
+ run_hook(project, hook + ["post"], dict(tool, tool_response={"stdout": "ok"}))
+ send_dispatch(project, "requirement-owner", "TASK-01_completed", "architect", state_path)
+ run_hook(project, hook + ["stop"], base)
+
+
+def build_dashboard(project):
+ skill = Path(project) / ".codebuddy/skills/agent-observability"
+ output = run([sys.executable, skill / "scripts/build_dashboard_data.py", "--project-root", project],
+ cwd=project, env=classic_hook_env(project)).stdout
+ return output, skill / "scripts/dashboard"
+
+
+class StaticServer(object):
+ """Serve a directory on a random 127.0.0.1 port for the duration of a test."""
+
+ def __init__(self, test_case, directory):
+ import functools
+ import threading
+ from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+
+ class Quiet(SimpleHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(Quiet, directory=str(directory)))
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+ test_case.addCleanup(self.server.server_close)
+ test_case.addCleanup(self.server.shutdown)
+ self.url = "http://127.0.0.1:{}/".format(self.server.server_address[1])
diff --git a/tests/e2e/test_e2e_classic_runtime.py b/tests/e2e/test_e2e_classic_runtime.py
index cd32faff..8399c49a 100644
--- a/tests/e2e/test_e2e_classic_runtime.py
+++ b/tests/e2e/test_e2e_classic_runtime.py
@@ -6,15 +6,9 @@ the auto-dispatch router between role agents, the observability hooks that
record a session, and the dashboard built from what they wrote.
"""
-import functools
import json
-import os
-import shutil
-import subprocess
-import threading
import unittest
import urllib.request
-from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
import harness as h
@@ -32,34 +26,10 @@ class ClassicCodeBuddyE2E(unittest.TestCase):
def setUp(self):
self.project = h.new_project(self)
h.loopforge("install", "codebuddy", "--project-root", self.project)
- self.env = dict(os.environ, CODEBUDDY_PROJECT_DIR=str(self.project), PYTHONDONTWRITEBYTECODE="1")
- for key in list(self.env):
- if key.startswith(("CLS_", "TENCENTCLOUD_", "TC_")):
- del self.env[key] # the dashboard must work fully offline
- self.state_path = self.project / "artifacts/e2e-classic/workflow-state.json"
- self.state_path.parent.mkdir(parents=True)
- shutil.copy(str(self.project / ".codebuddy/assets/workflow-state-template.json"), str(self.state_path))
+ self.state_path = h.seed_classic_run(self.project)
- def hook(self, command, payload):
- result = subprocess.run(
- command, input=json.dumps(payload), cwd=str(self.project), env=self.env,
- capture_output=True, text=True, timeout=60,
- )
- self.assertEqual(0, result.returncode, result.stderr)
- return result.stdout
-
- def dispatch(self, sender, event, target, **extra):
- payload = {"event": event, "dispatch_prompt": "continue as " + target,
- "next_target": {"subagent_name": target},
- "workflow_state_path": str(self.state_path)}
- payload.update(extra)
- output = self.hook(
- ["python3", str(self.project / ".codebuddy/hooks/devflow-auto-dispatch.py")],
- {"tool_name": "SendMessage", "tool_input": {
- "type": "message", "from": sender, "recipient": "main",
- "content": json.dumps(payload)}},
- )
- return json.loads(output)
+ def dispatch(self, sender, event, target):
+ return h.send_dispatch(self.project, sender, event, target, self.state_path)
def test_auto_dispatch_routes_each_completed_stage_to_the_next_role(self):
for sender, event, target in ROUTE:
@@ -85,36 +55,14 @@ class ClassicCodeBuddyE2E(unittest.TestCase):
self.assertNotIn("modifiedInput", json.dumps(gated))
def test_recorded_session_shows_up_in_the_local_dashboard(self):
- skill = self.project / ".codebuddy/skills/agent-observability"
- transcript = self.project / "session.jsonl"
- shutil.copy(str(h.REPO / ".codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl"),
- str(transcript))
- base = {"session_id": "e2e-session", "cwd": str(self.project), "transcript_path": str(transcript)}
- tool = dict(base, tool_name="Bash", tool_input={"command": "ls"}, tool_use_id="call-002")
- run_hook = [str(skill / "scripts/run_hook.sh")]
- self.hook(run_hook + ["session-start"], base)
- self.hook(run_hook + ["user-prompt-submit"], dict(base, prompt="/start-devflow fix slugify"))
- self.hook(run_hook + ["pre"], tool)
- self.hook(run_hook + ["post"], dict(tool, tool_response={"stdout": "ok"}))
- self.dispatch("requirement-owner", "TASK-01_completed", "architect")
- self.hook(run_hook + ["stop"], base)
-
- built = subprocess.run(
- ["python3", str(skill / "scripts/build_dashboard_data.py"), "--project-root", str(self.project)],
- cwd=str(self.project), env=self.env, capture_output=True, text=True, check=True,
- ).stdout
+ h.record_classic_session(self.project, self.state_path)
+ built, dashboard = h.build_dashboard(self.project)
self.assertIn("1 sessions, 1 devflow runs", built)
- dashboard = skill / "scripts/dashboard"
- handler = functools.partial(_QuietHandler, directory=str(dashboard))
- server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
- threading.Thread(target=server.serve_forever, daemon=True).start()
- self.addCleanup(server.server_close)
- self.addCleanup(server.shutdown)
- url = "http://127.0.0.1:{}/".format(server.server_address[1])
- page = urllib.request.urlopen(url, timeout=10).read().decode("utf-8")
+ server = h.StaticServer(self, dashboard)
+ page = urllib.request.urlopen(server.url, timeout=10).read().decode("utf-8")
self.assertIn("./dashboard-data.json", page)
- data = json.loads(urllib.request.urlopen(url + "dashboard-data.json", timeout=10).read())
+ data = json.loads(urllib.request.urlopen(server.url + "dashboard-data.json", timeout=10).read())
sessions = data["sessions"]
self.assertEqual(["e2e-session"], [item["id"] for item in sessions])
self.assertGreaterEqual(sessions[0]["toolCalls"], 1)
@@ -123,10 +71,5 @@ class ClassicCodeBuddyE2E(unittest.TestCase):
self.assertEqual(["e2e-classic"], [run["slug"] for run in data["devflowRuns"]])
self.assertEqual(1, data["autoDispatchStats"]["auto_dispatch"])
-class _QuietHandler(SimpleHTTPRequestHandler):
- def log_message(self, *args):
- pass
-
-
if __name__ == "__main__":
unittest.main()
diff --git a/tests/e2e/test_e2e_dashboard_browser.py b/tests/e2e/test_e2e_dashboard_browser.py
new file mode 100644
index 00000000..a265c17f
--- /dev/null
+++ b/tests/e2e/test_e2e_dashboard_browser.py
@@ -0,0 +1,119 @@
+"""E2E: render the local observability dashboard in a real browser (Playwright).
+
+Records a CodeBuddy session through the installed hooks, builds the dashboard
+snapshot, serves it on 127.0.0.1 and checks what a user actually sees: the
+data badge, KPI strip, session table, Devflow run card and auto-dispatch
+health, plus the empty and load-failure states. All requests that leave
+127.0.0.1 (for example web fonts) are blocked, so the page must work offline.
+
+Skipped when the ``playwright`` Python package is not installed, unless
+``LOOPFORGE_E2E_BROWSER=1`` (set in CI), in which case a missing browser fails.
+Screenshots go to ``$LOOPFORGE_E2E_ARTIFACTS`` when set.
+"""
+
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+import harness as h
+
+try:
+ from playwright.sync_api import sync_playwright
+except ImportError: # pragma: no cover - depends on the environment
+ sync_playwright = None
+
+REQUIRED = os.environ.get("LOOPFORGE_E2E_BROWSER") == "1"
+
+
+def setUpModule():
+ if sync_playwright is None:
+ if REQUIRED:
+ raise RuntimeError("LOOPFORGE_E2E_BROWSER=1 but the playwright package is not installed")
+ raise unittest.SkipTest("playwright is not installed (pip install playwright && playwright install chromium)")
+
+
+class DashboardBrowserE2E(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.playwright = sync_playwright().start()
+ cls.browser = cls.playwright.chromium.launch()
+ cls.shots = Path(os.environ.get("LOOPFORGE_E2E_ARTIFACTS") or tempfile.mkdtemp(prefix="loopforge-e2e-shots-"))
+ cls.shots.mkdir(parents=True, exist_ok=True)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.browser.close()
+ cls.playwright.stop()
+
+ def setUp(self):
+ self.project = h.new_project(self, with_sample=False)
+ h.loopforge("install", "codebuddy", "--project-root", self.project)
+ self.state_path = h.seed_classic_run(self.project)
+
+ def open(self, directory):
+ server = h.StaticServer(self, directory)
+ page = self.browser.new_page(viewport={"width": 1440, "height": 1000})
+ self.addCleanup(page.close)
+ errors = []
+ page.on("pageerror", lambda error: errors.append(str(error)))
+ page.on("console", lambda msg: errors.append(msg.text) if msg.type == "error" and "ERR_FAILED" not in msg.text else None)
+
+ def offline(route):
+ if route.request.url.startswith(server.url):
+ route.continue_()
+ else:
+ route.abort()
+
+ page.route("**/*", offline)
+ page.goto(server.url, wait_until="load")
+ return page, errors
+
+ def test_recorded_session_renders_in_every_panel(self):
+ h.record_classic_session(self.project, self.state_path)
+ _, dashboard = h.build_dashboard(self.project)
+ page, errors = self.open(dashboard)
+
+ badge = page.locator("#dataStatusBadge")
+ badge.filter(has_text="真实数据").wait_for(timeout=10000)
+ self.assertIn("条事件", badge.inner_text())
+ self.assertTrue(page.locator("#kpiStrip").inner_text().strip())
+ self.assertEqual(1, page.locator("#dailyTable tbody tr").count())
+ page.screenshot(path=str(self.shots / "dashboard-overview.png"), full_page=False)
+
+ page.click('button[data-tab="sessions"]')
+ self.assertTrue(page.locator("#panel-sessions").is_visible())
+ self.assertIn("e2e-session", page.locator("#sessionTable").inner_text())
+
+ page.click('button[data-tab="devflow"]')
+ self.assertTrue(page.locator("#panel-devflow").is_visible())
+ run_card = page.locator("#devflowRuns").inner_text()
+ self.assertIn("e2e-classic", run_card)
+ # The seeded run has only finished Phase 0, so it must not claim to be done.
+ self.assertIn("进行中", run_card)
+ self.assertNotIn("已完成", run_card)
+ self.assertEqual("1", page.locator("#hDispatch").inner_text().strip())
+ page.screenshot(path=str(self.shots / "dashboard-devflow.png"), full_page=False)
+ self.assertEqual([], errors)
+
+ def test_empty_snapshot_explains_how_to_get_data(self):
+ _, dashboard = h.build_dashboard(self.project)
+ data = json.loads((dashboard / "dashboard-data.json").read_text(encoding="utf-8"))
+ self.assertEqual([], data["sessions"])
+ page, errors = self.open(dashboard)
+ page.locator("#dataStatusBadge").filter(has_text="还没有数据").wait_for(timeout=10000)
+ page.click('button[data-tab="sessions"]')
+ self.assertIn("暂无会话记录", page.locator("#sessionTable").inner_text())
+ self.assertEqual([], errors)
+
+ def test_missing_snapshot_shows_a_load_error(self):
+ dashboard = self.project / ".codebuddy/skills/agent-observability/scripts/dashboard"
+ self.assertFalse((dashboard / "dashboard-data.json").exists())
+ page, _ = self.open(dashboard)
+ page.locator("#dataStatusBadge").filter(has_text="加载 dashboard-data.json 失败").wait_for(timeout=10000)
+ self.assertIn("build_dashboard_data.py", page.locator("footer.note").inner_text())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/e2e/test_e2e_install.py b/tests/e2e/test_e2e_install.py
index 8d6cf09f..d060b6d9 100644
--- a/tests/e2e/test_e2e_install.py
+++ b/tests/e2e/test_e2e_install.py
@@ -1,6 +1,7 @@
"""E2E: install the packaged CLI into a user project for every supported host."""
import json
+import re
import unittest
from pathlib import Path
@@ -48,12 +49,33 @@ class InstallMatrixE2E(unittest.TestCase):
self.assertEqual((edition, 0, 0), (rows[0]["edition"], rows[0]["missing"], rows[0]["changed"]))
self.assertIn("healthy", h.loopforge("doctor", "--project-root", project).stdout)
+ self.assert_no_runtime_logs(project)
+ self.assert_markdown_links_resolve(project)
+
update = h.loopforge(*install_args(edition, host, "update"), "--project-root", project).stdout
self.assertIn("written=0", update)
h.loopforge(*install_args(edition, host, "uninstall"), "--project-root", project)
self.assertEqual(before, h.files_under(project))
+ def assert_no_runtime_logs(self, project):
+ """Only the logs/.gitkeep placeholder may ship; run output stays local."""
+ for logs in project.glob(".*/skills/agent-observability/logs"):
+ self.assertEqual([".gitkeep"], sorted(p.name for p in logs.iterdir()), str(logs))
+
+ def assert_markdown_links_resolve(self, project):
+ """Every relative Markdown link the host Agent may follow must be installed."""
+ broken = []
+ for path in project.rglob("*.md"):
+ if ".git" in path.relative_to(project).parts:
+ continue
+ for target in re.findall(r"\[[^\]]*\]\(([^)\s]+)\)", path.read_text(encoding="utf-8")):
+ target = target.split("#", 1)[0]
+ if target and "://" not in target and not target.startswith("mailto:"):
+ if not (path.parent / target).exists():
+ broken.append("{} -> {}".format(path.relative_to(project), target))
+ self.assertEqual([], broken)
+
def test_classic_is_rejected_for_portable_only_host(self):
project = h.new_project(self, with_sample=False)
before = h.files_under(project)
diff --git a/tests/e2e/test_e2e_workflow_modes.py b/tests/e2e/test_e2e_workflow_modes.py
new file mode 100644
index 00000000..e97abc65
--- /dev/null
+++ b/tests/e2e/test_e2e_workflow_modes.py
@@ -0,0 +1,115 @@
+"""E2E: the other Portable routes and modes, driven by the fake host Agent.
+
+Complements test_e2e_portable_workflow.py (medium/auto) with small/SOLO, SOLO
+overflow escalation, large with a knowledge stage, manual approval gates and
+design-implementation tasks.
+"""
+
+import subprocess
+import sys
+import unittest
+
+import harness as h
+
+MEDIUM = ["REQUIREMENT", "DESIGN", "IMPLEMENT", "REVIEW", "TEST", "SUMMARY"]
+
+
+def events(project, kind):
+ return [item for item in h.transcript(project) if item["kind"] == kind]
+
+
+def completed_stages(project):
+ return [e["stage"] for e in events(project, "finish") if e["result"] == "completed"]
+
+
+class WorkflowModesE2E(unittest.TestCase):
+ def install(self, host):
+ project = h.new_project(self)
+ h.loopforge("skills", "install", host, "--project-root", project)
+ return project
+
+ def assert_code_fixed(self, project):
+ verify = subprocess.run(
+ [sys.executable, "-m", "unittest", "check_slugify"], cwd=str(project),
+ capture_output=True, text=True,
+ )
+ self.assertEqual(0, verify.returncode, verify.stderr)
+ self.assertIn("OK", h.loopforge_validate(project))
+
+ def test_small_task_uses_single_solo_agent(self):
+ for host in ("claude", "codebuddy"):
+ with self.subTest(host=host):
+ project = self.install(host)
+ h.fake_agent(host, project, "--size", "small", check_rc=0)
+ state = h.workflow_state(project)
+ self.assertEqual(["SOLO", "SUMMARY"], state["route"])
+ self.assertEqual("completed", state["status"])
+ self.assertEqual(["SOLO", "SUMMARY"], completed_stages(project))
+ # No requirement gate and no multi-agent pipeline on the small route.
+ self.assertFalse(events(project, "approved"))
+ self.assertEqual(["SOLO"], [e["stage"] for e in events(project, "spawned")])
+ self.assertTrue((project / "artifacts/e2e-task/01-solo/solo-report.md").is_file())
+ self.assert_code_fixed(project)
+
+ def test_solo_overflow_escalates_to_the_medium_route(self):
+ project = self.install("claude")
+ h.fake_agent("claude", project, "--size", "small", "--solo-overflow", check_rc=0)
+ state = h.workflow_state(project)
+ self.assertEqual("medium", state["size_class"])
+ self.assertEqual("skipped", state["stages"]["SOLO"]["status"])
+ self.assertEqual("completed", state["status"])
+ self.assertEqual(MEDIUM, completed_stages(project))
+ self.assertEqual(["REQUIREMENT"], [e["stage"] for e in events(project, "approved")])
+ self.assert_code_fixed(project)
+
+ def test_large_task_runs_the_knowledge_stage_when_it_has_value(self):
+ project = self.install("codex")
+ h.fake_agent("codex", project, "--size", "large", "--knowledge", "run", check_rc=0)
+ state = h.workflow_state(project)
+ self.assertEqual("large", state["size_class"])
+ self.assertEqual("run", state["stages"]["KNOWLEDGE"]["decision"])
+ self.assertEqual("completed", state["stages"]["KNOWLEDGE"]["status"])
+ self.assertEqual(MEDIUM[:-1] + ["KNOWLEDGE", "SUMMARY"], completed_stages(project))
+ self.assertIn("KNOWLEDGE", [e["stage"] for e in events(project, "spawned")])
+ self.assertTrue((project / "artifacts/e2e-task/05-knowledge").is_dir())
+ self.assert_code_fixed(project)
+
+ def test_manual_mode_stops_at_every_manual_gate(self):
+ project = self.install("cursor")
+ h.fake_agent("cursor", project, "--mode", "manual", check_rc=0)
+ approvals = [(e["stage"], e["user_confirmed"]) for e in events(project, "approved")]
+ self.assertEqual(
+ [("REQUIREMENT", True), ("DESIGN", False), ("REVIEW", False), ("TEST", False)],
+ approvals,
+ )
+ self.assertEqual("completed", h.workflow_state(project)["status"])
+ self.assert_code_fixed(project)
+
+ def test_manual_gate_holds_the_next_stage_until_approved(self):
+ project = self.install("cursor")
+ # The user does not answer the DESIGN gate: the run must wait there.
+ h.fake_agent("cursor", project, "--mode", "manual", "--hold-gate", "DESIGN", check_rc=3)
+ state = h.workflow_state(project)
+ self.assertEqual(("awaiting_approval", "DESIGN"), (state["status"], state["awaiting_approval"]))
+ self.assertIsNone(state["stages"]["IMPLEMENT"]["prepared_at"])
+ h.fake_agent("cursor", project, "--resume", "--mode", "manual", check_rc=0)
+ self.assertEqual("completed", h.workflow_state(project)["status"])
+
+ def test_design_implementation_task_requires_design_context(self):
+ project = self.install("claude")
+ h.fake_agent("claude", project, "--require-design-artifacts", check_rc=0)
+ context = project / "artifacts/e2e-task/02-design/design-context"
+ for relative in ("overview.md", "components.md", "sections/result-list.md", "evidence/mockup.png"):
+ self.assertTrue((context / relative).is_file(), relative)
+ self.assertEqual("completed", h.workflow_state(project)["status"])
+
+ def test_design_task_without_visual_evidence_is_rejected(self):
+ project = self.install("claude")
+ result = h.fake_agent("claude", project, "--require-design-artifacts",
+ "--leave-placeholders", "DESIGN", check_rc=2)
+ self.assertIn("finish DESIGN rejected", result.stderr)
+ self.assertEqual("in_progress", h.workflow_state(project)["stages"]["DESIGN"]["status"])
+
+
+if __name__ == "__main__":
+ unittest.main()