Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(codex): preserve credential-prefixed native plugin launches (#584)

Christopher Kevinfccab03003cd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fb035442..906d638f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -347,6 +347,10 @@ All notable changes to SkillEvaluator are documented in this file.
failed the member and bumped its `severity_counts`, but a catalog validates
each skill on its own, so nothing in that skill's output said what caused
either.
+- Native Codex plugin loading now recognizes Harbor launches prefixed with
+ empty credential assignments, preserving credential isolation while running
+ plugin setup before the agent. These launches previously failed after the
+ task with a missing native-launch error.
- Folder validation preserves author and metadata ID advisories from passing
skills. Catalog summaries include duplicate ID severity counts and consistent
member verdicts, while explicitly labeling linked reports as single-skill
diff --git a/src/skillevaluator/tier3/harbor/native_agents.py b/src/skillevaluator/tier3/harbor/native_agents.py
index 5ffcd388..43f83c77 100644
--- a/src/skillevaluator/tier3/harbor/native_agents.py
+++ b/src/skillevaluator/tier3/harbor/native_agents.py
@@ -235,8 +235,11 @@ class _NativeClaudeCodeMixin(_NativePluginLoadMixin):
class _NativeCodexMixin(_NativePluginLoadMixin):
- # Harbor: ``...; codex exec --dangerously-bypass-approvals-and-sandbox ... -- <prompt>``.
- _SKILLEVAL_LAUNCH_RE = re.compile(_COMMAND_START + r"codex[ \t]+exec\b")
+ # Harbor clears unselected fallback credentials with empty shell assignments
+ # before launching Codex. Keep them on the launch side of the setup group.
+ _SKILLEVAL_LAUNCH_RE = re.compile(
+ _COMMAND_START + r"(?:CODEX_API_KEY=[ \t]+)?(?:OPENAI_API_KEY=[ \t]+)?codex[ \t]+exec\b"
+ )
class _NativeOpenCodeMixin(_NativePluginLoadMixin):
diff --git a/tests/tier3/test_native_launch_insertion.py b/tests/tier3/test_native_launch_insertion.py
index 08b5d27d..7f666e9a 100644
--- a/tests/tier3/test_native_launch_insertion.py
+++ b/tests/tier3/test_native_launch_insertion.py
@@ -84,3 +84,15 @@ def test_second_launch_in_one_run_fails() -> None:
agent.skilleval_native_command(CODEX, None)
with pytest.raises(NativeLaunchError):
agent.skilleval_native_command(CODEX, None)
+
+
[email protected]("assignments", ["CODEX_API_KEY= ", "OPENAI_API_KEY= ", "CODEX_API_KEY= OPENAI_API_KEY= "])
+def test_codex_native_setup_preserves_harbor_credential_clearing(assignments: str) -> None:
+ command = CODEX.replace("codex exec", assignments + "codex exec")
+ agent = _mixin(native_agents._NativeCodexMixin)
+ rewritten, _ = agent.skilleval_native_command(command, None)
+ assert agent._skilleval_native_started
+ assert rewritten == command.replace(assignments + "codex exec", SETUP + assignments + "codex exec")
+ assert subprocess.run(["sh", "-n", "-c", rewritten], check=False).returncode == 0
+ with pytest.raises(NativeLaunchError):
+ agent.skilleval_native_command(command, None)