Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(codex): preserve credential-prefixed native plugin launches (#584)
Christopher Kevinfccab03003cd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fb035442..906d638f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -347,6 +347,10 @@ All notable changes to SkillEvaluator are documented in this file.failed the member and bumped its `severity_counts`, but a catalog validateseach skill on its own, so nothing in that skill's output said what causedeither.+- Native Codex plugin loading now recognizes Harbor launches prefixed with+ empty credential assignments, preserving credential isolation while running+ plugin setup before the agent. These launches previously failed after the+ task with a missing native-launch error.- Folder validation preserves author and metadata ID advisories from passingskills. Catalog summaries include duplicate ID severity counts and consistentmember verdicts, while explicitly labeling linked reports as single-skilldiff --git a/src/skillevaluator/tier3/harbor/native_agents.py b/src/skillevaluator/tier3/harbor/native_agents.pyindex 5ffcd388..43f83c77 100644--- a/src/skillevaluator/tier3/harbor/native_agents.py+++ b/src/skillevaluator/tier3/harbor/native_agents.py@@ -235,8 +235,11 @@ class _NativeClaudeCodeMixin(_NativePluginLoadMixin):class _NativeCodexMixin(_NativePluginLoadMixin):- # Harbor: ``...; codex exec --dangerously-bypass-approvals-and-sandbox ... -- <prompt>``.- _SKILLEVAL_LAUNCH_RE = re.compile(_COMMAND_START + r"codex[ \t]+exec\b")+ # Harbor clears unselected fallback credentials with empty shell assignments+ # before launching Codex. Keep them on the launch side of the setup group.+ _SKILLEVAL_LAUNCH_RE = re.compile(+ _COMMAND_START + r"(?:CODEX_API_KEY=[ \t]+)?(?:OPENAI_API_KEY=[ \t]+)?codex[ \t]+exec\b"+ )class _NativeOpenCodeMixin(_NativePluginLoadMixin):diff --git a/tests/tier3/test_native_launch_insertion.py b/tests/tier3/test_native_launch_insertion.pyindex 08b5d27d..7f666e9a 100644--- a/tests/tier3/test_native_launch_insertion.py+++ b/tests/tier3/test_native_launch_insertion.py@@ -84,3 +84,15 @@ def test_second_launch_in_one_run_fails() -> None:agent.skilleval_native_command(CODEX, None)with pytest.raises(NativeLaunchError):agent.skilleval_native_command(CODEX, None)++[email protected]("assignments", ["CODEX_API_KEY= ", "OPENAI_API_KEY= ", "CODEX_API_KEY= OPENAI_API_KEY= "])+def test_codex_native_setup_preserves_harbor_credential_clearing(assignments: str) -> None:+ command = CODEX.replace("codex exec", assignments + "codex exec")+ agent = _mixin(native_agents._NativeCodexMixin)+ rewritten, _ = agent.skilleval_native_command(command, None)+ assert agent._skilleval_native_started+ assert rewritten == command.replace(assignments + "codex exec", SETUP + assignments + "codex exec")+ assert subprocess.run(["sh", "-n", "-c", rewritten], check=False).returncode == 0+ with pytest.raises(NativeLaunchError):+ agent.skilleval_native_command(command, None)