Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Real Harbor agent-runtime smoke preflight regressions."""
from __future__ import annotations
import errno
import gzip
import hashlib
import json
import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from threading import Barrier, Event, Lock, Thread
from threading import enumerate as enumerate_threads
from unittest.mock import Mock
import pytest
from botocore import credentials as botocore_credentials
from botocore import exceptions as botocore_exceptions
from botocore import tokens as botocore_tokens
from botocore.exceptions import (
ApiVersionNotFoundError,
BaseEndpointResolverError,
ConfigNotFound,
ConfigParseError,
DataNotFoundError,
EndpointConnectionError,
EndpointProviderError,
InvalidConfigError,
InvalidDefaultsMode,
InvalidIMDSEndpointError,
InvalidIMDSEndpointModeError,
InvalidProxiesConfigError,
InvalidRegionError,
InvalidRetryConfigurationError,
InvalidSTSRegionalEndpointsConfigError,
MissingDependencyException,
NoAuthTokenError,
NoCredentialsError,
ParamValidationError,
PartialCredentialsError,
ProfileNotFound,
RefreshWithMFAUnsupportedError,
ServiceNotInRegionError,
SSOTokenLoadError,
UnauthorizedSSOTokenError,
UnknownCredentialError,
UnknownRegionError,
UnknownSignatureVersionError,
UnsupportedSignatureVersionError,
)
from botocore.exceptions import (
SSLError as BotocoreSSLError,
)
from botocore.session import Session as BotocoreSession
from skillevaluator.model_catalog import ModelCatalogError, ModelRecord
from skillevaluator.provider_config import ProviderConfig
from skillevaluator.tier3.harbor import runtime_preflight
from skillevaluator.tier3.harbor.collector import validate_harbor_job_result
def _dataset(tmp_path: Path) -> Path:
dataset = tmp_path / "tasks"
(dataset / "case-002").mkdir(parents=True)
(dataset / "case-001").mkdir()
(dataset / "case-001" / "task.toml").write_text('[task]\nname = "nvidia/skillevaluator-case-001"\n')
(dataset / "case-002" / "task.toml").write_text('[task]\nname = "nvidia/skillevaluator-case-002"\n')
return dataset
def _configure_isolated_aws_profile(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
*,
config: str,
credentials: str = "",
profile: str = "test",
) -> None:
config_path = tmp_path / "aws-config"
credentials_path = tmp_path / "aws-credentials"
config_path.write_text(config, encoding="utf-8")
credentials_path.write_text(credentials, encoding="utf-8")
for variable in tuple(os.environ):
if variable.startswith("AWS_"):
monkeypatch.delenv(variable)
monkeypatch.setenv("AWS_CONFIG_FILE", str(config_path))
monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(credentials_path))
monkeypatch.setenv("AWS_PROFILE", profile)
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
def _write_harbor_0132_unscored_result(jobs_dir: Path) -> Path:
job_dir = jobs_dir / "runtime-preflight-opencode"
trial_dir = job_dir / "case-001__attempt"
trial_dir.mkdir(parents=True)
result_path = job_dir / "result.json"
result_path.write_text(
json.dumps(
{
"n_total_trials": 1,
"stats": {
"n_completed_trials": 1,
"n_errored_trials": 0,
"n_running_trials": 0,
"n_pending_trials": 0,
"n_cancelled_trials": 0,
"n_retries": 0,
"evals": {
"opencode__model___harbor-tasks": {
"n_trials": 0,
"n_errors": 0,
"reward_stats": {},
}
},
},
}
),
encoding="utf-8",
)
(trial_dir / "result.json").write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": {
"n_input_tokens": 100,
"n_cache_tokens": 0,
"n_output_tokens": 1,
"cost_usd": None,
},
"exception_info": None,
}
),
encoding="utf-8",
)
return result_path
def test_runtime_preflight_runs_one_case_once_without_verification(monkeypatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
def build(**kwargs):
captured.update(kwargs)
return ["harbor", "run", "--safe"]
def run(command, **kwargs):
captured["command"] = command
captured["run_kwargs"] = kwargs
captured["cwd_entries"] = sorted(path.name for path in Path(kwargs["cwd"]).iterdir())
return subprocess.CompletedProcess(command, 0, "", "")
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", build)
monkeypatch.setattr(runtime_preflight.subprocess, "run", run)
monkeypatch.setattr(
runtime_preflight,
"validate_harbor_agent_only_job_result",
lambda *_args, **_kwargs: (True, "ok"),
)
operator_dir = tmp_path / "operator"
operator_dir.mkdir()
(operator_dir / ".env.local").write_text("SE_DOTENV_SENTINEL=leaked\n", encoding="utf-8")
monkeypatch.chdir(operator_dir)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={"NVIDIA_API_KEY": "secret"},
timeout_multiplier=2.0,
timeout_seconds=321,
)
assert result.ok is True
assert captured["n_attempts"] == 1
assert captured["n_concurrent"] == 1
assert captured["disable_verification"] is True
assert captured["include_task_names"] == ["case-001"]
assert captured["timeout_multiplier"] == 2.0
run_kwargs = captured["run_kwargs"]
assert isinstance(run_kwargs, dict)
assert run_kwargs["timeout"] == 321
assert run_kwargs["env"] == {"NVIDIA_API_KEY": "secret", "HARBOR_TELEMETRY": "0", "PYTHON_DOTENV_DISABLED": "1"}
# Harbor starts in an empty evaluator-owned directory, never the operator's.
assert Path(run_kwargs["cwd"]) != operator_dir
assert captured["cwd_entries"] == []
assert not Path(run_kwargs["cwd"]).exists()
def test_runtime_preflight_forwards_environment_kwargs_to_build_harbor_run_command(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Forward environment_kwargs to build_harbor_run_command during preflight."""
captured: dict[str, object] = {}
def build(**kwargs):
captured.update(kwargs)
return ["harbor", "run"]
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", build)
monkeypatch.setattr(
runtime_preflight.subprocess,
"run",
lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""),
)
monkeypatch.setattr(
runtime_preflight,
"validate_harbor_agent_only_job_result",
lambda *_args, **_kwargs: (True, "ok"),
)
env_kwargs = {"cluster_name": "test-cluster", "region": "us-central1"}
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={},
environment_kwargs=env_kwargs,
)
assert result.ok is True
assert captured["environment_kwargs"] == env_kwargs
def test_runtime_preflight_hands_nvidia_build_key_only_over_stdin(monkeypatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
secret = "nvidia-real-secret-value-for-test"
monkeypatch.setattr(
runtime_preflight,
"build_harbor_run_command",
lambda **_kwargs: ["harbor", "run", "--safe"],
)
def run(command, **kwargs):
captured["command"] = command
captured["run_kwargs"] = kwargs
return subprocess.CompletedProcess(command, 0, "", "")
monkeypatch.setattr(runtime_preflight.subprocess, "run", run)
monkeypatch.setattr(
runtime_preflight,
"validate_harbor_agent_only_job_result",
lambda *_args, **_kwargs: (True, "ok"),
)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={"SKILL_EVAL_LLM_PROVIDER": "nv_build", "NVIDIA_API_KEY": secret},
)
assert result.ok is True
run_kwargs = captured["run_kwargs"]
assert isinstance(run_kwargs, dict)
assert run_kwargs["input"] == secret
environment = run_kwargs["env"]
assert isinstance(environment, dict)
assert secret not in environment.values()
assert environment["NVIDIA_API_KEY"] == "skillevaluator-stdin-backed-nvidia-key"
assert environment["SKILLEVALUATOR_NVIDIA_API_KEY_STDIN"] == "1"
assert "SKILLEVALUATOR_NVIDIA_API_KEY_FILE" not in environment
def test_runtime_preflight_accepts_harbor_0132_unscored_agent_success(monkeypatch, tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
result_path = _write_harbor_0132_unscored_result(jobs_dir)
# A real completed agent run always leaves its transcript on the mounted host path.
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "opencode.txt").write_text("agent transcript", encoding="utf-8")
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
monkeypatch.setattr(
runtime_preflight.subprocess,
"run",
lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""),
)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=jobs_dir,
run_env={},
)
assert result.ok is True
def test_agent_only_validation_accepts_actual_harbor_022_single_step_result(tmp_path: Path) -> None:
from datetime import UTC, datetime
from uuid import UUID
from harbor.models.job.result import JobResult, JobStats
from harbor.models.trial.result import TrialResult
job_dir = tmp_path / "jobs" / "runtime-preflight-opencode"
trial_dir = job_dir / "case-001__attempt"
trial_dir.mkdir(parents=True)
now = datetime(2026, 8, 25, tzinfo=UTC)
trial_result = TrialResult.model_validate(
{
"id": UUID(int=2),
"task_name": "nvidia/skillevaluator-case-001",
"trial_name": trial_dir.name,
"trial_uri": trial_dir.as_uri(),
"task_id": {"path": str(job_dir / "task" / "case-001")},
"task_checksum": "harbor-0.22-agent-only-fixture",
"config": {
"task": {"path": str(job_dir / "task" / "case-001")},
"trial_name": trial_dir.name,
"verifier": {"disable": True},
},
"agent_info": {
"name": "opencode",
"version": "test",
"model_info": {"name": "test-model"},
},
"agent_result": {
"n_input_tokens": 100,
"n_cache_tokens": 20,
"n_output_tokens": 10,
},
"verifier_result": None,
"exception_info": None,
"started_at": now,
"finished_at": now,
"step_results": None,
}
)
job_result = JobResult(
id=UUID(int=1),
started_at=now,
updated_at=now,
finished_at=now,
n_total_trials=1,
stats=JobStats.from_trial_results([trial_result], n_total_trials=1),
trial_results=[trial_result],
)
result_path = job_dir / "result.json"
result_path.write_text(job_result.model_dump_json(indent=2), encoding="utf-8")
(trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
persisted_job = JobResult.model_validate_json(result_path.read_text(encoding="utf-8"))
persisted_trial = TrialResult.model_validate_json((trial_dir / "result.json").read_text(encoding="utf-8"))
assert persisted_job.stats.n_completed_trials == 1
assert persisted_job.stats.n_errored_trials == 0
assert persisted_job.stats.n_input_tokens == 100
assert persisted_job.stats.n_cache_tokens == 20
assert persisted_job.stats.n_output_tokens == 10
assert persisted_trial.step_results is None
assert runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1) == (True, "")
def test_agent_only_validation_accepts_harbor_0132_unscored_multistep_agent_success(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": None,
"exception_info": None,
"step_results": [
{
"step_name": "author-skill",
"agent_result": {
"n_input_tokens": 100,
"n_cache_tokens": 0,
"n_output_tokens": 10,
"cost_usd": None,
},
"verifier_result": None,
"exception_info": None,
"agent_execution": {
"started_at": "2026-07-08T17:00:00Z",
"finished_at": "2026-07-08T17:00:10Z",
},
"verifier": None,
},
{
"step_name": "reuse-skill",
"agent_result": {
"n_input_tokens": 200,
"n_cache_tokens": 20,
"n_output_tokens": 15,
"cost_usd": None,
},
"verifier_result": None,
"exception_info": None,
"agent_execution": {
"started_at": "2026-07-08T17:00:11Z",
"finished_at": "2026-07-08T17:00:20Z",
},
"verifier": None,
},
],
}
),
encoding="utf-8",
)
assert runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1) == (True, "")
def test_agent_only_validation_rejects_mixed_single_and_multistep_agent_results(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": {"n_input_tokens": 100},
"exception_info": None,
"step_results": [
{
"step_name": "author-skill",
"agent_result": {"n_input_tokens": 100},
"exception_info": None,
}
],
}
),
encoding="utf-8",
)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert "mixed top-level and step agent results" in detail.lower()
@pytest.mark.parametrize(
("step_results", "expected"),
[
([], "has no step results"),
({}, "has invalid step_results"),
([None], "invalid step result 1"),
([{"exception_info": None, "agent_result": {}}], "invalid step_name"),
([{"step_name": " ", "exception_info": None, "agent_result": {}}], "invalid step_name"),
([{"step_name": "author-skill", "agent_result": {}}], "missing exception_info"),
(
[
{
"step_name": "author-skill",
"exception_info": {"exception_type": "AgentTimeoutError"},
"agent_result": {},
}
],
"recorded an exception",
),
([{"step_name": "author-skill", "exception_info": None}], "has no agent result"),
(
[
{"step_name": "author-skill", "exception_info": None, "agent_result": {}},
{"step_name": "reuse-skill", "exception_info": None, "agent_result": None},
],
"has no agent result",
),
],
)
def test_agent_only_validation_rejects_empty_malformed_or_failed_multistep_results(
tmp_path: Path,
step_results: object,
expected: str,
) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": None,
"exception_info": None,
"step_results": step_results,
}
),
encoding="utf-8",
)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert expected in detail.lower()
def test_scored_job_validation_still_rejects_harbor_0132_unscored_result(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
ok, detail = validate_harbor_job_result(result_path, expected_trials=1)
assert ok is False
assert "account for 0/1" in detail
@pytest.mark.parametrize(
("counter", "expected"),
[
("n_errored_trials", "1 errored"),
("n_running_trials", "1 running"),
("n_pending_trials", "1 pending"),
("n_cancelled_trials", "1 cancelled"),
],
)
def test_agent_only_validation_rejects_non_successful_harbor_0132_states(
tmp_path: Path,
counter: str,
expected: str,
) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
payload = json.loads(result_path.read_text(encoding="utf-8"))
payload["stats"][counter] = 1
result_path.write_text(json.dumps(payload), encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert expected in detail
def test_agent_only_validation_surfaces_first_errored_trial_exception(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
payload = json.loads(result_path.read_text(encoding="utf-8"))
payload["stats"]["n_errored_trials"] = 1
result_path.write_text(json.dumps(payload), encoding="utf-8")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"exception_info": {
"exception_type": "NonZeroAgentExitCodeError",
"exception_message": (
"Command failed (exit 128): git -C /workspace init -q\n"
"stderr: fatal: Invalid path '/Users/example': Operation not permitted"
),
},
"agent_result": {},
}
),
encoding="utf-8",
)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert "1 errored" in detail
assert "case-001__attempt" in detail
assert "NonZeroAgentExitCodeError" in detail
assert "git -C /workspace init -q" in detail
assert "Operation not permitted" in detail
def test_agent_only_validation_surfaces_multistep_trial_exception(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
payload = json.loads(result_path.read_text(encoding="utf-8"))
payload["stats"]["n_errored_trials"] = 1
result_path.write_text(json.dumps(payload), encoding="utf-8")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"exception_info": None,
"agent_result": None,
"step_results": [
{
"step_name": "reuse-skill",
"exception_info": {
"exception_type": "AgentTimeoutError",
"exception_message": "agent step timed out",
},
"agent_result": None,
}
],
}
),
encoding="utf-8",
)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert "AgentTimeoutError" in detail
assert "agent step timed out" in detail
def test_runtime_preflight_redacts_and_sanitizes_retained_trial_exception(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
jobs_dir = tmp_path / "jobs"
result_path = _write_harbor_0132_unscored_result(jobs_dir)
payload = json.loads(result_path.read_text(encoding="utf-8"))
payload["stats"]["n_errored_trials"] = 1
result_path.write_text(json.dumps(payload), encoding="utf-8")
secret = "nvapi-super-secret"
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
trial_result_path.write_text(
json.dumps(
{
"exception_info": {
"exception_type": "NonZeroAgentExitCodeError",
"exception_message": f"token={secret}\x1b[2J {'detail ' * 300}",
},
"agent_result": {},
}
),
encoding="utf-8",
)
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
monkeypatch.setattr(
runtime_preflight.subprocess,
"run",
lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""),
)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="local",
jobs_dir=jobs_dir,
run_env={"NVIDIA_API_KEY": secret},
)
assert result.ok is False
assert "NonZeroAgentExitCodeError" in result.detail
assert secret not in result.detail
assert "\x1b" not in result.detail
assert len(result.detail) <= 2000
def test_agent_only_validation_rejects_incomplete_harbor_0132_state(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
payload = json.loads(result_path.read_text(encoding="utf-8"))
payload["stats"]["n_completed_trials"] = 0
result_path.write_text(json.dumps(payload), encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert "completed 0/1" in detail
@pytest.mark.parametrize("result_text", [None, "{not-json"])
def test_agent_only_validation_rejects_missing_or_malformed_job_result(
tmp_path: Path,
result_text: str | None,
) -> None:
result_path = tmp_path / "jobs" / "runtime-preflight-opencode" / "result.json"
if result_text is not None:
result_path.parent.mkdir(parents=True)
result_path.write_text(result_text, encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert "result.json" in detail
@pytest.mark.parametrize(
("trial_payload", "expected"),
[
(None, "did not produce 1 trial result"),
("{not-json", "unreadable trial result"),
(json.dumps({"exception_info": {"exception_type": "AgentTimeoutError"}, "agent_result": {}}), "exception"),
(json.dumps({"exception_info": None, "agent_result": None}), "no agent result"),
(json.dumps({"exception_info": None}), "no agent result"),
],
)
def test_agent_only_validation_rejects_missing_malformed_or_failed_trial_result(
tmp_path: Path,
trial_payload: str | None,
expected: str,
) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
if trial_payload is None:
trial_result_path.unlink()
else:
trial_result_path.write_text(trial_payload, encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1)
assert ok is False
assert expected in detail.lower()
def test_runtime_preflight_reports_agent_start_failure(monkeypatch, tmp_path: Path) -> None:
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
monkeypatch.setattr(
runtime_preflight.subprocess,
"run",
lambda *_args, **_kwargs: subprocess.CompletedProcess([], 17, "", "401 Unauthorized"),
)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={},
)
assert result.ok is False
assert result.agent == "opencode"
assert "401 Unauthorized" in result.detail
def test_runtime_preflight_timeout_is_bounded(monkeypatch, tmp_path: Path) -> None:
monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
secret = "nvidia-real-secret-value-for-test"
def timeout(*_args, **kwargs):
assert kwargs["input"] == secret
assert secret not in kwargs["env"].values()
raise subprocess.TimeoutExpired(["harbor", "run"], timeout=30)
monkeypatch.setattr(runtime_preflight.subprocess, "run", timeout)
result = runtime_preflight.run_agent_runtime_preflight(
dataset=_dataset(tmp_path),
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={"SKILL_EVAL_LLM_PROVIDER": "nv_build", "NVIDIA_API_KEY": secret},
timeout_seconds=30,
)
assert result.ok is False
assert "timed out after 30s" in result.detail
assert secret not in result.detail
def test_runtime_preflight_rejects_empty_task_tree(tmp_path: Path) -> None:
dataset = tmp_path / "tasks"
dataset.mkdir()
result = runtime_preflight.run_agent_runtime_preflight(
dataset=dataset,
agent="opencode",
model="nvidia/model",
env_mode="docker",
jobs_dir=tmp_path / "jobs",
run_env={},
)
assert result.ok is False
assert "no staged tasks" in result.detail.lower()
def test_task_timeout_plan_uses_largest_staged_timeout(tmp_path: Path) -> None:
from skillevaluator.tier3.harbor import runner
root = tmp_path / "tasks"
for name, timeout in (("case-1", 120), ("case-2", 300)):
task = root / name
task.mkdir(parents=True)
(task / "task.toml").write_text(f"[agent]\ntimeout_sec = {timeout}.0\n")
assert runner._task_timeout_plan([root], 2.0) == 600.0
@pytest.mark.parametrize(
"task_toml",
[
"[agent]\ntimeout_sec = 900.0\n",
"[verifier]\ntimeout_sec = 900.0\n",
"[environment]\nbuild_timeout_sec = 900.0\n",
'[[steps]]\nname = "one"\n[steps.agent]\ntimeout_sec = 900.0\n',
'[[steps]]\nname = "one"\n[steps.verifier]\ntimeout_sec = 900.0\n',
],
ids=["agent", "verifier", "environment-build", "step-agent", "step-verifier"],
)
def test_task_timeout_plan_rejects_staged_timeout_product_overflow(
tmp_path: Path,
task_toml: str,
) -> None:
from skillevaluator.tier3.evals_config import MAX_HARBOR_TIMEOUT_MULTIPLIER
from skillevaluator.tier3.harbor import runner
task = tmp_path / "tasks" / "case-1"
task.mkdir(parents=True)
(task / "task.toml").write_text(task_toml, encoding="utf-8")
with pytest.raises(ValueError, match="non-finite Harbor timeout"):
runner._task_timeout_plan([task.parent], MAX_HARBOR_TIMEOUT_MULTIPLIER)
def test_model_probe_delegates_to_shared_catalog_client_without_exposing_key(monkeypatch) -> None:
captured: dict[str, object] = {}
def fetch(provider, *, timeout_seconds):
captured.update(provider=provider, timeout_seconds=timeout_seconds)
return (ModelRecord("meta/llama-3.1-8b-instruct"),)
monkeypatch.setattr(runtime_preflight, "fetch_model_records", fetch)
provider = ProviderConfig(
provider="nv_build",
model="meta/llama-3.1-8b-instruct",
api_key="nvapi-secret",
base_url="https://integrate.api.nvidia.com/v1",
litellm_model="openai/meta/llama-3.1-8b-instruct",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=4.5)
assert result.ok is True
assert captured == {"provider": provider, "timeout_seconds": 4.5}
assert "nvapi-secret" not in result.detail
def test_model_probe_preserves_raw_catalog_id_that_begins_with_nvidia(monkeypatch) -> None:
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("nvidia/llama-test"),),
)
provider = ProviderConfig(
provider="nv_build",
model="nvidia/llama-test",
api_key="nvapi-secret",
base_url="https://integrate.api.nvidia.com/v1",
litellm_model="openai/nvidia/llama-test",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is True
assert result.model == "nvidia/llama-test"
def test_model_probe_reports_unlisted_model(monkeypatch) -> None:
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("different-model"),),
)
provider = ProviderConfig(
provider="openai-compatible",
model="requested-model",
api_key="secret-key",
base_url="https://provider.example/v1",
litellm_model="openai/requested-model",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert "requested-model" in result.detail
assert "not listed" in result.detail
def test_model_probe_reports_safe_shared_catalog_error(monkeypatch) -> None:
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
ModelCatalogError(
"model catalog returned HTTP 401",
kind="authentication",
http_status=401,
)
),
)
provider = ProviderConfig(
provider="openai",
model="gpt-test",
api_key="secret-key",
base_url="https://api.openai.com/v1",
litellm_model="openai/gpt-test",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert "HTTP 401" in result.detail
assert "secret-key" not in result.detail
assert getattr(result, "failure_kind", None) == "authentication"
assert getattr(result, "http_status", None) == 401
def test_anthropic_model_probe_resolves_alias_omitted_from_listing(monkeypatch) -> None:
captured: dict[str, object] = {}
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("claude-canonical-20260824"),),
)
def resolve_alias(provider, model_id, *, timeout_seconds):
captured.update(provider=provider, model_id=model_id, timeout_seconds=timeout_seconds)
return ModelRecord("claude-canonical-20260824")
monkeypatch.setattr(runtime_preflight, "fetch_anthropic_model_record", resolve_alias)
provider = ProviderConfig(
provider="anthropic",
model="claude-stable-alias",
api_key="secret-key",
base_url=None,
litellm_model="anthropic/claude-stable-alias",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=4.5)
assert result.ok is True
assert result.model == "claude-stable-alias"
assert "claude-canonical-20260824" in result.detail
assert captured["provider"] == provider
assert captured["model_id"] == "claude-stable-alias"
assert 0 < float(captured["timeout_seconds"]) <= 4.5
def test_anthropic_alias_lookup_uses_remaining_catalog_probe_deadline(monkeypatch) -> None:
captured: dict[str, object] = {}
clock = iter((100.0, 100.06))
monkeypatch.setattr(runtime_preflight, "monotonic", lambda: next(clock))
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("claude-canonical-20260824"),),
)
def resolve_alias(provider, model_id, *, timeout_seconds):
captured.update(provider=provider, model_id=model_id, timeout_seconds=timeout_seconds)
return ModelRecord("claude-canonical-20260824")
monkeypatch.setattr(runtime_preflight, "fetch_anthropic_model_record", resolve_alias)
provider = ProviderConfig(
provider="anthropic",
model="claude-stable-alias",
api_key="secret-key",
base_url=None,
litellm_model="anthropic/claude-stable-alias",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=0.1)
assert result.ok is True
assert captured["timeout_seconds"] == pytest.approx(0.04)
def test_anthropic_alias_lookup_stops_when_catalog_probe_deadline_is_exhausted(monkeypatch) -> None:
clock = iter((100.0, 100.1))
monkeypatch.setattr(runtime_preflight, "monotonic", lambda: next(clock))
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("claude-canonical-20260824"),),
)
monkeypatch.setattr(
runtime_preflight,
"fetch_anthropic_model_record",
lambda *_args, **_kwargs: pytest.fail("alias lookup must not start after the shared deadline"),
)
provider = ProviderConfig(
provider="anthropic",
model="claude-stable-alias",
api_key="secret-key",
base_url=None,
litellm_model="anthropic/claude-stable-alias",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=0.1)
assert result.ok is False
assert result.failure_kind == "unavailable"
assert "timed out" in result.detail
@pytest.mark.parametrize("timeout_seconds", [0, True, float("nan")])
def test_http_catalog_probe_rejects_invalid_shared_deadline(monkeypatch, timeout_seconds) -> None:
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: pytest.fail("invalid timeout must fail before catalog I/O"),
)
provider = ProviderConfig(
provider="openai",
model="gpt-test",
api_key="secret-key",
base_url="https://api.openai.com/v1",
litellm_model="openai/gpt-test",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=timeout_seconds)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert "positive number" in result.detail
def test_anthropic_model_probe_treats_native_single_model_404_as_fatal(monkeypatch) -> None:
monkeypatch.setattr(
runtime_preflight,
"fetch_model_records",
lambda *_args, **_kwargs: (ModelRecord("different-model"),),
)
monkeypatch.setattr(
runtime_preflight,
"fetch_anthropic_model_record",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
ModelCatalogError(
"model catalog returned HTTP 404",
kind="unsupported",
http_status=404,
)
),
)
provider = ProviderConfig(
provider="anthropic",
model="missing-model",
api_key="secret-key",
base_url="https://api.anthropic.com:443",
litellm_model="anthropic/missing-model",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert result.failure_kind == "model_not_found"
assert result.http_status == 404
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
@pytest.mark.parametrize(
("provider", "probe", "expected"),
[
(
ProviderConfig("openai", "gpt-test", "key", "https://api.openai.com/v1", "openai/gpt-test"),
{"ok": True, "failure_kind": None, "http_status": None},
"verified",
),
(
ProviderConfig(
"nv_build",
"nvidia/model",
"invalid-key",
"https://integrate.api.nvidia.com/v1",
"openai/nvidia/model",
),
{"ok": True, "failure_kind": None, "http_status": 200},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"invalid-key",
"https://gateway.example/v1",
"openai/gpt-test",
),
{"ok": True, "failure_kind": None, "http_status": 200},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://api.openai.com/v1",
"openai/gpt-test",
),
{"ok": True, "failure_kind": None, "http_status": 200},
"verified",
),
(
ProviderConfig("openai", "gpt-test", "key", "https://api.openai.com/v1", "openai/gpt-test"),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"fatal",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://gateway.example/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"degraded",
),
(
ProviderConfig(
"openai",
"gpt-test",
"key",
"https://gateway.example/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"degraded",
),
(
ProviderConfig(
"anthropic",
"claude-test",
"key",
"https://gateway.example/v1",
"anthropic/claude-test",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://api.openai.com/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"fatal",
),
(
ProviderConfig(
"anthropic",
"claude-test",
"key",
"https://api.anthropic.com/v1",
"anthropic/claude-test",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"fatal",
),
(
ProviderConfig(
"nv_build",
"nvidia/model",
"key",
"https://integrate.api.nvidia.com/v1",
"openai/nvidia/model",
),
{"ok": False, "failure_kind": "authentication", "http_status": 401},
"fatal",
),
(
ProviderConfig(
"nv_build",
"nvidia/model",
"key",
"https://integrate.api.nvidia.com/v1",
"openai/nvidia/model",
),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"fatal",
),
(
ProviderConfig(
"nv_build",
"nvidia/model",
"key",
"https://integrate.api.nvidia.com/v1",
"openai/nvidia/model",
),
{"ok": False, "failure_kind": None, "http_status": None},
"fatal",
),
(
ProviderConfig("openai", "gpt-test", "key", "https://api.openai.com/v1", "openai/gpt-test"),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"degraded",
),
(
ProviderConfig("openai", "gpt-test", "key", "https://gateway.example/v1", "openai/gpt-test"),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://api.openai.com/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"degraded",
),
(
ProviderConfig("bedrock", "model", None, None, "bedrock/model", region="us-west-2"),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"degraded",
),
(
ProviderConfig("anthropic", "claude-test", "key", None, "anthropic/claude-test"),
{"ok": False, "failure_kind": None, "http_status": None},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://gateway.example/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": None, "http_status": None},
"degraded",
),
(
ProviderConfig("openai", "gpt-test", "key", "https://api.openai.com/v1", "openai/gpt-test"),
{"ok": False, "failure_kind": "unavailable", "http_status": 429},
"degraded",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"https://gateway.example/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "invalid_configuration", "http_status": None},
"fatal",
),
(
ProviderConfig(
"openai-compatible",
"gpt-test",
"key",
"http://your-server:8000/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "unsupported", "http_status": None},
"degraded",
),
(
ProviderConfig(
"openai",
"gpt-test",
"key",
"https://api.openai.com:443/v1",
"openai/gpt-test",
),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"degraded",
),
(
ProviderConfig(
"anthropic",
"claude-alias",
"key",
"https://api.anthropic.com",
"anthropic/claude-alias",
),
{"ok": False, "failure_kind": None, "http_status": None},
"degraded",
),
(
ProviderConfig(
"anthropic",
"claude-alias",
"key",
"https://api.anthropic.com:443/v1",
"anthropic/claude-alias",
),
{"ok": False, "failure_kind": "authorization", "http_status": 403},
"fatal",
),
(
ProviderConfig(
"openai",
"gpt-test",
"key",
"https://api.openai.com:443/v1/",
"openai/gpt-test",
),
{"ok": False, "failure_kind": None, "http_status": None},
"degraded",
),
(
ProviderConfig(
"bedrock",
"application-inference-profile/model-id",
None,
None,
"bedrock/application-inference-profile/model-id",
region="us-west-2",
),
{"ok": False, "failure_kind": None, "http_status": None},
"degraded",
),
],
)
def test_credential_probe_disposition_is_endpoint_aware(provider, probe: dict[str, object], expected: str) -> None:
result = type(
"Probe",
(),
{
**probe,
"provider": provider.provider,
"model": provider.model,
"detail": "safe detail",
},
)()
assert runtime_preflight.credential_probe_disposition(provider, result) == expected
def test_model_probe_rejects_non_http_catalog_url() -> None:
provider = ProviderConfig(
provider="openai-compatible",
model="model",
api_key="secret-key",
base_url="file:///etc",
litellm_model="openai/model",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert "HTTP or HTTPS" in result.detail
def test_model_probe_checks_bedrock_foundation_catalog(monkeypatch) -> None:
captured: list[tuple[str, dict[str, object]]] = []
class Bedrock:
def list_foundation_models(self):
return {"modelSummaries": [{"modelId": "anthropic.claude-sonnet-test-v1:0"}]}
class Session:
def client(self, service, **kwargs):
captured.append((service, kwargs))
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=4.5)
assert result.ok is True
assert [service for service, _kwargs in captured] == ["bedrock", "bedrock-runtime"]
for _service, kwargs in captured:
assert kwargs["region_name"] == "us-west-2"
request_config = kwargs["config"]
assert request_config.connect_timeout == pytest.approx(4.5, rel=1e-3)
assert request_config.read_timeout == pytest.approx(4.5, rel=1e-3)
assert request_config.retries == {"max_attempts": 0}
@pytest.mark.parametrize(
("catalog_endpoint", "runtime_endpoint", "expected_disposition"),
[
(
"https://bedrock.us-west-2.amazonaws.com",
"https://bedrock-runtime.us-west-2.amazonaws.com",
"verified",
),
(
"https://bedrock-fips.us-west-2.amazonaws.com",
"https://bedrock-runtime-fips.us-west-2.amazonaws.com",
"verified",
),
(
"https://bedrock.us-west-2.api.aws",
"https://bedrock-runtime.us-west-2.api.aws",
"verified",
),
(
"http://127.0.0.1:18001",
"https://bedrock-runtime.us-west-2.amazonaws.com",
"degraded",
),
(
"https://bedrock.us-west-2.amazonaws.com",
"http://127.0.0.1:18002",
"degraded",
),
("http://127.0.0.1:18001", "http://127.0.0.1:18002", "degraded"),
],
)
def test_bedrock_catalog_success_only_verifies_native_catalog_and_runtime_endpoints(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
catalog_endpoint: str,
runtime_endpoint: str,
expected_disposition: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
botocore_session = BotocoreSession()
class Client:
def __init__(self, endpoint_url: str) -> None:
self.meta = type("Meta", (), {"endpoint_url": endpoint_url})()
def list_foundation_models(self):
return {"modelSummaries": [{"modelId": "test-model"}]}
class Session:
_session = botocore_session
def client(self, service: str, **_kwargs):
return Client(catalog_endpoint if service == "bedrock" else runtime_endpoint)
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is True
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
def test_bedrock_custom_endpoint_metadata_cannot_make_custom_hosts_authoritative(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
data_path.mkdir()
(data_path / "endpoints.json").write_text(
json.dumps(
{
"version": 3,
"partitions": [
{
"partition": "aws",
"partitionName": "AWS Standard",
"dnsSuffix": "example.invalid",
"regionRegex": "^(us)-\\w+-\\d+$",
"regions": {"us-west-2": {}},
"defaults": {
"hostname": "{service}.{region}.{dnsSuffix}",
"protocols": ["https"],
"signatureVersions": ["v4"],
},
"services": {
"bedrock": {"endpoints": {"us-west-2": {"hostname": "catalog.attacker.invalid"}}},
"bedrock-runtime": {"endpoints": {"us-west-2": {"hostname": "runtime.attacker.invalid"}}},
},
}
],
}
),
encoding="utf-8",
)
builtin_data_path = Path(BotocoreSession().get_component("data_loader").BUILTIN_DATA_PATH)
(data_path / "endpoints").symlink_to(builtin_data_path / "endpoints")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
real_session = runtime_preflight.boto3.session.Session()
clients = {
service_name: real_session.client(service_name, region_name="us-west-2")
for service_name in ("bedrock", "bedrock-runtime")
}
clients["bedrock"].list_foundation_models = lambda: {"modelSummaries": [{"modelId": "test-model"}]}
class Session:
_session = real_session._session
def get_credentials(self):
return real_session.get_credentials()
def client(self, service_name: str, **_kwargs):
return clients[service_name]
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert clients["bedrock"].meta.endpoint_url == "https://catalog.attacker.invalid"
assert clients["bedrock-runtime"].meta.endpoint_url == "https://runtime.attacker.invalid"
assert result.ok is True
assert result.catalog_authoritative is False
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
def test_bedrock_custom_endpoint_rules_cannot_make_custom_hosts_authoritative(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
endpoint_rule_set = {
"version": "1.0",
"parameters": {
"Region": {"builtIn": "AWS::Region", "required": True, "type": "String"},
"UseDualStack": {
"builtIn": "AWS::UseDualStack",
"required": True,
"default": False,
"type": "Boolean",
},
"UseFIPS": {
"builtIn": "AWS::UseFIPS",
"required": True,
"default": False,
"type": "Boolean",
},
},
"rules": [
{
"conditions": [],
"type": "endpoint",
"endpoint": {
"url": "https://rules.attacker.invalid",
"properties": {},
"headers": {},
},
}
],
}
for service_name, api_version in (
("bedrock", "2023-04-20"),
("bedrock-runtime", "2023-09-30"),
):
rule_path = data_path / service_name / api_version / "endpoint-rule-set-1.json"
rule_path.parent.mkdir(parents=True)
rule_path.write_text(json.dumps(endpoint_rule_set), encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
real_session = runtime_preflight.boto3.session.Session()
clients = {
service_name: real_session.client(service_name, region_name="us-west-2")
for service_name in ("bedrock", "bedrock-runtime")
}
clients["bedrock"].list_foundation_models = lambda: {"modelSummaries": [{"modelId": "test-model"}]}
class Session:
_session = real_session._session
def get_credentials(self):
return real_session.get_credentials()
def client(self, service_name: str, **_kwargs):
return clients[service_name]
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
routed_urls = {
clients[service_name]
._ruleset_resolver.construct_endpoint(
clients[service_name].meta.service_model.operation_model(operation_name),
{},
{},
)
.url
for service_name, operation_name in (
("bedrock", "ListFoundationModels"),
("bedrock-runtime", "InvokeModel"),
)
}
assert routed_urls == {"https://rules.attacker.invalid"}
assert result.ok is True
assert result.catalog_authoritative is False
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
@pytest.mark.parametrize(
("status_code", "expected_ok", "expected_kind", "expected_disposition"),
[(200, True, None, "degraded"), (401, False, "authentication", "fatal")],
)
def test_bedrock_bearer_token_only_uses_real_catalog_transport(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
status_code: int,
expected_ok: bool,
expected_kind: str | None,
expected_disposition: str,
) -> None:
auth_schemes = BotocoreSession().get_service_model("bedrock").metadata.get("auth", [])
if "smithy.api#httpBearerAuth" not in auth_schemes:
pytest.skip("installed Botocore does not support Bedrock bearer authentication")
seen_authorization: list[bool] = []
class CatalogResponse(BaseHTTPRequestHandler):
def do_GET(self) -> None:
seen_authorization.append(self.headers.get("Authorization") == "Bearer test-bearer-token")
body = (
{"modelSummaries": [{"modelId": "test-model"}]}
if status_code == 200
else {"message": "private rejected bearer"}
)
encoded = json.dumps(body).encode()
self.send_response(status_code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
def log_message(self, _format, *_args) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "test-bearer-token")
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
with ThreadingHTTPServer(("127.0.0.1", 0), CatalogResponse) as server:
endpoint = f"http://127.0.0.1:{server.server_port}"
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK", endpoint)
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", endpoint)
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
worker.join(timeout=5)
assert seen_authorization == [True]
assert result.ok is expected_ok
assert result.failure_kind == expected_kind
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
assert "test-bearer-token" not in result.detail
assert "private rejected bearer" not in result.detail
@pytest.mark.parametrize(
"bearer_token",
["bad\nvalue", "bad\rvalue", "bad\x7fvalue", "bad value", "bad\N{LATIN SMALL LETTER E WITH ACUTE}value"],
)
def test_bedrock_model_probe_rejects_unsafe_bearer_token_characters(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
bearer_token: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", bearer_token)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "InvalidConfigError" in result.detail
assert bearer_token not in result.detail
@pytest.mark.parametrize(
("error_code", "http_status", "expected_kind", "expected_disposition"),
[
("UnexpectedAuthCode", 401, "authentication", "fatal"),
("InvalidClientTokenId", 400, "authentication", "fatal"),
("SignatureDoesNotMatch", 403, "authentication", "fatal"),
("InvalidAccessKeyId", 403, "authentication", "fatal"),
("ExpiredToken", 403, "authentication", "fatal"),
("MissingAuthenticationToken", 403, "authentication", "fatal"),
("RequestExpired", 400, "invalid_configuration", "fatal"),
("AccessDeniedException", 403, "authorization", "degraded"),
("ThrottlingException", 429, "unavailable", "degraded"),
("InternalServerException", 500, "unavailable", "degraded"),
("ServiceUnavailableException", 503, "unavailable", "degraded"),
],
)
def test_bedrock_model_probe_classifies_client_errors(
monkeypatch,
error_code: str,
http_status: int,
expected_kind: str,
expected_disposition: str,
) -> None:
class Bedrock:
def list_foundation_models(self):
raise runtime_preflight.ClientError(
{
"Error": {"Code": error_code, "Message": "must stay private"},
"ResponseMetadata": {"HTTPStatusCode": http_status},
},
"ListFoundationModels",
)
class Session:
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert result.failure_kind == expected_kind
assert result.http_status == http_status
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
assert "must stay private" not in result.detail
@pytest.mark.parametrize(
("operation_name", "error_code", "http_status", "expected_kind", "expected_disposition"),
[
("AssumeRole", "AccessDenied", 403, "authentication", "fatal"),
("AssumeRoleWithWebIdentity", "InvalidIdentityToken", 400, "authentication", "fatal"),
("AssumeRoleWithWebIdentity", "IDPRejectedClaim", 400, "authentication", "fatal"),
("GetRoleCredentials", "AccessDeniedException", 403, "authentication", "fatal"),
("AssumeRoleWithWebIdentity", "IDPCommunicationError", 400, "unavailable", "degraded"),
("AssumeRole", "ThrottlingException", 429, "unavailable", "degraded"),
("AssumeRole", "InternalServerError", 500, "unavailable", "degraded"),
],
)
def test_bedrock_model_probe_classifies_credential_bootstrap_client_errors(
monkeypatch: pytest.MonkeyPatch,
operation_name: str,
error_code: str,
http_status: int,
expected_kind: str,
expected_disposition: str,
) -> None:
class Session:
def __init__(self) -> None:
raise runtime_preflight.ClientError(
{
"Error": {"Code": error_code, "Message": "must stay private"},
"ResponseMetadata": {"HTTPStatusCode": http_status},
},
operation_name,
)
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider)
assert result.failure_kind == expected_kind
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
assert result.http_status == http_status
assert "must stay private" not in result.detail
def test_bedrock_model_probe_degrades_when_active_imds_is_unavailable(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.delenv("AWS_EC2_METADATA_DISABLED")
monkeypatch.setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", "http://127.0.0.1:9")
monkeypatch.setenv("AWS_METADATA_SERVICE_TIMEOUT", "1")
monkeypatch.setenv("AWS_METADATA_SERVICE_NUM_ATTEMPTS", "1")
monkeypatch.setenv("NO_PROXY", "*")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "127.0.0.1" not in result.detail
@pytest.mark.parametrize("bearer_token", [None, " \t "])
def test_bedrock_model_probe_fails_when_credentials_are_absent_and_imds_is_disabled(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
bearer_token: str | None,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
if bearer_token is not None:
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", bearer_token)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "authentication"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
@pytest.mark.parametrize(
("status_code", "expected_kind", "expected_disposition"),
[(500, "unavailable", "degraded"), (404, "authentication", "fatal")],
)
def test_bedrock_model_probe_distinguishes_transient_imds_from_no_role(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
status_code: int,
expected_kind: str,
expected_disposition: str,
) -> None:
requests: list[str] = []
class MetadataResponse(BaseHTTPRequestHandler):
def _respond(self) -> None:
requests.append(self.path)
self.send_response(status_code)
self.end_headers()
def do_GET(self) -> None:
self._respond()
def do_PUT(self) -> None:
self._respond()
def log_message(self, _format, *_args) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.delenv("AWS_EC2_METADATA_DISABLED")
monkeypatch.setenv("AWS_METADATA_SERVICE_TIMEOUT", "1")
monkeypatch.setenv("AWS_METADATA_SERVICE_NUM_ATTEMPTS", "1")
monkeypatch.setenv("NO_PROXY", "*")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
with ThreadingHTTPServer(("127.0.0.1", 0), MetadataResponse) as server:
monkeypatch.setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", f"http://127.0.0.1:{server.server_port}")
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
worker.join(timeout=5)
assert requests
assert result.failure_kind == expected_kind
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
assert "127.0.0.1" not in result.detail
@pytest.mark.parametrize("credential_body", [b"{bad", b'{"AccessKeyId":"test"}', b""])
def test_bedrock_model_probe_degrades_malformed_imds_credential_responses(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
credential_body: bytes,
) -> None:
class MetadataResponse(BaseHTTPRequestHandler):
def _write(self, body: bytes) -> None:
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_PUT(self) -> None:
self._write(b"token")
def do_GET(self) -> None:
if self.path.endswith("/iam/security-credentials/"):
self._write(b"TestRole")
else:
self._write(credential_body)
def log_message(self, _format, *_args) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.delenv("AWS_EC2_METADATA_DISABLED")
monkeypatch.setenv("AWS_METADATA_SERVICE_TIMEOUT", "1")
monkeypatch.setenv("AWS_METADATA_SERVICE_NUM_ATTEMPTS", "1")
monkeypatch.setenv("NO_PROXY", "*")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
with ThreadingHTTPServer(("127.0.0.1", 0), MetadataResponse) as server:
monkeypatch.setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", f"http://127.0.0.1:{server.server_port}")
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
worker.join(timeout=5)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "TestRole" not in result.detail
def test_bedrock_model_probe_degrades_malformed_imds_role_listing(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
class MetadataResponse(BaseHTTPRequestHandler):
def _write(self, status_code: int, body: bytes = b"") -> None:
self.send_response(status_code)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_PUT(self) -> None:
self._write(200, b"token")
def do_GET(self) -> None:
if self.path.endswith("/iam/security-credentials/"):
self._write(200, b"Role1\nRole2")
else:
self._write(404)
def log_message(self, _format, *_args) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.delenv("AWS_EC2_METADATA_DISABLED")
monkeypatch.setenv("AWS_METADATA_SERVICE_TIMEOUT", "1")
monkeypatch.setenv("AWS_METADATA_SERVICE_NUM_ATTEMPTS", "1")
monkeypatch.setenv("NO_PROXY", "*")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
with ThreadingHTTPServer(("127.0.0.1", 0), MetadataResponse) as server:
monkeypatch.setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", f"http://127.0.0.1:{server.server_port}")
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
worker.join(timeout=5)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "Role1" not in result.detail
@pytest.mark.parametrize("proxy_url", ["http://[::1", "http://127.0.0.1:banana"])
def test_bedrock_model_probe_classifies_malformed_effective_proxy_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
proxy_url: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
for variable in ("https_proxy", "all_proxy", "ALL_PROXY", "no_proxy"):
monkeypatch.delenv(variable, raising=False)
monkeypatch.setenv("HTTPS_PROXY", proxy_url)
monkeypatch.setenv("NO_PROXY", "")
provider = ProviderConfig(
provider="bedrock",
model="test-model",
api_key=None,
base_url=None,
litellm_model="bedrock/test-model",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=2)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "HTTPClientError" in result.detail
assert proxy_url not in result.detail
def test_bedrock_model_probe_degrades_valid_but_unavailable_proxy(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
for variable in ("https_proxy", "all_proxy", "ALL_PROXY", "no_proxy"):
monkeypatch.delenv(variable, raising=False)
monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:9")
monkeypatch.setenv("NO_PROXY", "")
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=2)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "127.0.0.1" not in result.detail
def test_bedrock_model_probe_does_not_blame_proxy_bypassed_by_no_proxy(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
class CatalogResponse(BaseHTTPRequestHandler):
def do_GET(self) -> None:
body = b'{"modelSummaries":[{"modelId":"test-model"}]}'
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, _format, *_args) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
for variable in ("http_proxy", "all_proxy", "ALL_PROXY", "no_proxy"):
monkeypatch.delenv(variable, raising=False)
monkeypatch.setenv("HTTP_PROXY", "http://[::1")
monkeypatch.setenv("NO_PROXY", "127.0.0.1")
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
with ThreadingHTTPServer(("127.0.0.1", 0), CatalogResponse) as server:
endpoint = f"http://127.0.0.1:{server.server_port}"
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK", endpoint)
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", endpoint)
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
worker.join(timeout=5)
assert result.ok is True
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
@pytest.mark.parametrize(
("error", "expected_kind"),
[
(NoCredentialsError(), "authentication"),
(UnauthorizedSSOTokenError(), "authentication"),
(RefreshWithMFAUnsupportedError(), "authentication"),
(SSOTokenLoadError(error_msg="private SSO token detail"), "authentication"),
(NoAuthTokenError(), "authentication"),
(
PartialCredentialsError(
provider="env",
cred_var="AWS_SECRET_ACCESS_KEY",
),
"invalid_configuration",
),
(ProfileNotFound(profile="missing"), "invalid_configuration"),
(ConfigParseError(path="/private/config"), "invalid_configuration"),
(ConfigNotFound(path="/private/config"), "invalid_configuration"),
(
ApiVersionNotFoundError(data_path="/private/data", api_version="private version"),
"invalid_configuration",
),
(DataNotFoundError(data_path="/private/data"), "invalid_configuration"),
(BaseEndpointResolverError(), "invalid_configuration"),
(EndpointProviderError(msg="private endpoint detail"), "invalid_configuration"),
(InvalidConfigError(error_msg="invalid profile"), "invalid_configuration"),
(InvalidDefaultsMode(mode="private mode", valid_modes=["standard"]), "invalid_configuration"),
(InvalidIMDSEndpointError(endpoint="private endpoint"), "invalid_configuration"),
(
InvalidIMDSEndpointModeError(mode="private mode", valid_modes=["ipv4", "ipv6"]),
"invalid_configuration",
),
(InvalidProxiesConfigError(), "invalid_configuration"),
(InvalidRegionError(region_name="not a region"), "invalid_configuration"),
(
InvalidRetryConfigurationError(
retry_config_option="private option",
valid_options=["mode", "max_attempts"],
),
"invalid_configuration",
),
(
InvalidSTSRegionalEndpointsConfigError(sts_regional_endpoints_config="private mode"),
"invalid_configuration",
),
(MissingDependencyException(msg="private dependency"), "invalid_configuration"),
(ParamValidationError(report="private parameter detail"), "invalid_configuration"),
(
ServiceNotInRegionError(service_name="private service", region_name="private region"),
"invalid_configuration",
),
(UnknownCredentialError(name="private credential provider"), "invalid_configuration"),
(UnknownRegionError(region_name="private region", error_msg="private detail"), "invalid_configuration"),
(UnknownSignatureVersionError(signature_version="private signature"), "invalid_configuration"),
(UnsupportedSignatureVersionError(signature_version="private signature"), "invalid_configuration"),
(FileNotFoundError("private credential file"), "invalid_configuration"),
(IsADirectoryError("private credential directory"), "invalid_configuration"),
(NotADirectoryError("private credential parent"), "invalid_configuration"),
(PermissionError("private credential file"), "invalid_configuration"),
*(
[
(
botocore_exceptions.UnsupportedServiceProtocolsError(
botocore_supported_protocols="private protocols",
service="private service",
service_supported_protocols="private protocols",
),
"invalid_configuration",
)
]
if hasattr(botocore_exceptions, "UnsupportedServiceProtocolsError")
else []
),
*(
[
(
botocore_exceptions.InvalidChecksumConfigError(
config_key="request_checksum_calculation",
valid_options=["when_supported"],
config_value="private value",
),
"invalid_configuration",
)
]
if hasattr(botocore_exceptions, "InvalidChecksumConfigError")
else []
),
*(
[
(
botocore_exceptions.UnknownTokenProviderError(name="private token provider"),
"invalid_configuration",
)
]
if hasattr(botocore_exceptions, "UnknownTokenProviderError")
else []
),
*(
[(botocore_exceptions.LoginRefreshRequired(), "authentication")]
if hasattr(botocore_exceptions, "LoginRefreshRequired")
else []
),
],
)
def test_bedrock_model_probe_fails_on_deterministic_sdk_errors(
monkeypatch: pytest.MonkeyPatch,
error: Exception,
expected_kind: str,
) -> None:
class Session:
def __init__(self) -> None:
raise error
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider)
assert result.ok is False
assert result.failure_kind == expected_kind
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert type(error).__name__ in result.detail
assert str(error) not in result.detail
@pytest.mark.parametrize("phase", ["session", "catalog"])
def test_bedrock_model_probe_degrades_unattributed_value_errors(
monkeypatch: pytest.MonkeyPatch,
phase: str,
) -> None:
error = ValueError("private remote response detail")
class Bedrock:
def list_foundation_models(self):
raise error
class Session:
def __init__(self) -> None:
if phase == "session":
raise error
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider)
assert result.failure_kind == ("unknown" if phase == "session" else "invalid_response")
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private remote response detail" not in result.detail
def test_bedrock_model_probe_degrades_transport_sdk_errors(monkeypatch: pytest.MonkeyPatch) -> None:
class Session:
def __init__(self) -> None:
raise EndpointConnectionError(endpoint_url="https://bedrock.us-west-2.amazonaws.com/private")
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private" not in result.detail
def test_bedrock_model_probe_classifies_real_invalid_region_as_fatal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-access-key")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="not a region",
)
# Real SDK initialization counts against the probe deadline. The invalid region
# is rejected before any request is sent, so allow the same setup budget as the
# other real-SDK classification tests instead of measuring one-second throughput.
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "InvalidRegionError" in result.detail
assert "not a region" not in result.detail
@pytest.mark.parametrize(
("environment", "expected_error"),
[
({"AWS_DEFAULTS_MODE": "private invalid mode"}, "InvalidDefaultsMode"),
({"AWS_EC2_METADATA_SERVICE_ENDPOINT": "private invalid endpoint"}, "InvalidIMDSEndpointError"),
({"AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE": "private invalid mode"}, "InvalidIMDSEndpointModeError"),
({"AWS_RETRY_MODE": "private invalid mode"}, "InvalidRetryModeError"),
({"AWS_MAX_ATTEMPTS": "private invalid integer"}, "ValueError"),
({"AWS_METADATA_SERVICE_TIMEOUT": "private invalid integer"}, "ValueError"),
({"AWS_METADATA_SERVICE_TIMEOUT": "0"}, "InvalidConfigError"),
({"AWS_METADATA_SERVICE_TIMEOUT": "-1"}, "InvalidConfigError"),
({"AWS_METADATA_SERVICE_NUM_ATTEMPTS": "private invalid integer"}, "ValueError"),
({"AWS_ENDPOINT_URL": "private invalid endpoint"}, "ValueError"),
({"AWS_ENDPOINT_URL_BEDROCK": "private invalid endpoint"}, "ValueError"),
({"AWS_ENDPOINT_URL_BEDROCK_RUNTIME": "private invalid endpoint"}, "ValueError"),
(
{"AWS_CSM_ENABLED": "true", "AWS_CSM_PORT": "private invalid integer"},
"ValueError",
),
],
)
def test_bedrock_model_probe_classifies_real_invalid_aws_environment_as_fatal(
monkeypatch: pytest.MonkeyPatch,
environment: dict[str, str],
expected_error: str,
) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-access-key")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
for variable, value in environment.items():
monkeypatch.setenv(variable, value)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert expected_error in result.detail
assert all(value not in result.detail for value in environment.values())
def test_bedrock_model_probe_classifies_real_invalid_profile_api_version_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
region = us-west-2
api_versions =
bedrock = 1900-01-01
""",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "DataNotFoundError" in result.detail
assert "1900-01-01" not in result.detail
def test_bedrock_model_probe_classifies_real_invalid_profile_max_attempts_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
region = us-west-2
max_attempts = private invalid integer
""",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "ValueError" in result.detail
assert "private invalid integer" not in result.detail
@pytest.mark.parametrize("max_attempts", ["0", "-1"])
def test_bedrock_model_probe_classifies_out_of_range_profile_max_attempts_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
max_attempts: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
max_attempts = {max_attempts}
""",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "InvalidMaxRetryAttemptsError" in result.detail
assert max_attempts not in result.detail
@pytest.mark.parametrize("service_name", ["bedrock", "bedrock-runtime"])
def test_bedrock_model_probe_classifies_real_malformed_local_service_data_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
service_name: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
service_model = data_path / service_name / "9999-01-01" / "service-2.json"
service_model.parent.mkdir(parents=True)
service_model.write_text("{private malformed service data", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK", "http://127.0.0.1:1")
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed service data" not in result.detail
def test_bedrock_model_probe_classifies_invalid_compressed_local_sdk_data_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
endpoint_rules = data_path / "bedrock" / "2023-04-20" / "endpoint-rule-set-1.json.gz"
endpoint_rules.parent.mkdir(parents=True)
endpoint_rules.write_text("private invalid gzip data", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "BadGzipFile" in result.detail
assert "private invalid gzip data" not in result.detail
def test_bedrock_model_probe_does_not_blame_remote_bad_gzip_as_local_configuration(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Bedrock:
def list_foundation_models(self):
raise gzip.BadGzipFile("private remote response")
class Session:
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "unknown"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private remote response" not in result.detail
@pytest.mark.parametrize("credential_source", ["web_identity", "container"])
def test_bedrock_model_probe_classifies_invalid_local_credential_token_encoding_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
credential_source: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
token_path = tmp_path / "private-token"
token_path.write_bytes(b"\xff")
if credential_source == "web_identity":
monkeypatch.setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/Test")
monkeypatch.setenv("AWS_WEB_IDENTITY_TOKEN_FILE", str(token_path))
else:
monkeypatch.setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://127.0.0.1:9/credentials")
monkeypatch.setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", str(token_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "UnicodeDecodeError" in result.detail
assert str(token_path) not in result.detail
def test_bedrock_model_probe_does_not_blame_remote_unicode_error_as_local_configuration(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Bedrock:
def list_foundation_models(self):
raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "private remote response")
class Session:
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_response"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private remote response" not in result.detail
def test_bedrock_model_probe_classifies_malformed_assume_role_service_data_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
role_arn = arn:aws:iam::123456789012:role/Test
source_profile = source
region = us-west-2
""",
credentials="""\
[source]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
service_model = data_path / "sts" / "9999-01-01" / "service-2.json"
service_model.parent.mkdir(parents=True)
service_model.write_text("{private malformed service data", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed service data" not in result.detail
def test_bedrock_model_probe_classifies_malformed_assume_role_endpoint_rules_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
role_arn = arn:aws:iam::123456789012:role/Test
source_profile = source
region = us-west-2
""",
credentials="""\
[source]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
endpoint_rules = data_path / "sts" / "2011-06-15" / "endpoint-rule-set-1.json"
endpoint_rules.parent.mkdir(parents=True)
endpoint_rules.write_text("{private malformed endpoint rules", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed endpoint rules" not in result.detail
@pytest.mark.parametrize(
("service_name", "api_version"),
[("bedrock", "2023-04-20"), ("bedrock-runtime", "2023-09-30")],
)
def test_bedrock_model_probe_classifies_malformed_runtime_endpoint_rules_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
service_name: str,
api_version: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
endpoint_rules = data_path / service_name / api_version / "endpoint-rule-set-1.json"
endpoint_rules.parent.mkdir(parents=True)
endpoint_rules.write_text("{private malformed endpoint rules", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed endpoint rules" not in result.detail
def test_bedrock_model_probe_classifies_malformed_local_endpoints_data_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
data_path = tmp_path / "aws-data"
data_path.mkdir()
(data_path / "endpoints.json").write_text("{private malformed endpoints data", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed endpoints data" not in result.detail
def test_bedrock_model_probe_classifies_malformed_nested_sso_service_data_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
role_arn = arn:aws:iam::123456789012:role/Test
source_profile = source
region = us-west-2
[profile source]
sso_start_url = https://private.awsapps.example/start
sso_region = us-west-2
sso_account_id = 123456789012
sso_role_name = TestRole
""",
)
data_path = tmp_path / "aws-data"
service_model = data_path / "sso" / "9999-01-01" / "service-2.json"
service_model.parent.mkdir(parents=True)
service_model.write_text("{private malformed service data", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert "private malformed service data" not in result.detail
@pytest.mark.parametrize(
("expires_at", "expected_error"),
[
("private malformed timestamp", "ParserError"),
(None, "TypeError"),
(123, "TypeError"),
([], "TypeError"),
({}, "TypeError"),
],
)
def test_bedrock_model_probe_classifies_malformed_local_sso_token_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
expires_at: object,
expected_error: str,
) -> None:
start_url = "https://private.awsapps.example/start"
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
sso_start_url = {start_url}
sso_region = us-west-2
sso_account_id = 123456789012
sso_role_name = TestRole
""",
)
cache_dir = tmp_path / "sso-cache"
cache_dir.mkdir()
cache_key = hashlib.sha1(start_url.encode()).hexdigest()
(cache_dir / f"{cache_key}.json").write_text(
json.dumps(
{
"accessToken": "private-access-token",
"expiresAt": expires_at,
}
),
encoding="utf-8",
)
monkeypatch.setattr(botocore_credentials.SSOProvider, "_SSO_TOKEN_CACHE_DIR", str(cache_dir))
monkeypatch.setattr(botocore_tokens.SSOTokenProvider, "_SSO_TOKEN_CACHE_DIR", str(cache_dir))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "authentication"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert expected_error in result.detail or "UnauthorizedSSOTokenError" in result.detail
assert "private" not in result.detail
def test_bedrock_model_probe_classifies_malformed_local_login_token_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
if not hasattr(botocore_credentials, "LoginProvider"):
pytest.skip("installed Botocore has no login credential provider")
session_name = "private-login-session"
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
login_session = {session_name}
""",
)
cache_dir = tmp_path / "login-cache"
cache_dir.mkdir()
cache_key = hashlib.sha256(session_name.encode()).hexdigest()
(cache_dir / f"{cache_key}.json").write_text(
json.dumps(
{
"accessToken": {
"accessKeyId": "test-access-key",
"secretAccessKey": "test-secret-key",
"sessionToken": "test-token",
"accountId": "123456789012",
"expiresAt": "private malformed timestamp",
},
"refreshToken": "test-refresh-token",
"dpopKey": "test-key",
"clientId": "test-client",
}
),
encoding="utf-8",
)
monkeypatch.setenv("AWS_LOGIN_CACHE_DIRECTORY", str(cache_dir))
monkeypatch.setattr(botocore_credentials, "EC", object(), raising=False)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "authentication"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "ParserError" in result.detail
assert session_name not in result.detail
def test_bedrock_model_probe_classifies_malformed_credential_process_without_blame_unused_service_data(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
credential_process = tmp_path / "credential-process.py"
credential_process.write_text("print('{')\n", encoding="utf-8")
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
credential_process = "{sys.executable}" "{credential_process}"
""",
)
data_path = tmp_path / "aws-data"
service_model = data_path / "sts" / "9999-01-01" / "service-2.json"
service_model.parent.mkdir(parents=True)
service_model.write_text("{\n", encoding="utf-8")
monkeypatch.setenv("AWS_DATA_PATH", str(data_path))
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "JSONDecodeError" in result.detail
assert str(credential_process) not in result.detail
@pytest.mark.parametrize("configuration_source", ["environment", "profile"])
@pytest.mark.parametrize("bundle_state", ["missing", "invalid"])
def test_bedrock_model_probe_classifies_proven_invalid_ca_bundle_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
bundle_state: str,
configuration_source: str,
) -> None:
bundle_path = tmp_path / "private-ca-bundle.pem"
if bundle_state == "invalid":
bundle_path.write_text("private invalid CA data", encoding="utf-8")
if configuration_source == "environment":
monkeypatch.setenv("AWS_CA_BUNDLE", str(bundle_path))
real_session = None
else:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
ca_bundle = {bundle_path}
""",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
real_session = runtime_preflight.boto3.session.Session()
error = BotocoreSSLError(endpoint_url="https://private.example", error="private TLS detail")
class Session:
def __init__(self) -> None:
if real_session is None:
raise error
self._session = real_session._session
def client(self, *_args, **_kwargs):
raise error
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "SSLError" in result.detail
assert "private" not in result.detail
def test_bedrock_model_probe_degrades_unattributed_tls_errors(monkeypatch: pytest.MonkeyPatch) -> None:
error = BotocoreSSLError(endpoint_url="https://private.example", error="private TLS detail")
class Session:
def __init__(self) -> None:
raise error
monkeypatch.delenv("AWS_CA_BUNDLE", raising=False)
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private" not in result.detail
def test_bedrock_model_probe_rejects_invalid_ca_bundle_before_https_runtime(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
bundle_path = tmp_path / "missing-ca-bundle.pem"
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
monkeypatch.setenv("AWS_CA_BUNDLE", str(bundle_path))
real_session = runtime_preflight.boto3.session.Session()
class Client:
meta = type("Meta", (), {"endpoint_url": "https://private.example"})()
def list_foundation_models(self):
return {"modelSummaries": [{"modelId": "test-model"}]}
class Session:
def __init__(self) -> None:
self._session = real_session._session
def get_credentials(self):
return real_session.get_credentials()
def client(self, *_args, **_kwargs):
return Client()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert str(bundle_path) not in result.detail
def test_bedrock_model_probe_classifies_real_invalid_credential_process_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
region = us-west-2
credential_process = private-command "unterminated
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "ValueError" in result.detail
assert "private-command" not in result.detail
def test_bedrock_model_probe_classifies_real_unexecutable_credential_process_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
credential_process = tmp_path / "credential-process"
credential_process.write_text("private invalid executable", encoding="utf-8")
credential_process.chmod(0o700)
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
credential_process = {credential_process}
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "OSError" in result.detail
assert str(credential_process) not in result.detail
@pytest.mark.parametrize(
("payload", "exit_code", "expected_kind", "expected_disposition"),
[
({"Version": 1}, 0, "invalid_configuration", "fatal"),
({"Version": 2}, 0, "invalid_configuration", "fatal"),
(
{
"Version": 1,
"AccessKeyId": "test-access-key",
"SecretAccessKey": "test-secret-key",
"SessionToken": "test-session-token",
"Expiration": "private malformed timestamp",
},
0,
"invalid_configuration",
"fatal",
),
(None, 1, "unavailable", "degraded"),
],
)
def test_bedrock_model_probe_classifies_structural_credential_process_output_causally(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
payload: dict[str, object] | None,
exit_code: int,
expected_kind: str,
expected_disposition: str,
) -> None:
credential_process = tmp_path / "credential-process.py"
if exit_code != 0:
credential_process.write_text(
'import sys\nprint("private process failure", file=sys.stderr)\nsys.exit(1)\n',
encoding="utf-8",
)
else:
credential_process.write_text(f"print({json.dumps(payload)!r})\n", encoding="utf-8")
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config=f"""\
[profile test]
region = us-west-2
credential_process = "{sys.executable}" "{credential_process}"
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == expected_kind
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
assert "private" not in result.detail
assert str(credential_process) not in result.detail
def test_bedrock_model_probe_degrades_transient_credential_process_spawn_error(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
original_init = botocore_credentials.ProcessProvider.__init__
def failing_init(self, *args, **kwargs) -> None:
original_init(self, *args, **kwargs)
def fail_to_spawn(*_args, **_kwargs):
raise BlockingIOError(errno.EAGAIN, "private temporary spawn failure")
self._popen = fail_to_spawn
monkeypatch.setattr(botocore_credentials.ProcessProvider, "__init__", failing_init)
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
region = us-west-2
credential_process = private-command
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private" not in result.detail
@pytest.mark.parametrize(
("environment", "expected_error"),
[
({"AWS_STS_REGIONAL_ENDPOINTS": "private invalid mode"}, "InvalidSTSRegionalEndpointsConfigError"),
({"AWS_RETRY_MODE": "private invalid mode"}, "InvalidRetryModeError"),
({"AWS_MAX_ATTEMPTS": "private invalid integer"}, "ValueError"),
({"AWS_ENDPOINT_URL_STS": "private invalid endpoint"}, "ValueError"),
],
)
def test_bedrock_model_probe_classifies_real_invalid_assume_role_configuration_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
environment: dict[str, str],
expected_error: str,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
role_arn = arn:aws:iam::123456789012:role/Test
source_profile = source
region = us-west-2
""",
credentials="""\
[source]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
for variable, value in environment.items():
monkeypatch.setenv(variable, value)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert expected_error in result.detail
assert all(value not in result.detail for value in environment.values())
def test_bedrock_model_probe_classifies_real_invalid_assume_role_parameters_as_fatal(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="""\
[profile test]
role_arn = arn:aws:iam::123456789012:role/Test
role_session_name = !
source_profile = source
region = us-west-2
""",
credentials="""\
[source]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
provider = ProviderConfig(
provider="bedrock",
model="us.anthropic.claude-sonnet-test-v1:0",
api_key=None,
base_url=None,
litellm_model="bedrock/us.anthropic.claude-sonnet-test-v1:0",
region="us-west-2",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.failure_kind == "invalid_configuration"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert "ParamValidationError" in result.detail
assert "role_session_name" not in result.detail
def test_bedrock_model_probe_degrades_malformed_catalog_timestamp(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
lifecycle_shape = BotocoreSession().get_service_model("bedrock").shape_for("FoundationModelLifecycle")
if "startOfLifeTime" not in lifecycle_shape.members:
pytest.skip("installed Botocore service model has no Bedrock lifecycle timestamps")
requests: list[str] = []
class MalformedCatalog(BaseHTTPRequestHandler):
def do_GET(self) -> None:
requests.append(self.path)
payload = json.dumps(
{
"modelSummaries": [
{
"modelId": "test-model",
"modelLifecycle": {
"status": "ACTIVE",
"startOfLifeTime": "private malformed timestamp",
},
}
]
}
).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, _format: str, *_args: object) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
credentials="""\
[test]
aws_access_key_id = test-access-key
aws_secret_access_key = test-secret-key
""",
)
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
with ThreadingHTTPServer(("127.0.0.1", 0), MalformedCatalog) as server:
monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK", f"http://127.0.0.1:{server.server_port}")
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
thread.join(timeout=2)
assert requests == ["/foundation-models"]
assert result.failure_kind == "invalid_response"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private malformed timestamp" not in result.detail
def test_bedrock_model_probe_degrades_malformed_container_credential_timestamp(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
requests: list[str] = []
class MalformedCredentials(BaseHTTPRequestHandler):
def do_GET(self) -> None:
requests.append(self.path)
payload = json.dumps(
{
"AccessKeyId": "test-access-key",
"SecretAccessKey": "test-secret-key",
"Token": "test-token",
"Expiration": "private malformed timestamp",
}
).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, _format: str, *_args: object) -> None:
return
_configure_isolated_aws_profile(
monkeypatch,
tmp_path,
config="[profile test]\nregion = us-west-2\n",
)
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
with ThreadingHTTPServer(("127.0.0.1", 0), MalformedCredentials) as server:
monkeypatch.setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", f"http://127.0.0.1:{server.server_port}/creds")
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
provider = ProviderConfig("bedrock", "test-model", None, None, "bedrock/test-model", region="us-west-2")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
finally:
server.shutdown()
thread.join(timeout=2)
assert requests == ["/creds"]
assert result.failure_kind == "unknown"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert "private malformed timestamp" not in result.detail
def test_bedrock_model_probe_uses_a_fresh_session_per_route(monkeypatch) -> None:
sessions: list[object] = []
class Bedrock:
def list_foundation_models(self):
return {"modelSummaries": []}
class Session:
def __init__(self):
sessions.append(self)
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
first = ProviderConfig("bedrock", "model-a", None, None, "bedrock/model-a", region="us-west-2")
second = ProviderConfig("bedrock", "model-b", None, None, "bedrock/model-b", region="us-west-2")
runtime_preflight.probe_model(first)
runtime_preflight.probe_model(second)
assert len(sessions) == 2
assert sessions[0] is not sessions[1]
def test_bedrock_model_probe_allows_distinct_routes_to_overlap(monkeypatch) -> None:
rendezvous = Barrier(2)
counter_lock = Lock()
active = 0
max_active = 0
class Bedrock:
def list_foundation_models(self):
nonlocal active, max_active
with counter_lock:
active += 1
max_active = max(max_active, active)
try:
rendezvous.wait(timeout=0.5)
return {"modelSummaries": []}
finally:
with counter_lock:
active -= 1
class Session:
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
providers = [
ProviderConfig("bedrock", "model-a", None, None, "bedrock/model-a", region="us-west-2"),
ProviderConfig("bedrock", "model-b", None, None, "bedrock/model-b", region="us-west-2"),
]
with ThreadPoolExecutor(max_workers=2) as pool:
results = list(pool.map(runtime_preflight.probe_model, providers))
assert max_active == 2
assert all(result.failure_kind is None and "not listed" in result.detail for result in results)
def test_bedrock_model_probe_deadline_covers_blocked_credential_provider(monkeypatch) -> None:
release = Event()
credential_provider_finished = Event()
existing_threads = {id(thread) for thread in enumerate_threads()}
class Session:
def __init__(self):
release.wait(timeout=1)
credential_provider_finished.set()
def client(self, *_args, **_kwargs):
raise AssertionError("client creation must remain inside the deadline worker")
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "model", None, None, "bedrock/model", region="us-west-2")
try:
result = runtime_preflight.probe_model(provider, timeout_seconds=0.02)
credential_provider_finished_before_release = credential_provider_finished.is_set()
finally:
release.set()
for thread in enumerate_threads():
if id(thread) not in existing_threads and thread.name == "bedrock-model-catalog-probe":
thread.join(timeout=0.2)
assert result.ok is False
assert result.failure_kind == "unavailable"
assert "timed out" in result.detail
assert credential_provider_finished_before_release is False
def test_bedrock_model_probe_deadline_caps_blocked_workers(monkeypatch) -> None:
release = Event()
existing_threads = {id(thread) for thread in enumerate_threads()}
class Session:
def __init__(self):
release.wait(timeout=1)
def client(self, *_args, **_kwargs):
raise AssertionError("client creation must remain inside the deadline worker")
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
provider = ProviderConfig("bedrock", "model", None, None, "bedrock/model", region="us-west-2")
try:
results = [runtime_preflight.probe_model(provider, timeout_seconds=0.005) for _ in range(10)]
probe_threads = [
thread
for thread in enumerate_threads()
if id(thread) not in existing_threads and thread.name == "bedrock-model-catalog-probe"
]
finally:
release.set()
assert all(result.failure_kind == "unavailable" and "timed out" in result.detail for result in results)
assert len(probe_threads) == 4
for thread in probe_threads:
thread.join(timeout=0.2)
@pytest.mark.parametrize("failure_phase", ["constructor", "start"])
def test_bedrock_model_probe_releases_slot_when_worker_creation_fails(
monkeypatch: pytest.MonkeyPatch,
failure_phase: str,
) -> None:
real_thread = runtime_preflight.Thread
attempts = 0
class ThreadFailsOnce:
def __init__(self, *args, **kwargs) -> None:
nonlocal attempts
attempts += 1
if attempts == 1 and failure_phase == "constructor":
raise RuntimeError("private worker-constructor detail")
self._thread = real_thread(*args, **kwargs)
def start(self) -> None:
if attempts == 1 and failure_phase == "start":
raise RuntimeError("private worker-start detail")
self._thread.start()
class Bedrock:
def list_foundation_models(self):
return {"modelSummaries": [{"modelId": "model"}]}
class Session:
def client(self, *_args, **_kwargs):
return Bedrock()
monkeypatch.setattr(runtime_preflight, "Thread", ThreadFailsOnce)
monkeypatch.setattr(runtime_preflight.boto3.session, "Session", Session)
monkeypatch.setattr(runtime_preflight, "_BEDROCK_PROBE_SLOT", runtime_preflight.BoundedSemaphore(1))
provider = ProviderConfig("bedrock", "model", None, None, "bedrock/model", region="us-west-2")
failed = runtime_preflight.probe_model(provider, timeout_seconds=0.1)
recovered = runtime_preflight.probe_model(provider, timeout_seconds=0.1)
assert failed.ok is False
assert failed.failure_kind == "unknown"
assert runtime_preflight.credential_probe_disposition(provider, failed) == "degraded"
assert "RuntimeError" in failed.detail
assert "private worker" not in failed.detail
assert recovered.ok is True
def test_resolve_vertex_model_id_aliases():
"""Verify common Claude model names are mapped to Vertex versioned model IDs."""
from skillevaluator.tier3.harbor.runtime_preflight import _resolve_vertex_model_id
assert _resolve_vertex_model_id("Claude-3-7-Sonnet") == "claude-3-7-sonnet@20250219"
assert _resolve_vertex_model_id("Anthropic/claude-3-5-haiku") == "claude-3-5-haiku@20241022"
assert _resolve_vertex_model_id("Claude-3-7-Sonnet@20250219") == "claude-3-7-sonnet@20250219"
assert _resolve_vertex_model_id("claude-future-model") == "claude-future-model"
def test_vertex_model_probe_success(monkeypatch: pytest.MonkeyPatch):
"""Verify successful Vertex AI probe returns ok=True and disposition=VERIFIED."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setenv("CLOUD_ML_REGION", "us-east5")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
class MockResponse:
status = 200
def __enter__(self):
return self
def __exit__(self, *_args):
pass
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", lambda *_args, **_kwargs: MockResponse())
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is True
assert "available on Vertex AI" in result.detail
assert runtime_preflight.credential_probe_disposition(provider, result) == "verified"
def test_vertex_model_probe_auth_failure(monkeypatch: pytest.MonkeyPatch):
"""Reject probe with authorization failure and disposition=FATAL on 403."""
import io
import urllib.error
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
def mock_urlopen(*_args, **_kwargs):
raise urllib.error.HTTPError(
"https://us-east5-aiplatform.googleapis.com/...",
403,
"PermissionDenied",
{},
io.BytesIO(b'{"error": "Caller lacks aiplatform.endpoints.predict"}'),
)
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "authorization"
assert result.http_status == 403
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
def test_vertex_model_probe_model_not_found(monkeypatch: pytest.MonkeyPatch):
"""404 on Vertex rawPredict returns model_not_found and disposition=FATAL."""
import io
import urllib.error
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
def mock_urlopen(*_args, **_kwargs):
raise urllib.error.HTTPError(
"https://us-east5-aiplatform.googleapis.com/...",
404,
"NotFound",
{},
io.BytesIO(b'{"error": "Model not found"}'),
)
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig("anthropic", "claude-nonexistent", None, None, "anthropic/claude-nonexistent")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "model_not_found"
assert result.http_status == 404
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
def test_vertex_model_probe_missing_project_id(monkeypatch: pytest.MonkeyPatch):
"""Reject probe with invalid_configuration and disposition=FATAL when project ID is missing."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
for var in ("ANTHROPIC_VERTEX_PROJECT_ID", "GOOGLE_CLOUD_PROJECT", "GCP_PROJECT", "CLOUDSDK_CORE_PROJECT"):
monkeypatch.delenv(var, raising=False)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert "requires a Google Cloud project ID" in result.detail
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
def test_vertex_model_probe_missing_token(monkeypatch: pytest.MonkeyPatch):
"""Inability to obtain access token returns authentication failure, fatal by default, degraded in GKE mode."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: None)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "authentication"
assert "access token" in result.detail
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert runtime_preflight.credential_probe_disposition(provider, result, env_mode="gke") == "degraded"
def test_vertex_model_probe_network_timeout(monkeypatch: pytest.MonkeyPatch):
"""Network timeout during Vertex probe returns unavailable and disposition=DEGRADED."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
def mock_urlopen(*_args, **_kwargs):
raise TimeoutError("timed out")
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
def test_reject_redirects_handler():
"""_RejectRedirects returns None to reject all redirects."""
from skillevaluator.model_catalog import _RejectRedirects
handler = _RejectRedirects()
assert handler.redirect_request(None, None, 302, "Found", {}, "https://example.com") is None
def test_vertex_model_probe_rejects_redirect(monkeypatch: pytest.MonkeyPatch):
"""Redirect response during probe is rejected as invalid_configuration."""
import io
import urllib.error
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
def mock_redirect(*_args, **_kwargs):
raise urllib.error.HTTPError("https://...", 302, "Found", {}, io.BytesIO(b""))
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_redirect)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert "rejected unexpected redirect (302)" in result.detail
@pytest.mark.parametrize(
("env_var", "bad_val", "expected_err"),
[
("CLOUD_ML_REGION", "us/east5", "Invalid GCP region"),
("CLOUD_ML_REGION", "../../etc", "Invalid GCP region"),
("GOOGLE_CLOUD_PROJECT", "project/traversal", "Invalid GCP project ID"),
("GOOGLE_CLOUD_PROJECT", "proj..bad", "Invalid GCP project ID"),
],
)
def test_vertex_model_probe_rejects_invalid_identifiers(
monkeypatch: pytest.MonkeyPatch, env_var: str, bad_val: str, expected_err: str
):
"""Reject un-sanitized or traversal characters in GCP region or project ID."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "valid-project")
monkeypatch.setenv("CLOUD_ML_REGION", "us-east5")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
monkeypatch.setenv(env_var, bad_val)
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert expected_err in result.detail
def test_vertex_model_probe_rejects_invalid_model_id(monkeypatch: pytest.MonkeyPatch):
"""Reject un-sanitized model ID strings with path separators."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "valid-project")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
provider = ProviderConfig("anthropic", "claude/../traversal", None, None, "anthropic/claude/../traversal")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert "Invalid model ID" in result.detail
def test_vertex_model_probe_success_2xx(monkeypatch: pytest.MonkeyPatch):
"""Verify non-200 2xx responses (e.g. 204) are treated as successful probe."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setenv("CLOUD_ML_REGION", "us-east5")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
class MockResponse204:
status = 204
def __enter__(self):
return self
def __exit__(self, *_args):
pass
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", lambda *_args, **_kwargs: MockResponse204())
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is True
assert "available on Vertex AI" in result.detail
def test_vertex_model_probe_unexpected_status(monkeypatch: pytest.MonkeyPatch):
"""Verify non-2xx status without HTTPError returns other_http failure kind."""
monkeypatch.setenv("CLAUDE_CODE_USE_VERTEX", "1")
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-gcp-project")
monkeypatch.setenv("CLOUD_ML_REGION", "us-east5")
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "mock-token")
class MockResponse418:
status = 418
def __enter__(self):
return self
def __exit__(self, *_args):
pass
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", lambda *_args, **_kwargs: MockResponse418())
provider = ProviderConfig("anthropic", "claude-3-7-sonnet", None, None, "anthropic/claude-3-7-sonnet")
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "other_http"
assert result.http_status == 418
assert "unexpected HTTP status 418" in result.detail
def test_resolve_vertex_region_whitespace_resilience(monkeypatch: pytest.MonkeyPatch):
"""Whitespace-only region evaluates to fallback, while padded region is stripped."""
from skillevaluator.tier3.harbor.runtime_preflight import _resolve_vertex_region
monkeypatch.setenv("CLOUD_ML_REGION", "us-central1")
assert _resolve_vertex_region(" us-east5 ") == "us-east5"
assert _resolve_vertex_region(" ") == "us-central1"
def test_resolve_vertex_project_id_environ_dict():
"""Verify _resolve_vertex_project_id respects an explicit environ dictionary."""
from skillevaluator.tier3.harbor.runtime_preflight import _resolve_vertex_project_id
assert _resolve_vertex_project_id({"ANTHROPIC_VERTEX_PROJECT_ID": "custom-proj"}) == "custom-proj"
assert _resolve_vertex_project_id({"GOOGLE_CLOUD_PROJECT": "gcp-proj"}) == "gcp-proj"
assert _resolve_vertex_project_id({}) is None
def test_runtime_preflight_failure_stops_full_matrix(monkeypatch, tmp_path: Path) -> None:
from skillevaluator.tier3.harbor import runner
skill = tmp_path / "demo"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
provider = ProviderConfig(
provider="nv_build",
model="meta/llama-3.1-8b-instruct",
api_key="nvapi-test",
base_url="https://integrate.api.nvidia.com/v1",
litellm_model="openai/meta/llama-3.1-8b-instruct",
)
def emit(_skill, target, **_kwargs):
task = target / "case-001"
task.mkdir(parents=True)
return [task]
full_matrix = Mock(return_value=[])
preflight_run_env: dict[str, str] = {}
monkeypatch.setenv("LLM_JUDGE_MODEL", "host-legacy")
monkeypatch.delenv("SKILL_EVAL_JUDGE_MODEL", raising=False)
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals" / "evals.json")
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
monkeypatch.setattr(runner, "generate_harbor_tasks", emit)
monkeypatch.setattr(runner, "_run_agent_pair", full_matrix)
monkeypatch.setattr(
runtime_preflight,
"probe_model",
lambda selected_provider: runtime_preflight.ModelProbeResult(
True,
selected_provider.provider,
selected_provider.model,
f"model {selected_provider.model} is available",
),
)
def fail_preflight(**kwargs):
preflight_run_env.update(kwargs["run_env"])
return runtime_preflight.PreflightResult(
False,
"opencode",
"nvidia/meta/llama-3.1-8b-instruct",
"401 Unauthorized",
"runtime-preflight-opencode",
)
monkeypatch.setattr(runtime_preflight, "run_agent_runtime_preflight", fail_preflight)
result = runner.run_harbor_eval(
skill,
["opencode"],
output_dir=tmp_path / "results",
env_mode="docker",
agent_runtime_preflight=True,
)
assert result["execution_status"] == "failed"
assert result["execution_errors"] == ["opencode runtime preflight failed: 401 Unauthorized"]
assert result["error"] == result["execution_errors"][:1]
assert result["execution_error_details_total"] == 1
assert result["execution_error_details_shown"] == 1
assert result["execution_error_details_truncated"] is False
assert preflight_run_env["LLM_JUDGE_MODEL"] == "host-legacy"
assert preflight_run_env["SKILL_EVAL_JUDGE_MODEL"] == "host-legacy"
full_matrix.assert_not_called()
result_path = Path(result["run_dir"]) / "result.json"
assert result["result_path"] == str(result_path)
assert result_path.is_file()
assert result["harbor_jobs_retained"] is False
assert result["harbor_jobs_retention_reason"] == "not_retained"
assert not (Path(result["run_dir"]) / "_harbor-jobs").exists()
assert not (Path(result["run_dir"]) / "_harbor-tasks").exists()
assert result["run_config"]["credential_validation"]["status"] == "degraded"
validation_targets = result["run_config"]["credential_validation"]["targets"]
assert {target["status"] for target in validation_targets} == {"degraded"}
assert {target["detail"] for target in validation_targets} == {
"model catalog access does not verify runtime credentials for this endpoint"
}
persisted = json.loads(result_path.read_text(encoding="utf-8"))
assert result_path.stat().st_size <= 2 * 1024 * 1024
assert persisted["error"] == persisted["execution_errors"][:1]
assert persisted["run_config"] == result["run_config"]
run_config_path = Path(result["run_dir"]) / "run_config.json"
assert json.loads(run_config_path.read_text(encoding="utf-8")) == result["run_config"]
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic"])
def test_live_custom_catalog_401_is_inconclusive_without_exposing_secret(provider_name: str) -> None:
requests: list[str] = []
secret = "loopback-provider-secret"
class RejectingCatalog(BaseHTTPRequestHandler):
def do_GET(self) -> None:
requests.append(self.path)
self.send_response(401)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps({"error": f"rejected {secret}"}).encode())
def log_message(self, *_args) -> None:
return None
with ThreadingHTTPServer(("127.0.0.1", 0), RejectingCatalog) as server:
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
provider = ProviderConfig(
provider=provider_name,
model="requested-model",
api_key=secret,
base_url=f"http://127.0.0.1:{server.server_port}/v1",
litellm_model=f"{'anthropic' if provider_name == 'anthropic' else 'openai'}/requested-model",
)
result = runtime_preflight.probe_model(provider)
finally:
server.shutdown()
thread.join(timeout=2)
assert result.ok is False
assert result.failure_kind == "authentication"
assert result.http_status == 401
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
assert secret not in result.detail
assert requests == ["/v1/models"]
def test_agent_only_validation_accepts_visible_agent_artifacts_in_docker_mode(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "opencode.txt").write_text("agent transcript", encoding="utf-8")
assert runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
) == (True, "")
def test_agent_only_validation_accepts_visible_multistep_agent_artifacts_in_docker_mode(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_dir = result_path.parent / "case-001__attempt"
(trial_dir / "result.json").write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": None,
"exception_info": None,
"step_results": [
{
"step_name": "author-skill",
"agent_result": {"n_input_tokens": 100},
"exception_info": None,
}
],
}
),
encoding="utf-8",
)
step_agent_dir = trial_dir / "steps" / "author-skill" / "agent"
step_agent_dir.mkdir(parents=True)
(step_agent_dir / "opencode.txt").write_text("agent transcript", encoding="utf-8")
assert runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
) == (True, "")
def test_agent_only_validation_rejects_invisible_agent_artifacts_in_docker_mode(tmp_path: Path) -> None:
"""An empty host-side agent tree means the daemon never shared the results directory."""
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
# Harbor creates the mount points, but a daemon-side bind source leaves them empty.
(result_path.parent / "case-001__attempt" / "agent" / "setup").mkdir(parents=True)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_agent_only_validation_skips_mount_check_outside_docker_mode(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
(result_path.parent / "case-001__attempt" / "agent" / "setup").mkdir(parents=True)
assert runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="local"
) == (True, "")
def test_agent_only_validation_ignores_empty_files_when_checking_visibility(tmp_path: Path) -> None:
"""A zero-byte placeholder is not evidence that container writes reached the host."""
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "opencode.txt").write_text("", encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def _write_multistep_agent_only_result(jobs_dir: Path, step_name: str) -> Path:
"""Write a completed agent-only job whose single trial has one named step."""
result_path = _write_harbor_0132_unscored_result(jobs_dir)
(result_path.parent / "case-001__attempt" / "result.json").write_text(
json.dumps(
{
"trial_name": "case-001__attempt",
"agent_result": None,
"exception_info": None,
"step_results": [
{
"step_name": step_name,
"agent_result": {"n_input_tokens": 100},
"exception_info": None,
}
],
}
),
encoding="utf-8",
)
return result_path
def test_agent_only_validation_finds_nested_step_agent_artifacts(tmp_path: Path) -> None:
"""Harbor declares StepConfig.name as a bare str, so a step name may nest."""
result_path = _write_multistep_agent_only_result(tmp_path / "jobs", "phase/author")
agent_dir = result_path.parent / "case-001__attempt" / "steps" / "phase" / "author" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "opencode.txt").write_text("agent transcript", encoding="utf-8")
assert runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
) == (True, "")
def test_agent_only_validation_rejects_empty_nested_step_agent_directory(tmp_path: Path) -> None:
result_path = _write_multistep_agent_only_result(tmp_path / "jobs", "phase/author")
(result_path.parent / "case-001__attempt" / "steps" / "phase" / "author" / "agent").mkdir(parents=True)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_agent_only_validation_ignores_step_names_that_escape_the_trial(tmp_path: Path) -> None:
"""A traversing step name must not be joined onto the trial directory."""
result_path = _write_multistep_agent_only_result(tmp_path / "jobs", "../../escape")
outside = tmp_path / "jobs" / "escape" / "agent"
outside.mkdir(parents=True)
(outside / "opencode.txt").write_text("attacker controlled", encoding="utf-8")
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_agent_only_validation_does_not_follow_agent_authored_symlinks(tmp_path: Path) -> None:
"""A link into the host must not be accepted as proof the mount worked."""
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
outside = tmp_path / "outside.log"
outside.write_text("host file the container does not own", encoding="utf-8")
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "claimed.log").symlink_to(outside)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_agent_only_validation_does_not_follow_symlinked_agent_directory(tmp_path: Path) -> None:
"""The walk must reject a link at its root, not just links found below it."""
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
outside_dir = tmp_path / "outside"
outside_dir.mkdir()
(outside_dir / "opencode.txt").write_text("host file the container does not own", encoding="utf-8")
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.symlink_to(outside_dir, target_is_directory=True)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_agent_only_validation_does_not_follow_nested_step_ancestor_symlink(tmp_path: Path) -> None:
"""Each path component leading to a nested step's agent tree is untrusted."""
result_path = _write_multistep_agent_only_result(tmp_path / "jobs", "phase/author")
outside_phase = tmp_path / "outside-phase"
outside_agent = outside_phase / "author" / "agent"
outside_agent.mkdir(parents=True)
(outside_agent / "opencode.txt").write_text("host file the container does not own", encoding="utf-8")
steps_dir = result_path.parent / "case-001__attempt" / "steps"
steps_dir.mkdir()
(steps_dir / "phase").symlink_to(outside_phase, target_is_directory=True)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_visible_artifact_scan_stops_enumerating_at_entry_limit(monkeypatch, tmp_path: Path) -> None:
"""The limit must bound directory enumeration, not only later stat calls."""
agent_dir = tmp_path / "agent"
agent_dir.mkdir()
empty_file = agent_dir / "empty.log"
empty_file.touch()
def hostile_iterdir(_directory: Path):
for _ in range(runtime_preflight._MAX_ARTIFACT_SCAN_ENTRIES + 1):
yield empty_file
raise AssertionError("artifact scan enumerated beyond its declared limit")
monkeypatch.setattr(Path, "iterdir", hostile_iterdir)
assert runtime_preflight._has_visible_artifact(agent_dir, root=tmp_path) is False
def test_agent_only_validation_does_not_follow_symlinked_agent_subdirectories(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
outside_dir = tmp_path / "outside"
outside_dir.mkdir()
(outside_dir / "opencode.txt").write_text("host file the container does not own", encoding="utf-8")
agent_dir = result_path.parent / "case-001__attempt" / "agent"
agent_dir.mkdir(parents=True)
(agent_dir / "sessions").symlink_to(outside_dir, target_is_directory=True)
ok, detail = runtime_preflight.validate_harbor_agent_only_job_result(
result_path, expected_trials=1, env_mode="docker"
)
assert ok is False
assert "not visible to the Docker daemon" in detail
def test_is_vertex_openapi_endpoint():
"""Verify detection of Vertex AI Agent Platform OpenAPI base URLs."""
from skillevaluator.tier3.harbor.runtime_preflight import _is_vertex_openapi_endpoint
assert (
_is_vertex_openapi_endpoint(
"https://aiplatform.googleapis.com/v1beta1/projects/proj-123/locations/global/endpoints/openapi"
)
is True
)
assert (
_is_vertex_openapi_endpoint(
"https://aiplatform.googleapis.com/v1beta1/projects/proj-123/locations/global/endpoints/openapi/"
)
is True
)
assert (
_is_vertex_openapi_endpoint(
"https://us-central1-aiplatform.googleapis.com/v1/projects/proj-123/locations/us-central1/endpoints/openapi"
)
is True
)
assert (
_is_vertex_openapi_endpoint(
"https://europe-west4-aiplatform.googleapis.com/v1beta1/projects/p/locations/europe-west4/endpoints/openapi"
)
is True
)
# Negative cases
assert _is_vertex_openapi_endpoint("https://api.openai.com/v1") is False
assert _is_vertex_openapi_endpoint("https://integrate.api.nvidia.com/v1") is False
assert _is_vertex_openapi_endpoint("https://generativelanguage.googleapis.com/v1beta/openai") is False
assert (
_is_vertex_openapi_endpoint(
"http://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
)
is False
)
assert (
_is_vertex_openapi_endpoint(
"https://evil-aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
)
is False
)
assert (
_is_vertex_openapi_endpoint(
"https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/other"
)
is False
)
assert _is_vertex_openapi_endpoint(None) is False
assert _is_vertex_openapi_endpoint("") is False
def test_parse_vertex_openapi_metadata():
"""Verify extraction of project and location from Vertex OpenAPI base URL."""
from skillevaluator.tier3.harbor.runtime_preflight import _parse_vertex_openapi_metadata
proj, loc = _parse_vertex_openapi_metadata(
"https://aiplatform.googleapis.com/v1beta1/projects/vertical-datum-418119/locations/global/endpoints/openapi"
)
assert proj == "vertical-datum-418119"
assert loc == "global"
proj, loc = _parse_vertex_openapi_metadata(
"https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/endpoints/openapi"
)
assert proj == "my-project"
assert loc == "us-central1"
proj, loc = _parse_vertex_openapi_metadata("https://api.openai.com/v1")
assert proj is None
assert loc is None
def test_vertex_openapi_probe_success_with_api_key(monkeypatch: pytest.MonkeyPatch):
"""Verify successful 1-token probe returns ok=True and disposition=VERIFIED."""
from http import HTTPStatus
captured_requests = []
class MockResponse:
status = HTTPStatus.OK
def __enter__(self):
return self
def __exit__(self, *_args):
pass
def mock_urlopen(req, *_args, **_kwargs):
captured_requests.append(req)
return MockResponse()
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
"mock-token-123",
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is True
assert "available on Vertex AI OpenAPI" in result.detail
assert runtime_preflight.credential_probe_disposition(provider, result) == "verified"
assert len(captured_requests) == 1
req = captured_requests[0]
assert (
req.full_url
== "https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi/chat/completions"
)
assert req.headers["Authorization"] == "Bearer mock-token-123"
body = json.loads(req.data.decode("utf-8"))
assert body["model"] == "google/gemini-3.8-flash"
assert body["max_tokens"] == 1
def test_vertex_openapi_probe_success_falls_back_to_google_adc(monkeypatch: pytest.MonkeyPatch):
"""Verify probe acquires token via _get_google_access_token if provider api_key is None."""
from http import HTTPStatus
captured_requests = []
class MockResponse:
status = HTTPStatus.OK
def __enter__(self):
return self
def __exit__(self, *_args):
pass
def mock_urlopen(req, *_args, **_kwargs):
captured_requests.append(req)
return MockResponse()
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: "adc-token-456")
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
None,
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is True
assert runtime_preflight.credential_probe_disposition(provider, result) == "verified"
assert len(captured_requests) == 1
assert captured_requests[0].headers["Authorization"] == "Bearer adc-token-456"
@pytest.mark.parametrize(
("status_code", "expected_failure_kind", "expected_disposition"),
[
(401, "authentication", "fatal"),
(403, "authorization", "fatal"),
(404, "model_not_found", "fatal"),
(429, "unavailable", "degraded"),
(503, "other_http", "degraded"),
],
)
def test_vertex_openapi_probe_http_errors(
monkeypatch: pytest.MonkeyPatch,
status_code: int,
expected_failure_kind: str,
expected_disposition: str,
):
"""Verify HTTP error classification on Vertex AI OpenAPI probe."""
import io
import urllib.error
def mock_urlopen(*_args, **_kwargs):
raise urllib.error.HTTPError(
"https://aiplatform.googleapis.com/...",
status_code,
"Error",
{},
io.BytesIO(b'{"error": {"message": "Test error detail"}}'),
)
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
"mock-token",
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == expected_failure_kind
assert result.http_status == status_code
assert runtime_preflight.credential_probe_disposition(provider, result) == expected_disposition
def test_vertex_openapi_probe_rejects_redirect(monkeypatch: pytest.MonkeyPatch):
"""Verify redirect on Vertex AI OpenAPI probe fails fast as invalid configuration."""
import io
import urllib.error
def mock_urlopen(*_args, **_kwargs):
raise urllib.error.HTTPError(
"https://aiplatform.googleapis.com/...",
302,
"Found",
{},
io.BytesIO(b"Redirected"),
)
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
"mock-token",
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "invalid_configuration"
assert result.http_status == 302
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
def test_vertex_openapi_probe_missing_token(monkeypatch: pytest.MonkeyPatch):
"""Verify probe fails when no token can be acquired."""
monkeypatch.setattr(runtime_preflight, "_get_google_access_token", lambda **_kwargs: None)
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
None,
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "authentication"
assert runtime_preflight.credential_probe_disposition(provider, result) == "fatal"
assert runtime_preflight.credential_probe_disposition(provider, result, env_mode="gke") == "fatal"
def test_vertex_openapi_probe_timeout(monkeypatch: pytest.MonkeyPatch):
"""Verify timeout during Vertex OpenAPI probe returns unavailable and disposition=DEGRADED."""
def mock_urlopen(*_args, **_kwargs):
raise TimeoutError("Connection timed out")
monkeypatch.setattr(runtime_preflight, "_urlopen_without_redirects", mock_urlopen)
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
"mock-token",
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.probe_model(provider, timeout_seconds=5)
assert result.ok is False
assert result.failure_kind == "unavailable"
assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded"
def test_vertex_openapi_probe_authorization_failure_fatal_in_gke():
"""Verify HTTP 403 on Vertex OpenAPI probe remains FATAL in GKE mode."""
provider = ProviderConfig(
"openai-compatible",
"google/gemini-3.8-flash",
"mock-token",
"https://aiplatform.googleapis.com/v1beta1/projects/test-proj/locations/global/endpoints/openapi",
"openai-compatible/google/gemini-3.8-flash",
)
result = runtime_preflight.ModelProbeResult(
ok=False,
provider=provider.provider,
model=provider.model,
detail="Forbidden: missing aiplatform.endpoints.predict",
failure_kind="authorization",
http_status=403,
)
assert runtime_preflight.credential_probe_disposition(provider, result, env_mode="gke") == "fatal"
assert runtime_preflight.credential_probe_disposition(provider, result, env_mode="") == "fatal"