Skip to content

A complete AI agency at your fingertips - From frontend wizards to Reddit community ninjas, from whimsy injectors to reality checkers. Each agent is a specialized expert with personality, processes, and proven deliverables.

Shell

Penalize HTTP errors in tool performance evaluations

Rudy Celekliab776d7fdbb4
diff --git a/testing/testing-tool-evaluator.md b/testing/testing-tool-evaluator.md
index 3f8a9ff6..3a549433 100644
--- a/testing/testing-tool-evaluator.md
+++ b/testing/testing-tool-evaluator.md
@@ -195,14 +195,17 @@ class ToolEvaluator:
# Response time testing
response_times = []
+ failed_requests = 0
for _ in range(10):
start_time = time.time()
try:
response = requests.get(api_endpoint, timeout=10)
+ response.raise_for_status()
end_time = time.time()
response_times.append(end_time - start_time)
except requests.RequestException:
- response_times.append(10.0) # Timeout penalty
+ failed_requests += 1
+ response_times.append(10.0) # Network/HTTP failure penalty
avg_response_time = np.mean(response_times)
p95_response_time = np.percentile(response_times, 95)
@@ -219,7 +222,8 @@ class ToolEvaluator:
else:
speed_score = 2
- notes = f"Avg: {avg_response_time:.2f}s, P95: {p95_response_time:.2f}s"
+ notes = (f"Penalty-adjusted avg: {avg_response_time:.2f}s, P95: {p95_response_time:.2f}s; "
+ f"failed requests: {failed_requests}/{len(response_times)}")
return speed_score, notes
def calculate_total_cost_ownership(self, tool_config: Dict, years: int = 3) -> Dict: