AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { LiveScorecard, ScenarioScore } from './tagger'
// Render the scorecard as a scannable markdown report: headline metrics, a per-shape
// breakdown (which input kinds the harness handles vs. fumbles), and a per-scenario table.
function toMarkdown({ scorecard, label, mode, modelId }: { scorecard: LiveScorecard, label: string, mode: string, modelId: string }): string {
const discovery = mode === 'discovery'
const succeededLabel = discovery ? 'reached a runnable call' : 'succeeded (goal met)'
const gaveUpLabel = discovery ? 'stuck in discovery (never reached a call)' : 'gave up (expected exec, none succeeded)'
const lines: string[] = []
lines.push(`# Chat harness — failure-mode scorecard (${label})`)
lines.push('')
lines.push(`Model: \`${modelId}\` · mode: \`${mode}\` · scenarios: ${scorecard.scenarioCount}`)
if (discovery) {
lines.push('')
lines.push('> Discovery-only run: `ap_execute_action` was neutralized (no side effects). "reached a runnable call" = the agent navigated discovery to a well-formed execute; external-API success is not measured here.')
}
lines.push('')
lines.push('## Headline')
lines.push('')
lines.push('| metric | value |')
lines.push('| --- | --- |')
lines.push(`| reached/executed a call | ${scorecard.executedCount}/${scorecard.scenarioCount} |`)
lines.push(`| ${succeededLabel} | ${scorecard.succeededCount}/${scorecard.scenarioCount} |`)
lines.push(`| blocked on a missing connection (not a failure) | ${scorecard.blockedOnConnectionCount} |`)
lines.push(`| ${gaveUpLabel} | ${scorecard.gaveUpCount} |`)
lines.push(`| avg tool calls / scenario | ${scorecard.avgToolCalls} |`)
lines.push(`| avg discovery calls / scenario | ${scorecard.avgDiscoveryCalls} |`)
lines.push(`| avg hops before first execute | ${fmt(scorecard.avgHopsBeforeExecute)} |`)
lines.push(`| bad-arg rejections (total) | ${scorecard.totalBadArgRejections} |`)
lines.push(`| auth/connection blocked (total) | ${scorecard.totalAuthBlocked} |`)
lines.push(`| other tool errors (total) | ${scorecard.totalOtherErrors} |`)
lines.push(`| breaker hits (✋, total) | ${scorecard.totalBreakerHits} |`)
lines.push(`| schema re-fetches "forgot" (total) | ${scorecard.totalSchemaRefetches} |`)
lines.push(`| wrong instrument (find vs list) | ${scorecard.wrongInstrumentCount}/${scorecard.rightInstrumentGraded} graded |`)
lines.push(`| native task handled (HTTP/code) | ${scorecard.nativeHandledCount}/${scorecard.nativeGraded} graded |`)
lines.push('')
lines.push('## By input shape')
lines.push('')
lines.push('| shape | n | succeeded | avg hops | bad-args | breaker | re-fetch |')
lines.push('| --- | --- | --- | --- | --- | --- | --- |')
for (const [shape, group] of groupByShape(scorecard.scores)) {
const execHops = group.filter((s) => s.hopsBeforeFirstExecute !== null)
const avgHops = execHops.length > 0 ? round(execHops.reduce((a, s) => a + (s.hopsBeforeFirstExecute ?? 0), 0) / execHops.length) : null
lines.push(`| ${shape} | ${group.length} | ${group.filter((s) => s.executeSucceeded).length} | ${fmt(avgHops)} | ${sum(group, (s) => s.badArgRejections)} | ${sum(group, (s) => s.breakerHits)} | ${sum(group, (s) => s.schemaRefetches)} |`)
}
lines.push('')
lines.push('## Per scenario')
lines.push('')
lines.push('| scenario | shape | calls | hops | ok | bad-arg | auth | ✋ | re-fetch | tool sequence |')
lines.push('| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |')
for (const s of scorecard.scores) {
lines.push(`| ${s.scenarioId} | ${s.shape} | ${s.totalToolCalls} | ${fmt(s.hopsBeforeFirstExecute)} | ${outcomeGlyph(s.outcome)} | ${s.badArgRejections} | ${s.authBlocked} | ${s.breakerHits} | ${s.schemaRefetches} | ${truncate(s.toolSequence.join(' → '), 90)} |`)
}
lines.push('')
return lines.join('\n')
}
function outcomeGlyph(outcome: string): string {
if (outcome === 'did-work') return '✅'
if (outcome === 'blocked-connection') return '🔌'
return '🚫'
}
function groupByShape(scores: ScenarioScore[]): Array<[string, ScenarioScore[]]> {
const map = new Map<string, ScenarioScore[]>()
for (const score of scores) {
const group = map.get(score.shape) ?? []
group.push(score)
map.set(score.shape, group)
}
return [...map.entries()]
}
function sum(scores: ScenarioScore[], pick: (s: ScenarioScore) => number): number {
return scores.reduce((acc, s) => acc + pick(s), 0)
}
function round(value: number): number {
return Math.round(value * 100) / 100
}
function fmt(value: number | null): string {
return value === null ? 'n/a' : String(value)
}
function truncate(text: string, max: number): string {
return text.length > max ? `${text.slice(0, max)}…` : text
}
export const liveReport = {
toMarkdown,
}