Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

4.4 KB
Raw Blame
// Why a bench run made no scene tool calls. A run with 0 scene calls is either the machine's fault (infra: claude -p or
// the MCP server never got going) or the model's (it quit after ToolSearch, refused, or answered without tools).
// Infra runs are retried once and kept out of pass@1; model runs stay counted and carry a label.
//
// classifyZeroCalls(output, { exitCode, errors, toolCalls }) returns null when the run made scene calls, else
// { kind: 'infra' | 'model', cause, detail }.
//   infra  start-up-failure      no init event, or no result event, or a non-zero exit without a result
//   infra  mcp-server-failed     the init event lists the 67js server as not connected
//   infra  timeout               the run was killed by the timeout (no result, exit signal)
//   infra  toolsearch-empty      ToolSearch ran but returned no 67js tool, so the model never saw the tools
//   model  toolsearch-then-quit  ToolSearch returned the 67js tools and the model ended without calling one
//   model  refusal               the model declined (stop reason refusal, or a refusal in the result text)
//   model  no-tool-use           the model answered without ToolSearch or any tool call
const MCP_SERVER_NAME = '67js';
const REFUSAL_PATTERN = /\b(i can(?:'|’)?t (?:help|assist|do)|i(?:'|’)m (?:unable|not able) to|i (?:cannot|won(?:'|’)t)|i must decline)\b/i;

function textOf(content) {
  if (typeof content === 'string') return content;
  return (content ?? []).filter(block => block.type === 'text').map(block => block.text).join('');
}

/** Parses the stream-json lines into the facts the classifier needs. */
export function readRunFacts(output) {
  const facts = { initialized: false, serverStatus: null, toolSearchCalls: 0, toolSearchSawSceneTools: false, result: null };
  const toolSearchIds = new Set();
  for (const line of String(output ?? '').split('\n')) {
    if (!line.trim().startsWith('{')) continue;
    let event;
    try {
      event = JSON.parse(line);
    } catch {
      continue;
    }
    if (event.type === 'system' && event.subtype === 'init') {
      facts.initialized = true;
      const server = (event.mcp_servers ?? []).find(candidate => candidate.name === MCP_SERVER_NAME);
      facts.serverStatus = server ? server.status : 'missing';
    }
    if (event.type === 'assistant') {
      for (const block of event.message?.content ?? []) {
        if (block.type === 'tool_use' && block.name === 'ToolSearch') {
          facts.toolSearchCalls++;
          toolSearchIds.add(block.id);
        }
      }
    }
    if (event.type === 'user') {
      for (const block of event.message?.content ?? []) {
        if (block.type === 'tool_result' && toolSearchIds.has(block.tool_use_id) && !block.is_error) {
          if (textOf(block.content).includes(`mcp__${MCP_SERVER_NAME}__`)) facts.toolSearchSawSceneTools = true;
        }
      }
    }
    if (event.type === 'result') facts.result = event;
  }
  return facts;
}

export function classifyZeroCalls(output, { exitCode = 0, errors = '', toolCalls = 0 } = {}) {
  if (toolCalls > 0) return null;
  const facts = readRunFacts(output);
  const stderrNote = String(errors ?? '').trim().split('\n')[0].slice(0, 120);
  if (!facts.initialized) return { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no init event, exit ${exitCode}` };
  if (facts.serverStatus !== 'connected') return { kind: 'infra', cause: 'mcp-server-failed', detail: `67js server ${facts.serverStatus}` };
  if (!facts.result) {
    return exitCode === null || exitCode === undefined
      ? { kind: 'infra', cause: 'timeout', detail: 'killed before a result' }
      : { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no result event, exit ${exitCode}` };
  }
  const resultText = String(facts.result.result ?? '');
  if (facts.result.stop_reason === 'refusal' || (facts.toolSearchCalls === 0 && REFUSAL_PATTERN.test(resultText))) {
    return { kind: 'model', cause: 'refusal', detail: resultText.slice(0, 120) };
  }
  if (facts.toolSearchCalls > 0 && !facts.toolSearchSawSceneTools) {
    return { kind: 'infra', cause: 'toolsearch-empty', detail: `${facts.toolSearchCalls} ToolSearch calls returned no 67js tool` };
  }
  if (facts.toolSearchCalls > 0) return { kind: 'model', cause: 'toolsearch-then-quit', detail: `${facts.toolSearchCalls} ToolSearch calls, then no scene call` };
  return { kind: 'model', cause: 'no-tool-use', detail: resultText.slice(0, 120) };
}