Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

46.1 KB
Raw Blame
import { mkdir, readFile, readdir, rm } from 'fs/promises';
import { atomicWrite } from './internal/atomicWrite.js';
import { evaluateSecretEndpoint } from './endpointGuard.js';
import { existsSync } from 'fs';
import { join, extname, basename, isAbsolute } from 'path';
import { spawn, ChildProcess } from 'child_process';
import { randomUUID } from 'crypto';
import { analyzeError, ERROR_CATEGORIES } from './errorDetection.js';
import { apiGenerationOptions } from './internal/generationOptions.js';
import { buildOllamaNativeChatBody, ollamaNativeChatUrl, ollamaNativeFrameToChunk } from './internal/ollamaNativeChat.js';
import { describeTransportError, fetchWithPreHeaderRetry, isReplaySafeLocalRequest } from './internal/preHeaderRetry.js';
// The two streaming ceilings (no-progress bound, absolute runtime cap) live in
// a leaf module so a host's own backstop timer can read the absolute bound
// without dragging this file's fs/child_process closure with it.
import { DEFAULT_API_RUN_TIMEOUT_MS, apiRunAbsoluteTimeoutMs } from './internal/runTimeouts.js';
// undici arms its own 300s header/body ceilings under every fetch, which raced
// — and beat — the two above. The streaming request goes through a dispatcher
// with those disabled so the run's declared bounds are the only ones.
import { streamTransportDispatcher } from './internal/streamTransport.js';
import { createRunLifecycle } from './internal/runLifecycle.js';
import { createRunFinalizer } from './internal/runFinalizer.js';
import { resolveWindowsExecutable, prepareWindowsSafeSpawn } from './internal/windowsSafeSpawn.js';

// Windows process-tree teardown stays local; command preparation uses the shared leaf.
const IS_WIN32 = process.platform === 'win32';

const DEFAULT_OUTPUT_RESERVE_TOKENS = 8000;
const CHARS_PER_TOKEN = 4;

function deriveRequestCapabilities({ prompt, screenshots, requestCapabilities }) {
  if (requestCapabilities && typeof requestCapabilities === 'object') return requestCapabilities;
  return {
    requiredContextTokens: Math.ceil(String(prompt ?? '').length / CHARS_PER_TOKEN) + DEFAULT_OUTPUT_RESERVE_TOKENS,
    hasImages: Array.isArray(screenshots) && screenshots.length > 0,
  };
}

// On Windows, taskkill (used below) runs in a separate detached process that
// never touches the original ChildProcess object — `.killed` is set
// synchronously here (mirroring what Node's own `child.kill()` does on the
// POSIX branch) so callers elsewhere that gate re-entrant kill handling on
// `.killed` actually engage on Windows. A plain SIGTERM kills the cmd.exe
// wrapper but orphans the real child, so the whole process tree is taken
// down via `taskkill /T /F` there; SIGTERM is sufficient elsewhere.
//
// The win32 branch is gated on `instanceof ChildProcess`: stopRun/deleteRun
// below call this with whatever was registered via registerExternalRun,
// which can be a node-pty `IPty` TUI session (server/services/tuiPromptRunner.js,
// via the host's registerActiveRun) — it also exposes `.kill()`/`.pid`, but
// a raw `taskkill` against its pid bypasses node-pty's own Windows teardown
// (releasing its native ConPTY handle), leaking it. Any non-ChildProcess
// killable always uses its own `.kill()` instead, on every platform.
//
// On Windows node-pty additionally throws `Signals not supported on windows.`
// for any signal argument — so a signalled kill there killed nothing and threw
// past this function, and stopping a TUI run silently left it running. The
// signal is therefore offered first and the signal-free form used only for a
// handle that refuses it (retrying is harmless — the first attempt did nothing);
// dropping it unconditionally would downgrade other signal-forwarding killables.
// Mirrors server/lib/bufferedSpawn.js (this directory stays self-contained).
function killProcessTree(child) {
  const isChildProcess = child instanceof ChildProcess;
  if (IS_WIN32 && child.pid && isChildProcess) {
    child.killed = true;
    spawn('taskkill', ['/T', '/F', '/PID', String(child.pid)], { stdio: 'ignore', windowsHide: true })
      .on('error', () => {})
      .unref();
  } else if (IS_WIN32 && !isChildProcess) {
    try { child.kill('SIGTERM'); }
    catch { child.kill(); }
  } else {
    child.kill('SIGTERM');
  }
}

// Invoke a completion hook / callback so a throw is logged but never
// propagates — these run at out-of-request boundaries where an uncaught throw
// is an unhandled rejection that crashes the process. Each call is isolated so
// a throwing hook can't prevent a later `onComplete` from settling the caller.
// Handles both a synchronous throw AND a rejected promise from an async hook —
// the latter would otherwise surface as an unhandled rejection.
function safeSettle(fn, label) {
  try {
    const result = fn();
    if (result && typeof result.then === 'function') {
      result.catch(err => console.error(`❌ ${label} threw during recovery: ${err.message}`));
    }
  } catch (err) {
    console.error(`❌ ${label} threw during recovery: ${err.message}`);
  }
}

export function createRunnerService(config = {}) {
  const {
    dataDir = './data',
    runsDir = 'runs',
    screenshotsDir = './data/screenshots',
    providerService,
    providerStatusService = null,
    hooks = {},
    // Host-owned file-plus-record boundary; standalone toolkit has no coordinator.
    withAssetPublication = work => work(),
    maxConcurrentRuns: _maxConcurrentRuns = 5
  } = config;

  const RUNS_PATH = join(dataDir, runsDir);
  const activeRuns = new Map();

  // A run id arrives from an HTTP route param (`DELETE /api/runs/:id`), and
  // Express decodes `%2F`, so `..%2Fbrain` would otherwise `join` out of
  // RUNS_PATH and `rm -r` a sibling data directory. Minted ids are UUIDs; accept
  // only a single plain segment and treat anything else as "no such run".
  const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
  const existingRunDir = (runId) => {
    if (typeof runId !== 'string' || !SAFE_RUN_ID.test(runId)) return null;
    const runDir = join(RUNS_PATH, runId);
    return existsSync(runDir) ? runDir : null;
  };

  // ── Declared extension points ────────────────────────────────────────────
  // PortOS supplies a CLI runner that knows per-CLI argv conventions (Codex
  // `exec -`, Antigravity `agy --print`, Claude Code `-p -`) and a TUI runner
  // the toolkit has no built-in for. Rather than reaching into private props
  // from server/index.js, the host registers them via setCliRunner/setTuiRunner.
  // Externally-spawned runs (the host owns their child process / pty) are tracked
  // in `externalRuns` so the toolkit's stopRun/isRunActive/deleteRun see them.
  // See server/lib/aiToolkit/AGENTS.md (override contract).
  let cliRunnerOverride = null;
  const externalRuns = new Map();
  // A host runner sees only the child/PTY exit event. Remember when that exit
  // was initiated through stopRun so it can distinguish a user cancellation
  // from an unexpected provider crash and avoid fallback/bench telemetry.
  const externalStopRequests = new Set();
  // The same one-shot marker for the toolkit's OWN in-flight runs (`activeRuns`
  // — an API fetch's AbortController). `controller.abort()` with no reason
  // surfaces to the reader as Node's bare `AbortError: This operation was
  // aborted`, which no error pattern matches and which is indistinguishable
  // from a provider transport fault. Without this marker every Stop of an API
  // run finalized as an UNKNOWN provider failure, fired `onRunFailed`, and was
  // escalated to a tier-4 investigation task — a post-mortem over a human
  // pressing Stop. CLI/TUI runs already avoid that via `externalStopRequests`.
  const activeStopRequests = new Set();
  const consumeActiveStop = (runId) => {
    const requested = activeStopRequests.has(runId);
    activeStopRequests.delete(runId);
    return requested;
  };

  async function ensureRunsDir() {
    if (!existsSync(RUNS_PATH)) {
      await mkdir(RUNS_PATH, { recursive: true });
    }
  }

  function getMimeType(filepath) {
    const ext = extname(filepath).toLowerCase();
    const mimeTypes = {
      '.png': 'image/png',
      '.jpg': 'image/jpeg',
      '.jpeg': 'image/jpeg',
      '.gif': 'image/gif',
      '.webp': 'image/webp'
    };
    return mimeTypes[ext] || 'image/png';
  }

  async function loadImageAsBase64(imagePath) {
    if (typeof imagePath !== 'string' || !imagePath) {
      throw new Error(`Invalid screenshot path: ${imagePath}`);
    }
    // Relative references are anchored under screenshotsDir with `basename`
    // applied, so a `../`-traversal collapses to a bare filename and can't
    // escape the screenshots dir. Absolute paths come only from trusted
    // in-process callers (e.g. PortOS's Universe Builder, whose
    // `resolveImageSources` has already validated them against an approved
    // image root) and pass through unchanged. The untrusted POST /api/runs
    // surface is additionally sanitized at the route boundary
    // (sanitizeScreenshotRefs in routes/runs.js) so an attacker-supplied
    // absolute path never reaches this loader. See issue #1870 / #1820.
    const fullPath = isAbsolute(imagePath)
      ? imagePath
      : join(screenshotsDir, basename(imagePath));

    if (!existsSync(fullPath)) {
      throw new Error(`Image not found: ${fullPath}`);
    }

    const buffer = await readFile(fullPath);
    const mimeType = getMimeType(fullPath);
    return `data:${mimeType};base64,${buffer.toString('base64')}`;
  }

  function safeJsonParse(str, fallback = {}) {
    if (typeof str !== 'string' || !str.trim()) {
      return fallback;
    }

    try {
      const parsed = JSON.parse(str);
      return parsed ?? fallback;
    } catch {
      return fallback;
    }
  }

  async function handleProviderError(providerId, errorAnalysis, output) {
    hooks.onProviderError?.(providerId, errorAnalysis, output);

    if (providerStatusService) {
      if (errorAnalysis.category === ERROR_CATEGORIES.USAGE_LIMIT &&
          errorAnalysis.requiresFallback &&
          typeof providerStatusService.markUsageLimit === 'function') {
        await providerStatusService.markUsageLimit(providerId, {
          message: errorAnalysis.message,
          waitTime: errorAnalysis.waitTime,
          rateLimitWindow: errorAnalysis.rateLimitWindow,
        }).catch(err => {
          console.error(`❌ Failed to mark provider usage limit: ${err.message}`);
        });
      } else if (errorAnalysis.category === ERROR_CATEGORIES.RATE_LIMIT &&
                 typeof providerStatusService.markRateLimited === 'function') {
        await providerStatusService.markRateLimited(providerId, {
          rateLimitWindow: errorAnalysis.rateLimitWindow,
        }).catch(err => {
          console.error(`❌ Failed to mark provider rate limited: ${err.message}`);
        });
      }
    }
  }

  const service = {
    // ── Extension-point setters / external-run registry ──────────────────
    // Register a host CLI runner. Pass `null` to revert to the toolkit's
    // built-in executeCliRun. The override receives the same args the built-in
    // would and must return the runId.
    setCliRunner(fn) {
      if (fn != null && typeof fn !== 'function') {
        throw new Error('setCliRunner expects a function or null');
      }
      cliRunnerOverride = fn;
    },
    // Register a host TUI runner. The toolkit has no built-in TUI executor, so
    // the runs router gates on `typeof runnerService.executeTuiRun === 'function'`.
    // Attaching/detaching the method keeps that gate honest.
    setTuiRunner(fn) {
      if (fn != null && typeof fn !== 'function') {
        throw new Error('setTuiRunner expects a function or null');
      }
      if (fn) service.executeTuiRun = fn;
      else delete service.executeTuiRun;
    },
    // Track a host-spawned run's killable handle (ChildProcess or node-pty IPty —
    // both expose `.kill(signal?)`; AbortController exposes `.abort()`) so
    // stopRun/isRunActive/deleteRun account for it alongside the toolkit's own.
    registerExternalRun(runId, killable) {
      externalStopRequests.delete(runId);
      externalRuns.set(runId, killable);
    },
    unregisterExternalRun(runId) {
      externalRuns.delete(runId);
      externalStopRequests.delete(runId);
    },
    hasExternalRun(runId) { return externalRuns.has(runId); },
    consumeExternalRunStop(runId) {
      const requested = externalStopRequests.has(runId);
      externalStopRequests.delete(runId);
      return requested;
    },

    async createRun(options) {
      const {
        providerId,
        model,
        prompt,
        workspacePath = process.cwd(),
        workspaceName = 'default',
        timeout,
        source = 'devtools',
        fallbackProviderId = null,
        effort = null,
        requestCapabilities = null,
        screenshots = [],
        allowFallback = true,
      } = options;

      if (!providerService) {
        throw new Error('Provider service not configured');
      }

      let effectiveProviderId = providerId;
      let usedFallback = false;
      // Model hint to run on the fallback provider when we proactively swap
      // below. Stays null on the non-fallback path so the requested `model`
      // wins as before.
      let fallbackModelHint = null;
      const effectiveRequestCapabilities = deriveRequestCapabilities({
        prompt,
        screenshots,
        requestCapabilities,
      });

      if (providerStatusService && !providerStatusService.isAvailable(providerId)) {
        if (!allowFallback) {
          const status = providerStatusService.getStatus(providerId) || {};
          throw new Error(`Provider ${providerId} is unavailable (${status.reason || 'unknown'}) and fallback is disabled`);
        }
        const allProviders = await providerService.getAllProviders();
        const providersMap = {};
        for (const p of allProviders.providers) {
          providersMap[p.id] = p;
        }

        const fallback = providerStatusService.getFallbackProvider(
          providerId,
          providersMap,
          fallbackProviderId,
          null,
          effectiveRequestCapabilities
        );

        if (fallback) {
          effectiveProviderId = fallback.provider.id;
          usedFallback = true;
          fallbackModelHint = fallback.model || null;
          // Name the benched PRIMARY and why it was skipped, not just the
          // fallback — otherwise the log reads as an unexplained provider switch
          // ("why is my enabled provider not being used?"). `enabled` is the user
          // toggle; this swap is driven by the separate runtime availability state.
          const benched = providerStatusService.getStatus(providerId) || {};
          const recovery = providerStatusService.getTimeUntilRecovery(providerId);
          const primaryName = providersMap[providerId]?.name || providerId;
          console.log(`⚡ Primary ${primaryName} unavailable (${benched.reason || 'unknown'}${recovery ? `, recovers in ${recovery}` : ''}) — using fallback ${fallback.provider.name} (source: ${fallback.source})`);
        } else {
          const timeUntilRecovery = providerStatusService.getTimeUntilRecovery(providerId);
          throw new Error(
            `Provider ${providerId} is unavailable (${providerStatusService.getStatus(providerId).reason}) ` +
            `and no fallback is available. Recovery in: ${timeUntilRecovery || 'unknown'}`
          );
        }
      }

      const provider = await providerService.getProviderById(effectiveProviderId);
      if (!provider) {
        throw new Error('Provider not found');
      }

      if (!provider.enabled) {
        throw new Error('Provider is disabled');
      }

      await ensureRunsDir();

      const runId = randomUUID();
      const runDir = join(RUNS_PATH, runId);
      await mkdir(runDir);

      // On fallback, the requested `model` was resolved against the PRIMARY
      // provider and almost never exists on the fallback — use the configured
      // fallback model (or the fallback provider's own default) instead of
      // leaking the primary's model id onto the fallback's record + log line.
      const recordModel = usedFallback
        ? (fallbackModelHint || provider.defaultModel || null)
        : (model || provider.defaultModel || null);

      const metadata = {
        id: runId,
        type: 'ai',
        providerId: effectiveProviderId,
        providerName: provider.name,
        originalProviderId: usedFallback ? providerId : null,
        usedFallback,
        model: recordModel,
        effort,
        workspacePath,
        workspaceName,
        source,
        prompt: prompt.substring(0, 500),
        // Full prompt size (chars) — `prompt` above is truncated for display.
        // Hosts use this to estimate input-token usage on completion.
        promptLength: prompt.length,
        startTime: new Date().toISOString(),
        endTime: null,
        duration: null,
        exitCode: null,
        success: null,
        error: null,
        errorCategory: null,
        errorAnalysis: null,
        outputSize: 0
      };

      await withAssetPublication(async () => {
        // The record must never name missing prompt/output bytes after a failure.
        await atomicWrite(join(runDir, 'prompt.txt'), prompt);
        await atomicWrite(join(runDir, 'output.txt'), '');
        await atomicWrite(join(runDir, 'metadata.json'), metadata);
      });

      hooks.onRunCreated?.(metadata);
      console.log(`🤖 AI run [${source}]: ${provider.name}/${metadata.model}`);

      const effectiveTimeout = timeout ?? provider.timeout;

      // Surface `fallbackModel` so callers that re-resolve the model against
      // the fallback provider (e.g. stageRunner's args-baked-model logic) use
      // the configured fallback model instead of the primary's leaked one.
      return { runId, runDir, provider, metadata, timeout: effectiveTimeout, usedFallback, fallbackModel: fallbackModelHint };
    },

    async executeCliRun(opts) {
      // A host-registered CLI runner (see setCliRunner) wins — it tracks its own
      // child process via registerExternalRun, so stopRun/isRunActive/deleteRun
      // still account for it.
      if (cliRunnerOverride) return cliRunnerOverride(opts);
      const { runId, provider, prompt, workspacePath, onData, onComplete, timeout, beforeExecute } = opts;
      const runDir = join(RUNS_PATH, runId);
      const outputPath = join(runDir, 'output.txt');
      const metadataPath = join(runDir, 'metadata.json');

      const startTime = Date.now();
      let output = '';

      // Pass the prompt via stdin (not argv) and run without a shell so that
      // user-configurable `provider.command` cannot inject extra commands via
      // shell metacharacters, and so the full prompt isn't visible in
      // process listings as a single command-line argument. On Windows,
      // resolve+wrap a .cmd/.bat target instead of enabling a shell — see
      // internal/windowsSafeSpawn.js for why shell:true is unsafe here.
      const args = [...(provider.args || [])];
      console.log(`🚀 Executing CLI: ${provider.command} ${args.join(' ')} (${prompt.length} chars via stdin)`);

      // Pin PWD to the spawn cwd — `spawn({ cwd })` does not rewrite the
      // inherited PWD, and OpenCode resolves its project root as
      // `process.env.PWD ?? process.cwd()`, so it would run wherever the host
      // process was started instead of in `workspacePath`. Deliberately mirrors
      // PortOS's `withSpawnCwdEnv` line for line rather than importing it: this
      // toolkit is vendored and must not import out to other PortOS modules.
      // (PortOS overrides `executeCliRun` via `setCliRunner`, so this particular
      // spawn is dormant here — the copy keeps the vendored toolkit correct
      // standalone.) The case-insensitive delete matters on Windows, where a
      // spread of `process.env` can surface the variable as `Pwd`/`pwd`.
      const childEnv = { ...process.env, ...provider.envVars };
      if (workspacePath) {
        for (const key of Object.keys(childEnv)) if (/^pwd$/i.test(key)) delete childEnv[key];
        childEnv.PWD = workspacePath;
      }
      // Resolved against `childEnv` (not bare process.env) so a
      // provider-configured PATH override is honored.
      const resolvedCommand = resolveWindowsExecutable(provider.command, undefined, childEnv) || provider.command;
      const { command: spawnCommand, args: spawnArgs } = prepareWindowsSafeSpawn(resolvedCommand, args, undefined, childEnv);
      if (beforeExecute) {
        try { await beforeExecute(); }
        catch {
          const metadata = { ...safeJsonParse(await readFile(metadataPath, 'utf8').catch(() => '{}')),
            endTime: new Date().toISOString(), duration: Date.now() - startTime,
            success: false, canceled: true, completionReason: 'canceled', error: 'CLI run canceled', errorCategory: ERROR_CATEGORIES.CANCELED };
          await atomicWrite(metadataPath, metadata);
          safeSettle(() => hooks.onRunCanceled?.({ runId }), `Run ${runId} onRunCanceled hook`);
          safeSettle(() => onComplete?.(metadata), `Run ${runId} onComplete`);
          return runId;
        }
      }
      const childProcess = spawn(spawnCommand, spawnArgs, {
        cwd: workspacePath,
        env: childEnv,
        windowsHide: true
      });

      // Same tick as spawn. Node emits 'error' (ENOENT, EACCES, EMFILE) and
      // does not follow it with 'close' when the process never starts; an
      // unlistened 'error' is an uncaught exception that exits the server.
      // A child that dies before reading stdin emits EPIPE on the pipe — that
      // stream error is the same crash if nobody is listening. 'close' and
      // 'error' share one settlement so a race cannot double-fire hooks.
      let settled = false;
      let timeoutHandle = null;
      const releaseRun = () => {
        if (timeoutHandle) clearTimeout(timeoutHandle);
        activeRuns.delete(runId);
      };

      childProcess.stdin?.on('error', () => {});
      childProcess.on('error', (err) => {
        if (settled) return;
        settled = true;
        releaseRun();
        const message = err?.message || 'CLI spawn failed';
        console.error(`❌ Run ${runId} spawn error: ${message}`);
        void (async () => {
          // The 'error' event is the classification: a missing binary's message
          // matches SPAWN_ERROR, but EACCES/EMFILE do not, and a null category
          // is read as an unknown provider failure. Keep the raw message — the
          // pattern's extracted text drops the spawn detail callers match on.
          const errorAnalysis = analyzeError(message, -1);
          errorAnalysis.category = ERROR_CATEGORIES.SPAWN_ERROR;
          errorAnalysis.hasError = true;
          errorAnalysis.requiresFallback = true;
          errorAnalysis.message = message;
          const failMetadata = {
            endTime: new Date().toISOString(),
            duration: Date.now() - startTime,
            exitCode: -1,
            success: false,
            error: message,
            errorCategory: ERROR_CATEGORIES.SPAWN_ERROR,
            errorAnalysis,
            outputSize: Buffer.byteLength(output),
          };
          try {
            let metadata = safeJsonParse(await readFile(metadataPath, 'utf-8').catch(() => '{}'));
            if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) metadata = {};
            Object.assign(metadata, failMetadata);
            await withAssetPublication(async () => {
              await atomicWrite(outputPath, output);
              await atomicWrite(metadataPath, metadata);
            });
            Object.assign(failMetadata, metadata);
          } catch (writeErr) {
            console.error(`❌ Run ${runId} spawn-error finalization failed: ${writeErr.message}`);
          }
          safeSettle(() => hooks.onRunFailed?.(failMetadata, failMetadata.error, output), `Run ${runId} onRunFailed hook`);
          safeSettle(() => onComplete?.(failMetadata), `Run ${runId} onComplete`);
        })();
      });

      if (childProcess.stdin) {
        childProcess.stdin.write(prompt);
        childProcess.stdin.end();
      }

      activeRuns.set(runId, childProcess);
      hooks.onRunStarted?.({ runId, provider: provider.name, model: provider.defaultModel });

      if (timeout > 0) timeoutHandle = setTimeout(() => {
        if (childProcess && !childProcess.killed) {
          console.log(`⏱️ Run ${runId} timed out after ${timeout}ms`);
          killProcessTree(childProcess);
        }
      }, timeout);

      childProcess.stdout?.on('data', (data) => {
        const text = data.toString();
        output += text;
        onData?.(text);
      });

      childProcess.stderr?.on('data', (data) => {
        const text = data.toString();
        output += text;
        onData?.(text);
      });

      childProcess.on('close', async (code) => {
        if (settled) return;
        settled = true;
        // Runs outside the request lifecycle — an uncaught throw from
        // atomicWrite/handleProviderError/hooks would surface as an unhandled
        // rejection and crash the process, so guard the body and still settle
        // the caller on failure.
        try {
          releaseRun();

          const metadata = await withAssetPublication(async () => {
            await atomicWrite(outputPath, output);
            const metadata = safeJsonParse(await readFile(metadataPath, 'utf-8').catch(() => '{}'));
            metadata.endTime = new Date().toISOString();
            metadata.duration = Date.now() - startTime;
            metadata.exitCode = code;
            metadata.success = code === 0;
            metadata.outputSize = Buffer.byteLength(output);
            if (!metadata.success) {
              const errorAnalysis = analyzeError(output, code, { prompt });
              metadata.error = errorAnalysis.message || `Process exited with code ${code}`;
              metadata.errorCategory = errorAnalysis.category;
              metadata.errorAnalysis = errorAnalysis;
            }
            await atomicWrite(metadataPath, metadata);
            return metadata;
          });
          const errorAnalysis = metadata.errorAnalysis;
          if (errorAnalysis?.hasError &&
              (errorAnalysis.category === ERROR_CATEGORIES.RATE_LIMIT ||
               errorAnalysis.category === ERROR_CATEGORIES.USAGE_LIMIT)) {
            await handleProviderError(provider.id, errorAnalysis, output);
          }

          // Isolate the completion hooks + onComplete from the outer catch — a
          // throwing onRunCompleted must NOT be reinterpreted as a finalization
          // failure that flips a successful run to success:false for the caller.
          if (metadata.success) {
            safeSettle(() => hooks.onRunCompleted?.(metadata, output), `Run ${runId} onRunCompleted hook`);
          } else {
            safeSettle(() => hooks.onRunFailed?.(metadata, metadata.error, output), `Run ${runId} onRunFailed hook`);
          }
          safeSettle(() => onComplete?.(metadata), `Run ${runId} onComplete`);
        } catch (err) {
          console.error(`❌ Run ${runId} close handler error: ${err.message}`);
          const failMetadata = {
            endTime: new Date().toISOString(),
            duration: Date.now() - startTime,
            exitCode: code,
            success: false,
            error: `Run finalization failed: ${err.message}`,
            outputSize: Buffer.byteLength(output),
          };
          safeSettle(() => hooks.onRunFailed?.(failMetadata, failMetadata.error, output), `Run ${runId} onRunFailed hook`);
          safeSettle(() => onComplete?.(failMetadata), `Run ${runId} onComplete`);
        }
      });

      return runId;
    },

    async executeApiRun({ runId, provider, model, prompt, workspacePath, screenshots, onData, onComplete, timeout, absoluteTimeoutMs, maxTokens, beforeExecute, onSettled, onPersistenceFailure }) {
      const runDir = join(RUNS_PATH, runId);
      const outputPath = join(runDir, 'output.txt');
      const metadataPath = join(runDir, 'metadata.json');

      const startTime = Date.now();
      let output = '';
      let reasoning = '';

      const headers = {
        'Content-Type': 'application/json'
      };
      if (provider.apiKey) {
        headers['Authorization'] = `Bearer ${provider.apiKey}`;
      }

      const controller = new AbortController();
      activeStopRequests.delete(runId);
      activeRuns.set(runId, controller);

      const stallTimeout = timeout ?? provider.timeout ?? DEFAULT_API_RUN_TIMEOUT_MS;
      const absoluteTimeout = apiRunAbsoluteTimeoutMs(stallTimeout, absoluteTimeoutMs);
      let finalizer;
      const lifecycle = createRunLifecycle({
        runId,
        controller,
        stallTimeout,
        absoluteTimeout,
        onTimeout: bound => finalizer.finalize({ type: 'timeout', bound }),
      });
      finalizer = createRunFinalizer({
        withAssetPublication,
        runId,
        provider,
        startTime,
        activeRuns,
        lifecycle,
        stallTimeout,
        absoluteTimeout,
        callerRuntimeBudget: absoluteTimeout < apiRunAbsoluteTimeoutMs(stallTimeout),
        outputPath,
        metadataPath,
        getOutput: () => output,
        getReasoning: () => reasoning,
        providerStatusService,
        hooks,
        onComplete,
        onPersistenceFailure,
        handleProviderError,
        safeJsonParse,
        safeSettle,
        consumeActiveStop,
      });
      lifecycle.start();

      hooks.onRunStarted?.({ runId, provider: provider.name, model });

      let messageContent;
      if (screenshots && screenshots.length > 0) {
        console.log(`📸 Loading ${screenshots.length} screenshots for vision API`);
        const contentParts = [];

        for (const screenshotPath of screenshots) {
          const imageDataUrl = await loadImageAsBase64(screenshotPath).catch(err => {
            console.error(`❌ Failed to load screenshot ${screenshotPath}: ${err.message}`);
            return null;
          });
          if (imageDataUrl) {
            contentParts.push({
              type: 'image_url',
              image_url: { url: imageDataUrl }
            });
          }
        }

        contentParts.push({ type: 'text', text: prompt });
        messageContent = contentParts;
      } else {
        messageContent = prompt;
      }

      // Never send the API key to an arbitrary/metadata host (SSRF / key
      // exfiltration). Keyless local-LLM runs are unaffected.
      const endpointGuard = provider.apiKey
        ? evaluateSecretEndpoint(provider.endpoint, { allowCustomEndpoint: provider.allowCustomEndpoint === true })
        : { allowed: true, reason: null };

      const ensureProviderReady = hooks.ensureProviderReady || (async () => ({ success: true }));
      const ready = endpointGuard.allowed
        ? await ensureProviderReady(provider).catch((err) => ({ success: false, error: err.message }))
        : { success: false, error: `Endpoint blocked: ${endpointGuard.reason}` };
      if (controller.signal.aborted) {
        await finalizer.finalize({ type: 'canceled' });
        await finalizer.settled();
        onSettled?.();
        return runId;
      }
      if (ready.success && beforeExecute) {
        try { await beforeExecute(); }
        catch {
          await finalizer.finalize({ type: 'canceled' });
          await finalizer.settled();
          onSettled?.();
          return runId;
        }
      }
      // An Ollama provider with a `numCtx` goes to the native `/api/chat`:
      // `/v1/chat/completions` cannot carry a context window (it ignores
      // `num_ctx` in any position), so the model would load at the daemon
      // default and silently drop the prompt's head. See
      // ./internal/ollamaNativeChat.js.
      const nativeOllamaUrl = ollamaNativeChatUrl(provider);
      const requestModel = model || provider.defaultModel;
      const requestBody = nativeOllamaUrl
        ? buildOllamaNativeChatBody({ model: requestModel, messageContent, provider, generation: apiGenerationOptions(provider), maxTokens })
        : {
            model: requestModel,
            messages: [{ role: 'user', content: messageContent }],
            stream: true,
            ...apiGenerationOptions(provider),
            ...(Number.isInteger(maxTokens) && maxTokens > 0 ? { max_tokens: maxTokens } : {}),
          };
      const response = !endpointGuard.allowed
        ? { ok: false, error: `Endpoint blocked: ${endpointGuard.reason}`, status: 0 }
        : ready.success
        ? await fetchWithPreHeaderRetry(() => fetch(nativeOllamaUrl || `${provider.endpoint}/chat/completions`, {
            method: 'POST',
            headers,
            signal: controller.signal,
            // Hands the request undici's ceilings disabled, leaving `stallTimeout`
            // / `absoluteTimeout` above as the run's only bounds — see
            // ./internal/streamTransport.js for what fired first without it.
            dispatcher: streamTransportDispatcher(),
            body: JSON.stringify(requestBody)
          }), {
            signal: controller.signal,
            // A caller-owned budget/admission hook accounts for one invocation;
            // do not silently replay a possibly processed local generation.
            allowReplay: !beforeExecute && isReplaySafeLocalRequest(provider),
          }).catch(err => ({ ok: false, error: describeTransportError(err), status: 0 }))
        : { ok: false, error: ready.error || 'Provider readiness check failed', status: 0 };

      if (!response.ok) {
        let responseBody = response.error || '';
        if (response.text) {
          responseBody = await response.text().catch(() => response.error || '');
        }
        await finalizer.finalize({
          type: 'response-error',
          status: response.status,
          statusText: response.statusText,
          body: responseBody,
          headers: response.headers,
        });
        await finalizer.settled();
        onSettled?.();
        return runId;
      }

      const reader = response.body.getReader();
      const decoder = new TextDecoder();

      // A read returns an arbitrary SLICE of bytes, never a whole SSE frame, so
      // both boundaries have to be carried across iterations:
      //  - `buffer` holds the partial trailing LINE. Parsing per-chunk instead
      //    fed JSON.parse a frame cut in half at the ~8KB read boundary, which
      //    threw `Unterminated string in JSON at position 8064` and failed a
      //    220s NVIDIA NIM nemotron run with outputSize 0 — a reasoning model's
      //    hidden-channel frames are big enough to straddle a read routinely.
      //  - `{ stream: true }` holds the partial trailing CHARACTER, so a
      //    multi-byte rune split across reads doesn't decode to U+FFFD.
      // This mirrors every other SSE consumer in the tree (openAiChatStream.js,
      // ollamaManager.js, voice/llm.js); this loop was the lone holdout.
      let buffer = '';
      let finishReason = null;

      const consumeLine = (rawLine) => {
        // A CRLF transport leaves `\r` on each line: it has to come off before
        // the terminal-sentinel comparison, or `[DONE]\r` falls through to
        // JSON.parse and throws at the very end of an otherwise-good stream.
        const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine;
        // The native Ollama stream is NDJSON — one bare JSON object per line,
        // no `data: ` framing — reshaped into the OpenAI chunk read below.
        if (nativeOllamaUrl) {
          if (!line.trim()) return;
          const frame = safeJsonParse(line, null);
          if (!frame) {
            console.error(`❌ Run ${runId} skipped an unparseable stream frame (${line.length} chars)`);
            return;
          }
          consumeChunk(ollamaNativeFrameToChunk(frame));
          return;
        }
        if (!line.startsWith('data: ')) return;

        const data = line.slice(6);
        // `data: ` with an empty payload is a heartbeat on some providers, not a
        // frame — it must not reach the parse-failure log below, which would
        // otherwise emit a line per keep-alive for the life of the stream.
        if (!data.trim() || data === '✅' || data === '[DONE]') return;

        // One malformed frame must not discard the tokens around it. Before the
        // buffering above, every truncated frame reached here and its throw
        // aborted the whole run; now a parse failure can only mean the provider
        // actually emitted a bad frame, which is worth a log line, not the run.
        const parsed = safeJsonParse(data, null);
        if (!parsed) {
          console.error(`❌ Run ${runId} skipped an unparseable stream frame (${data.length} chars)`);
          return;
        }
        consumeChunk(parsed);
      };

      // One OpenAI-shaped chunk — an SSE `data:` frame, or a native Ollama
      // frame already reshaped by ollamaNativeFrameToChunk.
      const consumeChunk = (parsed) => {
        const choice = parsed?.choices?.[0];
        if (typeof choice?.finish_reason === 'string') finishReason = choice.finish_reason;
        const delta = choice?.delta;

        if (delta?.content) {
          const text = delta.content;
          output += text;
          onData?.({ text });
        }

        // The hidden channel goes by three names: OpenRouter-style endpoints
        // use `reasoning`, NVIDIA NIM / vLLM / DeepSeek-R1-compatible servers
        // use `reasoning_content`, and llama.cpp/MTPLX sometimes say
        // `thinking`. First one present wins — a provider sends one spelling,
        // and the precedence matches ../openAiChatStream.js so a frame that
        // somehow carried two would resolve identically on both readers.
        // Reading only `reasoning` silently discarded EVERY reasoning token
        // from NIM — `nvidia/nemotron-3.5-lightning-30b-a3b` sends 126 of 128
        // frames as `reasoning_content` — so the fallback below never fired and
        // a cut-off run reported `outputSize: 0`, indistinguishable from a
        // provider that answered nothing at all. Restated rather than imported
        // because this directory stays self-contained — see ./AGENTS.md.
        const reasoningDelta = delta?.reasoning || delta?.reasoning_content || delta?.thinking;
        if (reasoningDelta) {
          reasoning += reasoningDelta;
        }
      };

      const processStream = async () => {
        while (true) {
          const { done, value } = await reader.read();
          if (done) break;

          // Bytes arrived, so the upstream is alive: push the no-progress bound
          // out. An EMPTY chunk is deliberately not progress — it proves the
          // reader woke, not that the provider produced anything, and treating
          // it as a heartbeat would let a reader spinning on zero-length reads
          // hold the stall bound open indefinitely. The absolute cap is what
          // bounds a provider that DOES trickle real bytes forever.
          if (value?.byteLength > 0) lifecycle.noteStreamProgress();

          buffer += decoder.decode(value, { stream: true });
          const lines = buffer.split('\n');
          // The last element is either an incomplete line or '' — either way it
          // is not ready to parse, so it becomes the next iteration's prefix.
          buffer = lines.pop() || '';

          for (const line of lines) consumeLine(line);
        }

        // A stream that ends without a trailing newline (or without [DONE])
        // still has a complete frame sitting in the carry buffer; dropping it
        // would silently truncate the tail of the answer.
        buffer += decoder.decode();
        if (buffer.trim()) consumeLine(buffer);

        // Capture the fallback decision BEFORE mutating `output` — otherwise
        // the metadata check below is always false on the reasoning-only path
        // because `output` was just overwritten with the reasoning text.
        const usedReasoningAsFallback = !output.trim() && reasoning.trim().length > 0;
        if (usedReasoningAsFallback) {
          console.log(`🧠 Reasoning model detected - using reasoning as output (${reasoning.length} chars)`);
          output = reasoning;
          onData?.({ text: reasoning, isReasoning: true });
        }

        await finalizer.finalize({
          type: 'success',
          finishReason,
          usedReasoningAsFallback,
        });
      };

      processStream().catch(async err => {
        await finalizer.finalize({ type: 'stream-error', error: err });
      }).finally(async () => {
        // A timeout can finish metadata before the aborted reader settles.
        // Resource ownership ends only after both transport and saving finish.
        try { await reader.cancel?.(); reader.releaseLock?.(); }
        catch (err) { onPersistenceFailure?.(); console.error(`❌ Run ${runId} reader cleanup failed: ${err.message}`); }
        await finalizer.settled();
        onSettled?.();
      }).catch(handlerErr => {
        onPersistenceFailure?.();
        console.error(`❌ Run ${runId} failure handler error: ${handlerErr.message}`);
      });

      return runId;
    },

    async stopRun(runId) {
      // Host-spawned runs (CLI/TUI) take precedence — kill the registered handle
      // and drop it so a later isRunActive reports false.
      const external = externalRuns.get(runId);
      if (external) {
        externalStopRequests.add(runId);
        if (external.kill && !external.killed) killProcessTree(external);
        else if (external.abort) external.abort();
        externalRuns.delete(runId);
        return true;
      }

      const active = activeRuns.get(runId);
      if (!active) return false;

      if (active.kill) {
        killProcessTree(active);
      } else if (active.abort) {
        // Mark BEFORE aborting: the abort rejects the in-flight reader
        // synchronously enough that the finalizer can already be running by
        // the time this returns, and it must see an intentional stop. Only the
        // AbortController (API) branch marks — the built-in CLI executor's
        // exit-code classification is unchanged by this, so a marker there
        // would never be consumed.
        activeStopRequests.add(runId);
        active.abort();
      }

      activeRuns.delete(runId);
      return true;
    },

    async getRun(runId) {
      const runDir = existingRunDir(runId);
      if (!runDir) return null;

      const metadata = safeJsonParse(await readFile(join(runDir, 'metadata.json'), 'utf-8').catch(() => '{}'));
      return metadata;
    },

    async getRunOutput(runId) {
      const runDir = existingRunDir(runId);
      if (!runDir) return null;

      return readFile(join(runDir, 'output.txt'), 'utf-8');
    },

    async getRunPrompt(runId) {
      const runDir = existingRunDir(runId);
      if (!runDir) return null;

      return readFile(join(runDir, 'prompt.txt'), 'utf-8');
    },

    async listRuns(limit = 50, offset = 0, source = 'all') {
      await ensureRunsDir();

      const entries = await readdir(RUNS_PATH, { withFileTypes: true });
      const runIds = entries.filter(e => e.isDirectory()).map(e => e.name);

      const runs = [];
      for (const runId of runIds) {
        const metadataPath = join(RUNS_PATH, runId, 'metadata.json');
        if (existsSync(metadataPath)) {
          const metadata = safeJsonParse(await readFile(metadataPath, 'utf-8').catch(() => '{}'));
          if (metadata.id) runs.push(metadata);
        }
      }

      let filteredRuns = runs;
      if (source !== 'all') {
        filteredRuns = runs.filter(run => {
          const runSource = run.source || 'devtools';
          return runSource === source;
        });
      }

      filteredRuns.sort((a, b) => new Date(b.startTime) - new Date(a.startTime));

      return {
        total: filteredRuns.length,
        runs: filteredRuns.slice(offset, offset + limit)
      };
    },

    async deleteRun(runId) {
      // Kill an in-flight host-spawned run before removing its dir so deleting a
      // live run doesn't leave a zombie child process behind.
      if (externalRuns.has(runId)) {
        await service.stopRun(runId);
      }

      const runDir = existingRunDir(runId);
      if (!runDir) return false;

      await withAssetPublication(() => rm(runDir, { recursive: true }));
      return true;
    },

    async deleteFailedRuns() {
      await ensureRunsDir();

      const entries = await readdir(RUNS_PATH, { withFileTypes: true });
      const runIds = entries.filter(e => e.isDirectory()).map(e => e.name);

      let deletedCount = 0;
      for (const runId of runIds) {
        const metadataPath = join(RUNS_PATH, runId, 'metadata.json');
        if (existsSync(metadataPath)) {
          const metadata = safeJsonParse(await readFile(metadataPath, 'utf-8').catch(() => '{}'));
          if (metadata.success === false) {
            await withAssetPublication(() => rm(join(RUNS_PATH, runId), { recursive: true }));
            deletedCount++;
          }
        }
      }

      return deletedCount;
    },

    async isRunActive(runId) {
      return externalRuns.has(runId) || activeRuns.has(runId);
    },

    // Activity projection only: never return the persisted prompt, output,
    // workspace, or error fields. Bound I/O to the live registry, not history.
    async getActiveRunSummaries(limit = 50) {
      const ids = [...new Set([...activeRuns.keys(), ...externalRuns.keys()])].slice(0, limit);
      const runs = await Promise.all(ids.map(async (runId) => {
        const metadata = safeJsonParse(await readFile(join(RUNS_PATH, runId, 'metadata.json'), 'utf8').catch(() => '{}'));
        const text = (value) => typeof value === 'string' ? value.slice(0, 200) : null;
        return {
          runId,
          providerId: text(metadata.providerId),
          model: text(metadata.model),
          source: text(metadata.source),
          startedAt: text(metadata.startTime),
        };
      }));
      return runs.filter(({ runId }) => activeRuns.has(runId) || externalRuns.has(runId));
    },

    /**
     * How many runs this instance is tracking right now — API-based
     * (`activeRuns`) and host-spawned CLI/TUI (`externalRuns`) alike. A count
     * only: no run's prompt or output crosses this surface, so a host can
     * safely fold it into an activity/idle gate (PortOS's `systemIdle.js`
     * does exactly that) without leaking record content.
     */
    async getActiveRunCount() {
      return activeRuns.size + externalRuns.size;
    }
  };

  return service;
}