Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

22.2 KB
Raw Blame
/**
 * Everything the Cursor desktop app calls, and nothing else.
 *
 *   POST /api/cursor/stt         `api_key` header, multipart file field `file` → { transcript, lang }
 *   POST /api/cursor/summarize   `api_key` header, JSON { prompt } → { text, model }
 *
 * DELIBERATELY SELF-CONTAINED. The device's voice path (firmware streams raw PCM over /api/device-ws,
 * deviceWs.ts hands it to lib/stt.ts) is untouched and unshared: this file owns its own Deepgram call,
 * its own LLM call, its own auth and its own multipart scope, so nothing here can regress that path
 * and nothing there constrains this one. The only imports are leaf utilities (env, logger, response,
 * sha256hex).
 *
 * Both routes are in the auth middleware's skip-list — NOT unauthenticated, they simply validate their
 * own shared secret rather than an SSO token, the same arrangement /api/analytics/report has with
 * machineAuth. One credential (CURSOR_STT_API_KEY) gates both: the app holds a single key.
 */
import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify'
import multipart from '@fastify/multipart'
import { timingSafeEqual } from 'crypto'
import { env } from '../config/env.js'
import { sha256hex } from '../utils/crypto.js'
import { sendSuccess, sendError } from '../utils/response.js'
import { logger } from '../utils/logger.js'
import { VOICE_WAV_LANGS, transcribeWavWithDeepgram } from '../lib/deepgramWav.js'

export const CURSOR_STT_PATH = '/api/cursor/stt'
export const CURSOR_SUMMARIZE_PATH = '/api/cursor/summarize'
export const CURSOR_ROUTE_PATH = '/api/cursor/route'


/** Most conversations one routing request may weigh. The device sends the ones it is showing, newest
 *  first; past this the prompt grows without the pick getting better. */
const MAX_ROUTE_AGENTS = 8

/** Per-agent recent-activity text. Enough for three turns' worth of gist, bounded so one chatty agent
 *  cannot crowd the others out of the prompt. */
const MAX_RECENT_CHARS = 600

/** ~25 MB — far above any dictation clip this serves, while still bounding one request's buffer. */
const MAX_AUDIO_BYTES = 25 * 1024 * 1024

/** Far above any dictation-sized prompt, but a bound: a looping client must not burn LLM credit. */
const MAX_PROMPT_CHARS = 100_000

/**
 * Generation can legitimately think for a while; without a deadline a stalled upstream pins a socket.
 *
 * Kept UNDER the gateway's own limit, deliberately. ingress-nginx cuts a proxied response at 60s
 * (`proxy_read_timeout`), so a 120s budget here could never be honoured: a slow relay produced a bare 504
 * with no body, and the client saw a generic failure instead of this service's answer. Measured during a
 * relay degradation — "Say OK" took 49s and a real routing call 504'd at 60.3s. Finishing first is what
 * lets the fallback below run at all.
 */
const LLM_TIMEOUT_MS = 45_000


/**
 * The LLM behind /api/cursor/summarize: an OpenAI-compatible endpoint, called with plain `fetch`.
 *
 * No `openai` SDK dependency — with a `base_url` that already ends in `/v1`, the SDK's
 * `chat.completions.create` is exactly this one POST, so pulling in a package to make it would buy
 * nothing and cost this file its self-containment.
 *
 * Base URL, key and model all come from env: the relay URL embeds a network id and the key is a
 * long-lived JWT, neither of which belongs in the repo.
 */
async function callLlm(prompt: string): Promise<{ text: string; model: string }> {
  const model = env.CURSOR_LLM_MODEL
  const res = await fetch(`${env.CURSOR_LLM_BASE_URL!.replace(/\/+$/, '')}/chat/completions`, {
    method: 'POST',
    headers: { Authorization: `Bearer ${env.CURSOR_LLM_API_KEY}`, 'Content-Type': 'application/json' },
    // The prompt goes up verbatim as the single user message — no system prompt, no wrapping. What the
    // app asks for is what the model sees, so changing the wording is a client-side change, not a deploy.
    body: JSON.stringify({ model, messages: [{ role: 'user', content: prompt }] }),
    signal: AbortSignal.timeout(LLM_TIMEOUT_MS),
  })
  if (!res.ok) {
    const detail = await res.text().catch(() => '')
    throw new Error(`LLM HTTP ${res.status}: ${detail.slice(0, 200)}`)
  }
  const data = (await res.json()) as {
    model?: string
    choices?: Array<{ message?: { content?: string } }>
  }
  return { text: (data.choices?.[0]?.message?.content ?? '').trim(), model: data.model || model }
}

/**
 * Both endpoints' only credential: a shared secret in the `api_key` header.
 *
 * Fails CLOSED when CURSOR_STT_API_KEY is unset — an absent secret must never read as "no auth needed"
 * on routes that spend Deepgram and LLM credit. Both sides are hashed before comparison so
 * timingSafeEqual receives equal-length buffers: it throws on a length mismatch, and the real key's
 * length is itself something better not leaked through a distinguishable error.
 */
async function cursorAuth(req: FastifyRequest, reply: FastifyReply): Promise<void> {
  const expected = env.CURSOR_STT_API_KEY
  if (!expected) {
    logger.error('cursor: CURSOR_STT_API_KEY is not configured — refusing the request')
    sendError(reply, 'This endpoint is not configured', 'CURSOR_NOT_CONFIGURED', 503)
    return
  }
  // BOTH spellings, and the dashed one is the reason this works in production at all: ingress-nginx
  // defaults to `underscores_in_headers off`, which DROPS any header containing `_`. So `api_key` — the
  // name this endpoint was specified with — arrives when you call the service directly and silently
  // vanishes behind the ingress, surfacing as "Missing api key" for a request that did send it.
  // Measured, not guessed: on prod a dashed `x-api-key` reaches another route fine while `api_key` does
  // not, and locally (no proxy) `api_key` works. Keep both — `api_key` for direct/local callers,
  // `api-key` for anything crossing the ingress.
  const raw = req.headers['api_key'] ?? req.headers['api-key']
  const provided = Array.isArray(raw) ? raw[0] : raw
  if (typeof provided !== 'string' || !provided) {
    sendError(reply, 'Missing api key', 'UNAUTHORIZED', 401)
    return
  }
  const a = Buffer.from(sha256hex(provided), 'hex')
  const b = Buffer.from(sha256hex(expected), 'hex')
  if (!timingSafeEqual(a, b)) {
    sendError(reply, 'Invalid api key', 'UNAUTHORIZED', 401)
    return
  }
}

/**
 * Pin the language rather than auto-detect: on a short clip detection is worse than being told.
 *
 * Anything unrecognised falls back to 'vi' — this endpoint's callers are Vietnamese-first, and that was
 * the behaviour before the other five languages were accepted.
 */
function pickLang(value: unknown): string {
  const v = typeof value === 'string' ? value.toLowerCase() : ''
  return VOICE_WAV_LANGS.has(v) ? v : 'vi'
}


/**
 * One routing decision, parsed defensively.
 *
 * The model behind this endpoint is NOT the one the prompt was written for — the brain's router runs Haiku
 * through the Claude CLI, this runs whatever CURSOR_LLM_MODEL names (DeepSeek at the time of writing). So
 * "respond with only JSON" is a request, not a guarantee: a fenced block or a sentence of preamble is a
 * normal thing to get back, and the caller cannot act on either.
 */
function parseDecision(text: string, ids: Set<string>): { agentId: string; confidence: number; reason: string } | null {
  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i)
  const body = (fenced ? fenced[1] : text).trim()
  const brace = body.indexOf('{')
  const close = body.lastIndexOf('}')
  if (brace < 0 || close <= brace) return null
  let obj: unknown
  try {
    obj = JSON.parse(body.slice(brace, close + 1))
  } catch {
    return null
  }
  const o = obj as { agentId?: unknown; confidence?: unknown; reason?: unknown }
  const agentId = typeof o.agentId === 'string' ? o.agentId : ''
  // The id MUST be one we offered. A model that invents an id is not a low-confidence pick, it is a failed
  // request: routing to an id the caller does not have would put the words nowhere and report success.
  if (!ids.has(agentId)) return null
  const raw = typeof o.confidence === 'number' ? o.confidence : 0
  return {
    agentId,
    confidence: Math.max(0, Math.min(1, raw)),
    reason: typeof o.reason === 'string' ? o.reason.slice(0, 120) : '',
  }
}

/**
 * Ported from machine-node/brain's prefrontal/voiceRouter.ts, which routes voice for BACKEND machines and
 * cannot serve this one: it runs inside a machine-node, and a cabled Cursor machine has no node. Same job,
 * same shape of answer, so the wording is kept close enough that the two can be compared — including the
 * instruction not to translate, which is what stops a Vietnamese task coming back as an English summary.
 */
function buildRoutePrompt(transcript: string, agents: Array<{ id: string; name: string; recent: string }>): string {
  const lines = agents
    .map((a) => `- id=${a.id} | name="${a.name}" | recent: ${a.recent.trim() || '(no activity yet)'}`)
    .join('\n')
  return (
    `You are a ROUTER. Assign ONE incoming voice task to the single best-fit conversation from the fixed ` +
    `list below. You MUST always choose exactly one id from the list — there is NO "none" option and you ` +
    `may NOT decline. The NAME is a strong signal. Each conversation's RECENT activity disambiguates when ` +
    `names alone are ambiguous. If nothing matches well, still pick the CLOSEST one and give it a low ` +
    `confidence.\n\n` +
    `Voice task (verbatim; may be Vietnamese — do NOT translate it): "${transcript}"\n\n` +
    `Conversations:\n${lines}\n\n` +
    `Set confidence 0..1 for how good the fit is:\n` +
    `- 0.85+ when the name and/or recent activity clearly match\n` +
    `- ~0.6 when it is a reasonable but not certain match\n` +
    `- ~0.3 when nothing fits well but this is the closest.\n\n` +
    `Respond with ONLY a single JSON object, no prose, no markdown fence:\n` +
    `{"agentId":"<one id from the list>","confidence":<0..1>,"reason":"<max 12 words>"}`
  )
}

export async function cursorRoutes(app: FastifyInstance): Promise<void> {
  // Registered inside this plugin's scope, not in server.ts: multipart parsing exists for the STT route
  // and nothing else, so no other endpoint's body handling changes by adding it. The JSON route below
  // is unaffected — Fastify still parses application/json normally.
  await app.register(multipart, { limits: { fileSize: MAX_AUDIO_BYTES, files: 1 } })

  app.post(CURSOR_STT_PATH, { preHandler: [cursorAuth] }, async (req, reply) => {
    let file
    try {
      file = await req.file()
    } catch {
      sendError(reply, 'Expected a multipart/form-data upload', 'BAD_REQUEST', 400)
      return
    }
    if (!file) {
      sendError(reply, 'Missing audio file (multipart field "file")', 'NO_FILE', 400)
      return
    }

    // Oversize is NOT checked here. @fastify/multipart throws FST_REQ_FILE_TOO_LARGE past the limit and
    // the shared errorHandler already renders that as a 413 in the usual envelope. An explicit
    // `file.file.truncated` branch would be dead code — do not add one.
    const audio = await file.toBuffer()
    if (audio.length === 0) {
      sendError(reply, 'Audio file is empty', 'EMPTY_FILE', 400)
      return
    }

    // Language may ride the query string or the form; the query wins so a caller can override without
    // rebuilding the body. Form fields arrive as objects carrying `.value`, not bare strings.
    const formLang = file.fields?.lang
    const formLangValue = formLang && !Array.isArray(formLang) && 'value' in formLang ? formLang.value : undefined
    const lang = pickLang((req.query as { lang?: unknown } | undefined)?.lang ?? formLangValue)

    try {
      const transcript = await transcribeWavWithDeepgram(audio, file.mimetype || 'audio/wav', lang)
      logger.info('cursor-stt transcribe', { bytes: audio.length, mimetype: file.mimetype, lang, chars: transcript.length })
      sendSuccess(reply, { transcript, lang })
    } catch (err) {
      // Deepgram's message can name the upstream project/key, so it stays in the log and the caller gets
      // a plain 502 — this endpoint's one client cannot act on the detail anyway.
      logger.error('cursor-stt transcribe failed', { error: err instanceof Error ? err.message : String(err) })
      sendError(reply, 'Transcription failed', 'STT_FAILED', 502)
    }
  })

  app.post(CURSOR_SUMMARIZE_PATH, { preHandler: [cursorAuth] }, async (req, reply) => {
    // Same fail-closed rule as the shared secret: an unconfigured LLM is a 503, never a silent fallback.
    if (!env.CURSOR_LLM_BASE_URL || !env.CURSOR_LLM_API_KEY) {
      logger.error('cursor-summarize: CURSOR_LLM_BASE_URL/CURSOR_LLM_API_KEY not configured — refusing')
      sendError(reply, 'Summarization is not configured', 'LLM_NOT_CONFIGURED', 503)
      return
    }

    const prompt = (req.body as { prompt?: unknown } | undefined)?.prompt
    if (typeof prompt !== 'string' || !prompt.trim()) {
      sendError(reply, 'Missing prompt', 'BAD_REQUEST', 400)
      return
    }
    if (prompt.length > MAX_PROMPT_CHARS) {
      sendError(reply, `Prompt exceeds ${MAX_PROMPT_CHARS} characters`, 'PROMPT_TOO_LARGE', 400)
      return
    }

    try {
      const { text, model } = await callLlm(prompt)
      logger.info('cursor-summarize', { chars: prompt.length, outChars: text.length, model })
      sendSuccess(reply, { text, model })
    } catch (err) {
      // Same reasoning as the STT branch: the upstream message can name the relay network or the key,
      // so it stays in the log and the caller gets a plain 502.
      logger.error('cursor-summarize failed', { error: err instanceof Error ? err.message : String(err) })
      sendError(reply, 'Summarization failed', 'LLM_FAILED', 502)
    }
  })

  /**
   * Pick which Cursor conversation a spoken task belongs to.
   *
   * The cabled machine's Overview voice: the dial records, the Mac ships the audio to /stt above, then asks
   * here which of its open conversations should receive the words. The decision is a pure function of what
   * the caller sends — this endpoint holds no state and knows nothing about any machine.
   */
/**
 * Function words only. A preposition matching a name is noise wearing a signal's clothes — verbs and nouns
 * stay, because "research", "deploy" and "parser" are exactly what routing runs on. The last row is the
 * Vietnamese equivalents, which arrive unaccented from the tokenizer below.
 */
const ROUTE_STOP = new Set([
  'the', 'a', 'an', 'and', 'or', 'for', 'to', 'of', 'in', 'on', 'at',
  'about', 'with', 'from', 'into', 'over', 'under', 'up', 'down', 'out',
  'my', 'our', 'your', 'his', 'her', 'its', 'their', 'me', 'you', 'we',
  'it', 'this', 'that', 'these', 'those', 'is', 'are', 'was', 'were',
  'be', 'been', 'do', 'does', 'did', 'have', 'has', 'had', 'will',
  'would', 'can', 'could', 'should', 'please', 'what', 'when', 'where',
  'which', 'who', 'how', 'why',
  'agent', 'task', 'chat', 'new', 'cua', 'va', 'cho', 'voi',
])

/**
 * Fold to the alphabet the matcher works in: lowercase, unaccented, split on anything that is not a letter
 * or digit.
 *
 * Unaccented because a spoken "sửa parser" transcribed by an English recogniser arrives as "sua parser",
 * and a conversation named "Sửa parser" would otherwise never match itself. `đ` needs its own rule — NFD
 * does not decompose it, so stripping combining marks alone leaves "đăng" as "đang" and never as "dang".
 */
function routeTokens(s: string): string[] {
  return s
    .normalize('NFD')
    .replace(/[\u0300-\u036f]/g, '')
    .replace(/đ/gi, 'd')
    .toLowerCase()
    .split(/[^a-z0-9]+/)
    .filter((t) => t.length > 1 && !ROUTE_STOP.has(t))
}

/**
 * The name is worth far more than the history, and deliberately so: people name conversations by what they
 * are for, and say what they want done. Recent activity is a tie-breaker for names that do not settle it.
 * Normalised by the NAME's length, not the sentence's — matching both words of a two-word name is a strong
 * signal, and dividing by a long sentence would bury it.
 */
function scoreAgent(a: { name: string; recent: string }, said: Set<string>) {
  const name = new Set(routeTokens(a.name))
  const recent = new Set(routeTokens(a.recent))
  if (said.size === 0) return { score: 0, named: 0 }
  const hits = (set: Set<string>) => [...set].filter((t) => said.has(t)).length
  const nameHits = hits(name)
  const nameScore = name.size === 0 ? 0 : nameHits / name.size
  const recentScore = recent.size === 0 ? 0 : Math.min(hits(recent) / 3, 1)
  return { score: nameScore * 3 + recentScore, named: nameHits }
}

/**
 * Decide by counting words, and say when counting is not enough.
 *
 * A PORT of QuickDialKit/VoiceRouter.swift in autonomous-macroz, thresholds included — it was measured
 * against real misroutes there, and this endpoint answers the same question for the same device. The
 * reason it exists here rather than only in the model: an LLM call is a network dependency on every spoken
 * turn, and when the relay degrades (49s for "Say OK", measured) an unambiguous name match should not be
 * waiting on it at all.
 *
 * THE NAME OUTRANKS THE TOPIC. Saying words from exactly one conversation's name IS the decision, however
 * much another one's recent activity overlaps the subject: a name is chosen by the user, a topic history
 * accretes — and accretes fastest on whichever conversation earlier misroutes fed.
 */
function pickByScore(transcript: string, agents: Array<{ id: string; name: string; recent: string }>) {
  const said = new Set(routeTokens(transcript))
  const scored = agents.map((a) => ({ a, ...scoreAgent(a, said) }))

  // Several names said: they compete among THEMSELVES — a conversation matched only by topic does not
  // referee a contest between named ones.
  const named = scored.filter((s) => s.named > 0)
  if (named.length === 1) {
    const only = named[0]
    return { agentId: only.a.id, confidence: Math.max(0.8, Math.min(1, 0.6 + only.score / 6)),
             reason: 'only name said', ambiguous: false }
  }
  const pool = named.length > 1 ? named : scored
  const ranked = [...pool].sort((x, y) => y.score - x.score)
  const top = ranked[0]
  const gap = top.score - (ranked[1]?.score ?? 0)

  // Nothing scored at all: the words matched no name and no history, so there is nothing to be confident
  // about and the model gets it.
  if (top.score <= 0) return { agentId: top.a.id, confidence: 0, reason: 'nothing matched', ambiguous: true }
  // A clear winner both scores something AND leads the runner-up by half a name-word. Below that the two
  // are not distinguishable by counting words, which is the definition of a question for a model.
  if (gap >= 1.5) return { agentId: top.a.id, confidence: Math.min(1, 0.6 + gap / 6),
                           reason: 'name matched', ambiguous: false }
  return { agentId: top.a.id, confidence: 0.3 + gap / 5, reason: 'closest of several', ambiguous: true }
}

  app.post(CURSOR_ROUTE_PATH, { preHandler: [cursorAuth] }, async (req, reply) => {
    if (!env.CURSOR_LLM_BASE_URL || !env.CURSOR_LLM_API_KEY) {
      logger.error('cursor-route: CURSOR_LLM_BASE_URL/CURSOR_LLM_API_KEY not configured — refusing')
      sendError(reply, 'Routing is not configured', 'LLM_NOT_CONFIGURED', 503)
      return
    }

    const body = req.body as { transcript?: unknown; agents?: unknown } | undefined
    const transcript = typeof body?.transcript === 'string' ? body.transcript.trim() : ''
    if (!transcript) {
      sendError(reply, 'Missing transcript', 'BAD_REQUEST', 400)
      return
    }
    if (!Array.isArray(body?.agents) || body.agents.length === 0) {
      sendError(reply, 'Missing agents', 'BAD_REQUEST', 400)
      return
    }

    const agents = (body.agents as Array<Record<string, unknown>>)
      .map((a) => ({
        id: typeof a.id === 'string' ? a.id : '',
        name: typeof a.name === 'string' ? a.name : '',
        recent: typeof a.recent === 'string' ? a.recent.slice(0, MAX_RECENT_CHARS) : '',
      }))
      .filter((a) => a.id && a.name)
      .slice(0, MAX_ROUTE_AGENTS)
    if (agents.length === 0) {
      sendError(reply, 'No usable agents', 'BAD_REQUEST', 400)
      return
    }

    // One conversation is not a decision. Answering it locally saves a call, and more importantly it means
    // the single-agent case cannot fail for a reason that has nothing to do with the user.
    if (agents.length === 1) {
      sendSuccess(reply, { agentId: agents[0].id, confidence: 1, reason: 'only conversation' })
      return
    }

    // Count words first. Most spoken turns name the conversation they are about, and for those the model
    // adds latency and a failure mode without adding an answer.
    const scored = pickByScore(transcript, agents)
    if (!scored.ambiguous) {
      logger.info('cursor-route: by score', { agents: agents.length, chars: transcript.length, reason: scored.reason })
      sendSuccess(reply, { agentId: scored.agentId, confidence: scored.confidence, reason: scored.reason })
      return
    }

    try {
      const { text, model } = await callLlm(buildRoutePrompt(transcript, agents))
      const decision = parseDecision(text, new Set(agents.map((a) => a.id)))
      if (!decision) {
        logger.error('cursor-route: unusable answer', { model, sample: text.slice(0, 200) })
        sendError(reply, 'Routing failed', 'ROUTE_FAILED', 502)
        return
      }
      logger.info('cursor-route', { agents: agents.length, chars: transcript.length, model, confidence: decision.confidence })
      sendSuccess(reply, decision)
    } catch (err) {
      // The model was the TIE-BREAKER, not the router. When it cannot answer, the scored pick is still the
      // best information available, and returning it beats throwing away words the user has already spoken
      // — the device's alternative is an apology and a lost turn.
      //
      // Only when something actually matched, though: `nothing matched` means the words hit no name and no
      // history, and delivering that to an arbitrary conversation is worse than saying so.
      logger.error('cursor-route failed', { error: err instanceof Error ? err.message : String(err) })
      if (scored.confidence > 0) {
        logger.warn('cursor-route: falling back to the scored pick', { reason: scored.reason })
        sendSuccess(reply, { agentId: scored.agentId, confidence: scored.confidence,
                             reason: `${scored.reason} (model unavailable)` })
        return
      }
      sendError(reply, 'Routing failed', 'ROUTE_FAILED', 502)
    }
  })
}