The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
import { isUndeliveredCorrectionRefusal } from '@hermes/shared/json-rpc-channel'
import type { GatewayClient } from '../gatewayClient.js'
import type { InputDetectDropResponse, PromptSubmitResponse, SessionActivateResponse } from '../gatewayTypes.js'
import type { QueueItem } from '../hooks/useQueue.js'
import { t } from '../i18n/runtime.js'
import { stageImagePath } from '../lib/imageAttachments.js'
import { pendingInputOwner, savePendingInput } from '../lib/pendingInputs.js'
import type { Msg } from '../types.js'
import { markBubbleShown } from './pendingBubbles.js'
import { retireCanonicalControls } from './slash/canonicalSessionControls.js'
import { captureDestination, isCurrentDestination, type SubmissionDestination } from './submissionDestination.js'
import { turnController } from './turnController.js'
import { getUiState, patchUiState } from './uiStore.js'
const SESSION_BUSY_RE = /session busy|waiting for model response/i
export const isSessionBusyError = (e: unknown) => e instanceof Error && SESSION_BUSY_RE.test(e.message)
export interface SubmitPromptDeps {
appendMessage: (msg: Msg) => void
enqueue: (text: string, display?: string, destination?: SubmissionDestination) => void
expand: (text: string) => string
gw: GatewayClient
setLastUserMsg: (value: string) => void
sys: (text: string) => void
}
// Optimistically flip the session to busy the INSTANT a prompt is accepted for
// submission — synchronously, before we await anything.
//
// This is the fix for the queue-mode race (display.busy_input_mode: queue):
// the submit path first fires an async `input.detect_drop` RPC and only marked
// the session busy inside that RPC's `.then`. A second Enter pressed inside
// that round-trip window read `busy === false` in dispatchSubmission and raced
// a second `prompt.submit` onto the backend instead of landing in the local
// queue. That produced the reported symptom: the second message "waited for
// the first to respond, then went to the queue", and the client lost track of
// it (the backend accepts a mid-turn submit as {status:"queued"} — a success,
// not an error — so the local drain effect that watches the client-side queue
// never fires, leaving the UI stuck on "analyzing…" until Ctrl+C).
//
// Marking busy at the choke point closes the gap for every caller: the mainline
// submit, queue-edit picks, and the drain effect all funnel through here.
export function markSubmitting(): void {
patchUiState({ busy: true, status: 'running…' })
}
// A voice-mode transcript about to be submitted: the matching prompt.submit carries
// `voice_turn` so the gateway runs it on `auxiliary.voice_chat`. Matched by text, so a
// transcript the user edited or that went to the queue still lands as typed text.
let pendingVoiceTranscript: null | string = null
export function markNextSubmitVoice(text: string): void {
pendingVoiceTranscript = text
}
function takeVoiceTranscript(): null | string {
const transcript = pendingVoiceTranscript
pendingVoiceTranscript = null
return transcript
}
interface SubmitPromptOpts {
attachments?: Array<{ path: string; mime: string }>
skipDetectDrop?: boolean
destination?: SubmissionDestination
queueItem?: QueueItem
behindTurn?: boolean
}
type PendingOwner = ReturnType<typeof pendingInputOwner>
function beginOwnedTurn(
deps: SubmitPromptDeps,
text: string,
displayOverride: string | undefined,
displayText: string,
show: boolean
): void {
turnController.clearStatusTimer()
deps.setLastUserMsg(text)
if (show) {
deps.appendMessage({ role: 'user', text: displayOverride || displayText })
}
patchUiState({ busy: true, status: 'running…' })
turnController.bufRef = ''
turnController.interrupted = false
}
function prepareQueueItem(
item: QueueItem,
submitText: string,
opts: SubmitPromptOpts,
ownsTurn: () => boolean,
show: boolean
): void {
item.preparedText ??= submitText
// Busy controls have no deduplication receipt: never replay after an ambiguous reply.
if (item.controlMethod) {
item.legacyAttempted = true
}
item.attachments ??= opts.attachments?.map(attachment => ({ ...attachment }))
if (opts.behindTurn) {
item.queued = true
}
if (ownsTurn() && show) {
markBubbleShown(item.submissionId)
}
savePendingInput(item)
}
function submitParams(
sid: string,
item: QueueItem | undefined,
submitText: string,
attachments: SubmitPromptOpts['attachments'],
voice: boolean,
clarifyBatch: boolean
) {
return {
session_id: sid,
text: item?.preparedText ?? submitText,
...((item?.attachments ?? attachments)?.length ? { attachments: item?.attachments ?? attachments } : {}),
...(item?.controlMethod ? { execution_generation: item.executionGeneration } : {}),
...(item && !item.controlMethod ? { submission_id: item.submissionId, queued: item.queued !== false } : {}),
// A busy correction (steer/redirect) is not a turn: it never claims the voice route.
...(!item?.controlMethod && voice && { voice_turn: true }),
// Ink renders a whole question set as one form; undeclared, the owner asks one card per question.
...(!item?.controlMethod && clarifyBatch && { clarify_batch: true })
}
}
/** A correction the owner confirmed it never delivered becomes the next prompt: the same row and
* submission_id, admitted durably behind any running turn. Kept as a control it would be refused
* again on every retry, since the generation it names is over (#141511). */
export function readmitAsPrompt(item: QueueItem): void {
delete item.controlMethod
delete item.executionGeneration
item.legacyAttempted = false
item.queued = true
savePendingInput(item)
}
/** Settles a busy-correction reply; false when the owner refused it before any agent saw the text. */
function settleControlReply(
item: QueueItem,
r: PromptSubmitResponse,
deps: SubmitPromptDeps,
focused: () => boolean,
sid: string
): boolean {
// A bare stop phrase typed as a busy correction ended the voice chat and reached
// no turn (TUI-26): settled, rendered as the stop-phrase event, the turn untouched.
if (r?.voice_stopped === true) {
item.settle?.(true)
if (deps.gw.isCanonical) {
deps.gw.publishLocalEvent({ payload: { stop_phrase: true, typed: true }, session_id: sid, type: 'voice.transcript' })
}
return true
}
const control = r as PromptSubmitResponse & { delivered?: false; execution_generation?: number }
// Only a 'rejected' the owner marks undelivered (no live agent took it) becomes the next prompt. A
// plain 'rejected' can follow delivery (a Codex steer whose ACK timed out): re-sending it would
// run the correction twice, so it settles as refused with the input retained.
if (
control?.status === 'rejected' &&
control.delivered === false &&
control.execution_generation === item.executionGeneration
) {
return false
}
const accepted =
control.execution_generation === item.executionGeneration && ['queued', 'redirected'].includes(control.status ?? '')
item.settle?.(accepted)
if (focused()) {
deps.sys(accepted ? t('canonical.submit.correction', control.status) : t('canonical.submit.correctionRejected'))
}
return true
}
interface AdmissionContext {
deps: SubmitPromptDeps
destination: SubmissionDestination
focused: () => boolean
owner: PendingOwner
sid: string
}
function settleAdmissionReply(item: QueueItem, r: PromptSubmitResponse, ctx: AdmissionContext): void {
const { deps, destination, focused } = ctx
const accepted =
(r?.input_id ?? r?.admission_id) === item.submissionId &&
Boolean(destination.storedSid) &&
r?.target_session_id === destination.storedSid &&
r?.target_profile_home === (destination.ownerHome ?? destination.profileHome) &&
['queued', 'started', 'terminal', 'unknown'].includes(r?.status ?? '')
item.settle?.(accepted)
// A replay receipt ends this admission, not necessarily the current
// execution. Refresh the same live attachment without its transcript.
if (accepted && focused() && ['terminal', 'unknown'].includes(r.status ?? '')) {
refreshExecutionStatus(ctx)
}
if (!accepted && focused()) {
deps.sys(t('canonical.submit.admissionNotConfirmed'))
patchUiState({ status: t('canonical.submit.statusAdmissionUnconfirmed') })
}
}
function refreshExecutionStatus({ deps, focused, owner, sid }: AdmissionContext): void {
const observed = getUiState().info
void deps.gw
.request<SessionActivateResponse>('session.activate', { session_id: sid, omit_messages: true })
.then(snapshot => {
const current = getUiState().info
const info = snapshot.info
// Push lifecycle events (including same-generation completion)
// win over an in-flight snapshot; attachment changes win too.
if (
!focused() ||
current !== observed ||
snapshot.session_id !== sid ||
(snapshot.session_key || info?.stored_session_id) !== owner.storedSid ||
typeof snapshot.running !== 'boolean' ||
(current?.execution_generation !== undefined &&
(info?.execution_epoch !== current.execution_epoch ||
!Number.isSafeInteger(info?.execution_generation) ||
(info?.execution_generation ?? -1) < current.execution_generation))
) {
return
}
patchUiState({
busy: snapshot.running,
status: snapshot.running ? 'running…' : 'ready',
info: current && {
...current,
execution_epoch: info?.execution_epoch ?? current.execution_epoch,
execution_generation: info?.execution_generation ?? current.execution_generation,
running: snapshot.running
}
})
})
.catch((error: Error) => {
if (focused()) {
deps.sys(t('canonical.submit.executionUnconfirmed', error.message))
}
})
}
async function submitLegacy(
item: QueueItem,
sid: string,
submitText: string,
deps: SubmitPromptDeps,
focused: () => boolean
): Promise<void> {
if (focused()) {
deps.sys(t('canonical.submit.legacyDelivery'))
}
try {
item.legacyAttempted = true
savePendingInput(item)
const r = await deps.gw.request<PromptSubmitResponse>('prompt.submit', {
session_id: sid,
text: item.preparedText ?? submitText,
queued: item.queued !== false
})
const accepted = Boolean(
r?.voice_stopped || ['streaming', 'queued', 'steered', 'redirected'].includes(r?.status ?? '')
)
item.settle?.(accepted)
if (focused()) {
if (r?.voice_stopped) {
patchUiState({ busy: false, status: 'ready' })
} else if (!accepted) {
deps.sys(t('canonical.submit.legacyUnconfirmed'))
patchUiState({ status: t('canonical.submit.statusDeliveryUnconfirmed') })
}
}
} catch (error) {
item.settle?.(false)
if (focused()) {
deps.sys(t('canonical.submit.legacyUnconfirmedWith', error instanceof Error ? error.message : String(error)))
patchUiState({ status: t('canonical.submit.statusDeliveryUnconfirmed') })
}
}
}
// Submit a ready prompt (already resolved to be neither a slash command nor a
// shell escape, with a live session). Pulled out of useSubmission so the
// synchronous-busy invariant above is unit-testable without React test infra.
//
// `displayOverride` is what the transcript shows when it differs from what the
// agent receives — a `/skill` invocation expands into the whole skill body, and
// that scaffolding is model-facing only.
export function submitPrompt(
text: string,
deps: SubmitPromptDeps,
showUserMessage = true,
displayOverride?: string,
opts: SubmitPromptOpts = {}
): void {
// The voice marker belongs to the input submitted right after it, whatever happens to that input
// (busy correction, refusal): consumed here, so a later identical typed prompt never inherits it.
const voiceTranscript = takeVoiceTranscript()
const destination = opts.destination ?? captureDestination()
const owner = pendingInputOwner(opts.queueItem?.ownerDestination ?? destination)
const { sid } = owner
const focused = () => isCurrentDestination(owner)
// A busy-time admission joins the authority FIFO behind the running turn:
// it must not touch the in-flight stream buffer, status or transcript. Its
// user bubble is painted when the fanout reports the row started.
const ownsTurn = () => focused() && !opts.behindTurn
if (!sid) {
return deps.sys('session not ready yet')
}
// An identityless write cannot be deduplicated, even after the owner upgrades.
if (opts.queueItem?.legacyAttempted) {
opts.queueItem.settle?.(false)
if (focused()) {
deps.sys(t('canonical.submit.deliveryNotResent'))
patchUiState({ status: t('canonical.submit.statusDeliveryUnconfirmed') })
}
return
}
// Close the async-busy gap up front, before the detect_drop round-trip.
if (ownsTurn()) {
markSubmitting()
}
const readmit = (item: QueueItem) => {
readmitAsPrompt(item)
submitPrompt(text, deps, showUserMessage, displayOverride, {
behindTurn: true,
destination,
queueItem: item,
skipDetectDrop: true
})
}
const startSubmit = (displayText: string, submitText: string, show = true) => {
if (ownsTurn()) {
beginOwnedTurn(deps, text, displayOverride, displayText, show)
}
const item = opts.queueItem
if (item) {
prepareQueueItem(item, submitText, opts, ownsTurn, show)
}
if (item?.controlMethod && item.attachments?.length) {
item.settle?.(false)
if (focused()) {
deps.sys(t('canonical.submit.busyTextOnly'))
}
return
}
retireCanonicalControls(deps.gw, sid)
deps.gw
.request<PromptSubmitResponse>(
item?.controlMethod ?? 'prompt.submit',
submitParams(
sid,
item,
submitText,
opts.attachments,
voiceTranscript === submitText.trim(),
deps.gw.clarifyBatch === true
)
)
.then(r => {
if (item?.controlMethod) {
if (!settleControlReply(item, r, deps, focused, sid)) {
readmit(item)
}
return
}
// The gateway consumed a typed voice stop phrase server-side (voice
// chat ended, nothing admitted) — release the busy latch. The sidecar
// sends a voice.transcript {stop_phrase} event for the mode flags and
// notice; the shared gateway answers in this reply only (TUI-26), so
// render that same event locally.
if (r?.voice_stopped === true) {
item?.settle?.(true)
if (deps.gw.isCanonical) {
deps.gw.publishLocalEvent({
payload: { stop_phrase: true, typed: true },
session_id: sid,
type: 'voice.transcript'
})
}
// Only the busy latch this submission took: a stop typed behind a running
// turn ends the voice chat, not that turn, so its busy/status stay.
if (ownsTurn()) {
patchUiState({ busy: false, status: 'ready' })
}
return
}
if (item) {
settleAdmissionReply(item, r, { deps, destination, focused, owner, sid })
}
})
.catch(async (e: Error & { code?: number }) => {
// 4094 is a pre-admission refusal, not an ambiguous write; so is 4000 when the
// legacy `hermes serve` contract refuses `submission_id` as an unknown key. Special
// compute modes still support legacy submit; retry only these refusals,
// keeping the prepared payload, destination and queue mode unchanged.
const preAdmissionRefusal = e.code === 4094 || (e.code === 4000 && /submission_id/.test(e.message ?? ''))
if (item && preAdmissionRefusal && !deps.gw.isCanonical && !item.attachments?.length) {
await submitLegacy(item, sid, submitText, deps, focused)
return
}
if (item?.controlMethod && isUndeliveredCorrectionRefusal(e)) {
readmit(item)
return
}
if (item) {
item.settle?.(false)
if (focused()) {
deps.sys(t('canonical.submit.inputRetained', e.message))
patchUiState({ status: t('canonical.submit.statusAdmissionUnconfirmed') })
}
return
}
// Defensive: prompt.submit no longer rejects a mid-turn send with
// "session busy" (the gateway queues it and returns success), but keep
// the re-queue path as a safety net for any future/legacy gateway that
// still errors, so a message is never silently dropped.
if (isSessionBusyError(e)) {
deps.enqueue(submitText, displayOverride, destination)
if (!focused()) {
return
}
patchUiState({ busy: true, status: 'queued for next turn' })
return deps.sys(`queued: "${submitText.slice(0, 50)}${submitText.length > 50 ? '…' : ''}"`)
}
if (!focused()) {
return
}
deps.sys(`error: ${e.message}`)
patchUiState({ busy: false, status: 'ready' })
})
}
// Always ask the backend whether this looks like a file drop. The backend's
// _detect_file_drop handles paths with spaces, quotes, Windows drive letters,
// and escaped characters correctly. Literal submissions (startup -q queries)
// skip it: launcher-provided text must reach the agent untouched.
//
// No notice is emitted for a match: an image dropped into the composer already
// shows as an `[[ Image N ]]` token, and a matched non-image path is rewritten
// in place. Announcing it a second time above the status bar was the old
// out-of-band attachment UI.
if (opts.queueItem?.preparedText !== undefined) {
return startSubmit(text, opts.queueItem.preparedText, showUserMessage)
}
if (opts.skipDetectDrop) {
return startSubmit(text, deps.expand(text), showUserMessage)
}
if (deps.gw.isCanonical) {
if (
/^(?:["']?(?:[/.~]|[A-Za-z]:[/\\])|file:\/\/)/.test(text) &&
/\.(?:png|jpe?g|gif|webp)(?:["']?)(?:\s|$)/i.test(text)
) {
void stageImagePath(text, deps.gw, destination)
.then(image => {
opts.attachments = [{ path: image.path, mime: image.mime }]
startSubmit(text, image.remainder || 'What do you see in this image?', showUserMessage)
})
.catch((error: Error) => {
opts.queueItem?.settle?.(false)
if (focused()) {
deps.sys(t('canonical.submit.imageNotSubmitted', error.message))
if (ownsTurn()) {
patchUiState({ busy: false, status: t('canonical.submit.statusImageNotSubmitted') })
}
}
})
return
}
return startSubmit(text, deps.expand(text), showUserMessage)
}
deps.gw
.request<InputDetectDropResponse>('input.detect_drop', { session_id: sid, text })
.then(r => {
if (!r?.matched) {
return startSubmit(text, deps.expand(text), showUserMessage)
}
startSubmit(r.text || text, deps.expand(r.text || text), showUserMessage)
})
.catch(() => startSubmit(text, deps.expand(text), showUserMessage))
}