Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(shell): async shell jobs for web and Lite (#4580)

nullptr-error92dfa6a1eef2
diff --git a/frontend/src/api/chat/background-work.ts b/frontend/src/api/chat/background-work.ts
new file mode 100644
index 000000000..98a1f791b
--- /dev/null
+++ b/frontend/src/api/chat/background-work.ts
@@ -0,0 +1,112 @@
+import { get, post } from '../../utils/request'
+
+/** States the UI and the model both see. `unknown` stays on the server. */
+export type BackgroundWorkState =
+ | 'running'
+ | 'succeeded'
+ | 'failed'
+ | 'cancelled'
+ | 'lost'
+
+/**
+ * List rows from GET .../background-work.
+ * The handler never fills parent_tool_call_id, so it is not part of this type.
+ */
+export interface BackgroundWorkItem {
+ work_id: string
+ kind: string
+ title: string
+ state: BackgroundWorkState
+ started_at?: string
+ finished_at?: string
+}
+
+/** Byte windows on service.ShellJobPayload. */
+export interface ShellJobByteOffsets {
+ stdout: number
+ stderr: number
+}
+
+/**
+ * JSON shape of service.ShellJobPayload, returned as detail `result`.
+ * A user-facing read must not send run_id; the server keeps RunID empty so
+ * the model's incremental cursor does not move.
+ */
+export interface ShellJobPayload {
+ display_type: string
+ work_id: string
+ state: string
+ session_id: string
+ command: string
+ work_dir: string
+ exit_code?: number
+ duration_ms: number
+ started_at: string
+ stdout: string
+ stderr: string
+ from_bytes: ShellJobByteOffsets
+ total_bytes: ShellJobByteOffsets
+ omitted_bytes: ShellJobByteOffsets
+ truncated: boolean
+ note?: string
+}
+
+export interface BackgroundWorkDetail {
+ work_id: string
+ state: BackgroundWorkState
+ title: string
+ result: ShellJobPayload | null
+}
+
+interface ApiData<T> {
+ success?: boolean
+ data?: T
+}
+
+function sessionWorkPath(sessionId: string, workId?: string): string {
+ const base = `/api/v1/sessions/${encodeURIComponent(sessionId)}/background-work`
+ return workId === undefined ? base : `${base}/${encodeURIComponent(workId)}`
+}
+
+export async function listBackgroundWork(
+ sessionId: string,
+ signal?: AbortSignal,
+): Promise<BackgroundWorkItem[]> {
+ const res = await get<ApiData<BackgroundWorkItem[]>>(sessionWorkPath(sessionId), signal ? { signal } : undefined)
+ return Array.isArray(res?.data) ? res.data : []
+}
+
+export async function getBackgroundWork(
+ sessionId: string,
+ workId: string,
+): Promise<BackgroundWorkDetail> {
+ const res = await get<ApiData<BackgroundWorkDetail>>(sessionWorkPath(sessionId, workId))
+ if (!res?.data) {
+ throw new Error('background work not found')
+ }
+ return res.data
+}
+
+const cancelInflight = new Map<string, Promise<{ work_id: string; state: string }>>()
+
+export function cancelBackgroundWork(
+ sessionId: string,
+ workId: string,
+): Promise<{ work_id: string; state: string }> {
+ const key = `${sessionId}\0${workId}`
+ const pending = cancelInflight.get(key)
+ if (pending) return pending
+ const request = post<ApiData<{ work_id: string; state: string }>>(
+ `${sessionWorkPath(sessionId, workId)}/cancel`,
+ {},
+ ).then((res) => {
+ if (!res?.data?.work_id) {
+ throw new Error('background work not found')
+ }
+ return res.data
+ }).finally(() => {
+ if (cancelInflight.get(key) === request) cancelInflight.delete(key)
+ })
+ cancelInflight.set(key, request)
+ return request
+}
diff --git a/frontend/src/components/Input-field.vue b/frontend/src/components/Input-field.vue
index c213a6aee..a50f46e9f 100644
--- a/frontend/src/components/Input-field.vue
+++ b/frontend/src/components/Input-field.vue
@@ -2990,7 +2990,9 @@ defineExpose({
</svg>
</div>
</t-tooltip>
+ <slot name="background-work" />
</template>
+ <slot v-if="embeddedMode" name="background-work" />
<slot name="controls" />
diff --git a/frontend/src/composables/useChatStreamHandler.test.mjs b/frontend/src/composables/useChatStreamHandler.test.mjs
index 5eac5abcb..fb65f4222 100644
--- a/frontend/src/composables/useChatStreamHandler.test.mjs
+++ b/frontend/src/composables/useChatStreamHandler.test.mjs
@@ -294,3 +294,47 @@ test('the live answer path carries truncated onto the event and the message', ()
assert.equal(message._eventMap.get('r1').truncated, true)
assert.equal(message.truncated, true)
})
+
+test('work_event command output attaches to the pending async tool card only', () => {
+ const start = source.indexOf("case 'work_event': {")
+ const block = source.slice(start, source.indexOf("case 'work_finished':", start))
+ const asyncCard = { type: 'tool_call', tool_name: 'shell_exec_async', tool_call_id: 'tc9', pending: true }
+ const syncCard = { type: 'tool_call', tool_name: 'shell_exec', tool_call_id: 'tc1', pending: true }
+ const message = { agentEventStream: [syncCard, asyncCard] }
+ const process = vm.runInNewContext(ts.transpile(
+ `(payload) => { const dataPayload = payload; const toolCallId = payload.inner?.data?.tool_call_id; switch ('work_event') { ${block} } }`),
+ { message })
+ process({ work_id: 'sj_0123', kind: 'shell',
+ inner: { response_type: 'command_output', data: { tool_call_id: 'tc9', output: 'building', done: false } } })
+ assert.equal(asyncCard.command_output?.output, 'building')
+ assert.equal(asyncCard.pending, true)
+ assert.equal(syncCard.command_output, undefined, '不串到同步 shell 卡')
+})
+
+test('work_event command output still attaches after the async tool has returned', () => {
+ const start = source.indexOf("case 'work_event': {")
+ const block = source.slice(start, source.indexOf("case 'work_finished':", start))
+ const asyncCard = {
+ type: 'tool_call', tool_name: 'shell_exec_async', tool_call_id: 'tc9', pending: false,
+ }
+ const message = { agentEventStream: [asyncCard] }
+ const process = vm.runInNewContext(ts.transpile(
+ `(payload) => { const dataPayload = payload; switch ('work_event') { ${block} } }`),
+ { message })
+ process({
+ work_id: 'sj_0123', kind: 'shell',
+ inner: { response_type: 'command_output', data: { tool_call_id: 'tc9', output: 'still going', done: false } },
+ })
+ assert.equal(asyncCard.command_output?.output, 'still going')
+ assert.equal(asyncCard.pending, false)
+})
+
+test('work_finished records terminal state without ending the run', () => {
+ const start = source.indexOf("case 'work_finished': {")
+ const block = source.slice(start, source.indexOf('case ', start + 20) === -1
+ ? source.length : source.lastIndexOf('}', source.indexOf('case ', start + 20)))
+ const message = { background_work: {} }
+ const process = vm.runInNewContext(ts.transpile(`(payload) => { ${block} }`), { message })
+ process({ work_id: 'sj_0123', kind: 'shell', state: 'succeeded' })
+ assert.equal(message.background_work['sj_0123']?.state, 'succeeded')
+})
diff --git a/frontend/src/composables/useChatStreamHandler.ts b/frontend/src/composables/useChatStreamHandler.ts
index 4c22f8d36..b4cf1fe64 100644
--- a/frontend/src/composables/useChatStreamHandler.ts
+++ b/frontend/src/composables/useChatStreamHandler.ts
@@ -800,6 +800,15 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
}
if (!message) {
+ // Background notices attach to a turn that already exists. Opening a
+ // shell here would start a new assistant turn after the parent run.
+ if (
+ data.response_type === 'work_event' ||
+ data.response_type === 'work_started' ||
+ data.response_type === 'work_finished'
+ ) {
+ return
+ }
const newMsg: ChatMessage = {
id: dataId,
request_id: dataId,
@@ -851,6 +860,9 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
.filter(m => m.role === 'assistant' && (m === message || (turnID && persistedAssistantId(m) === turnID)))
.flatMap(m => (m.agentEventStream as ChatMessage[] | undefined) || [])
}
+ // work_* cases read `payload` so the source-extracted tests, which call
+ // the case with the event object, and this handler share one binding.
+ const payload = dataPayload
switch (responseType) {
case 'collaboration_status': {
@@ -1195,6 +1207,44 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
}
break
}
+ case 'work_event': {
+ const inner = dataPayload?.inner as { response_type?: string; data?: ChatMessage } | undefined
+ if (inner?.response_type === 'command_output' && inner.data?.tool_call_id) {
+ // The async tool returns at the yield window, so later previews
+ // arrive after pending is cleared. Attach them to that card anyway.
+ // done on the preview hides the live tail; it does not complete the tool.
+ const toolCallId = inner.data.tool_call_id
+ const tool = (message.agentEventStream as ChatMessage[] | undefined)?.find(
+ event => event.type === 'tool_call' && event.tool_call_id === toolCallId
+ && (event.tool_name === 'shell_exec' || event.tool_name === 'shell_exec_async'),
+ )
+ if (tool) {
+ tool.command_output = { ...inner.data }
+ }
+ }
+ break
+ }
+ case 'work_started':
+ case 'work_finished': {
+ const event = payload as ChatMessage | undefined
+ const workId = typeof event?.work_id === 'string' ? event.work_id : ''
+ if (workId) {
+ if (!message.background_work) message.background_work = {}
+ const backgroundWork = message.background_work as Record<string, {
+ state: unknown
+ title: unknown
+ kind: unknown
+ payload: unknown
+ }>
+ backgroundWork[workId] = {
+ state: (event?.state as string | undefined) ?? 'running',
+ title: event?.title,
+ kind: event?.kind,
+ payload: event?.payload,
+ }
+ }
+ }
+ break
case 'tool_result':
case 'error': {
if (dataPayload) {
@@ -1591,6 +1641,9 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
data.response_type === 'tool_call' ||
data.response_type === 'tool_result' ||
data.response_type === 'command_output' ||
+ data.response_type === 'work_event' ||
+ data.response_type === 'work_started' ||
+ data.response_type === 'work_finished' ||
data.response_type === 'reflection' ||
data.response_type === 'artifacts_pending' ||
data.response_type === 'context_compacted' ||
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts
index 5a8ca88d8..a84b17fd0 100755
--- a/frontend/src/i18n/locales/en-US.ts
+++ b/frontend/src/i18n/locales/en-US.ts
@@ -4483,6 +4483,12 @@ export default {
noKnowledgeBases: 'No knowledge bases available',
enterDescription: 'Enter description',
rawOutputLabel: 'Raw output',
+ backgroundWork: {
+ running: 'No background tasks | {count} background task | {count} background tasks',
+ stop: 'Stop',
+ loading: 'Loading output…',
+ unavailable: 'Output is unavailable',
+ },
wikiWritePageTitle: 'Wiki Page Write',
wikiReplaceTextTitle: 'Wiki Text Replace',
wikiRenamePageTitle: 'Wiki Page Rename',
diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts
index 2ece8545e..cbd323882 100644
--- a/frontend/src/i18n/locales/ja-JP.ts
+++ b/frontend/src/i18n/locales/ja-JP.ts
@@ -4483,6 +4483,12 @@ export default {
noKnowledgeBases: '利用可能なナレッジベースがありません',
enterDescription: '説明を入力',
rawOutputLabel: '生の出力',
+ backgroundWork: {
+ running: 'バックグラウンドの作業はありません | バックグラウンドの作業 {count} 件 | バックグラウンドの作業 {count} 件',
+ stop: '停止',
+ loading: '出力を読み込んでいます…',
+ unavailable: '出力を表示できません',
+ },
wikiWritePageTitle: 'Wikiページの書き込み',
wikiReplaceTextTitle: 'Wikiテキストの置換',
wikiRenamePageTitle: 'Wikiページの名前変更',
diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts
index 4fa9a5814..b1c37c0e0 100755
--- a/frontend/src/i18n/locales/ko-KR.ts
+++ b/frontend/src/i18n/locales/ko-KR.ts
@@ -4834,6 +4834,12 @@ export default {
noKnowledgeBases: '사용 가능한 지식베이스 없음',
enterDescription: '설명 입력',
rawOutputLabel: '원시 출력',
+ backgroundWork: {
+ running: '백그라운드 작업 없음 | 백그라운드 작업 {count}개 | 백그라운드 작업 {count}개',
+ stop: '중지',
+ loading: '출력을 불러오는 중…',
+ unavailable: '출력을 볼 수 없습니다',
+ },
wikiWritePageTitle: 'Wiki 페이지 작성',
wikiReplaceTextTitle: 'Wiki 텍스트 교체',
wikiRenamePageTitle: 'Wiki 페이지 이름 변경',
diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts
index 4ec9782ac..a0522c3e6 100755
--- a/frontend/src/i18n/locales/ru-RU.ts
+++ b/frontend/src/i18n/locales/ru-RU.ts
@@ -4834,6 +4834,12 @@ export default {
noKnowledgeBases: 'Нет доступных баз знаний',
enterDescription: 'Введите описание',
rawOutputLabel: 'Исходный вывод',
+ backgroundWork: {
+ running: 'Нет фоновых задач | {count} фоновая задача | {count} фоновых задач',
+ stop: 'Остановить',
+ loading: 'Загрузка вывода…',
+ unavailable: 'Вывод недоступен',
+ },
wikiWritePageTitle: 'Запись страницы Wiki',
wikiReplaceTextTitle: 'Замена текста Wiki',
wikiRenamePageTitle: 'Переименование страницы Wiki',
diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts
index 5319b2ec4..e24ad26e9 100755
--- a/frontend/src/i18n/locales/zh-CN.ts
+++ b/frontend/src/i18n/locales/zh-CN.ts
@@ -4836,6 +4836,12 @@ export default {
noKnowledgeBases: '没有可用的知识库',
enterDescription: '输入描述',
rawOutputLabel: '原始输出',
+ backgroundWork: {
+ running: '没有后台任务 | {count} 个后台任务 | {count} 个后台任务',
+ stop: '停止',
+ loading: '正在加载输出…',
+ unavailable: '暂时无法查看输出',
+ },
wikiWritePageTitle: 'Wiki 页面写入',
wikiReplaceTextTitle: 'Wiki 文本替换',
wikiRenamePageTitle: 'Wiki 页面重命名',
diff --git a/frontend/src/i18n/locales/zh-TW.ts b/frontend/src/i18n/locales/zh-TW.ts
index e0fba751f..4bf6cd5dc 100644
--- a/frontend/src/i18n/locales/zh-TW.ts
+++ b/frontend/src/i18n/locales/zh-TW.ts
@@ -4836,6 +4836,12 @@ export default {
noKnowledgeBases: '沒有可用的知識庫',
enterDescription: '輸入描述',
rawOutputLabel: '原始輸出',
+ backgroundWork: {
+ running: '沒有背景工作 | {count} 個背景工作 | {count} 個背景工作',
+ stop: '停止',
+ loading: '正在載入輸出…',
+ unavailable: '暫時無法查看輸出',
+ },
wikiWritePageTitle: 'Wiki 頁面寫入',
wikiReplaceTextTitle: 'Wiki 文字取代',
wikiRenamePageTitle: 'Wiki 頁面重新命名',
diff --git a/frontend/src/types/tool-results.ts b/frontend/src/types/tool-results.ts
index 30c2f22d8..cacba70a1 100644
--- a/frontend/src/types/tool-results.ts
+++ b/frontend/src/types/tool-results.ts
@@ -37,7 +37,8 @@ export type DisplayType =
| 'find_operations'
| 'run_operation'
| 'search_weknora_docs'
- | 'read_weknora_doc';
+ | 'read_weknora_doc'
+ | 'work_wait';
// Search result item
export interface SearchResultItem {
diff --git a/frontend/src/utils/backgroundWork.test.ts b/frontend/src/utils/backgroundWork.test.ts
new file mode 100644
index 000000000..aa2388f03
--- /dev/null
+++ b/frontend/src/utils/backgroundWork.test.ts
@@ -0,0 +1,129 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import {
+ mergeBackgroundWork,
+ createBackgroundWorkPoller,
+} from './backgroundWork.ts'
+import type { BackgroundWorkItem } from '../api/chat/background-work.ts'
+import { get, post } from '../utils/request.ts'
+
+test('merge keeps newest state per work_id, running first', () => {
+ const existing: BackgroundWorkItem[] = [
+ { work_id: 'sj_a', kind: 'shell', title: 'build', state: 'running' },
+ { work_id: 'sj_b', kind: 'shell', title: 'tests', state: 'running' },
+ ]
+ const incoming: BackgroundWorkItem[] = [
+ { work_id: 'sj_b', kind: 'shell', title: 'tests', state: 'succeeded', finished_at: '2026-10-16T00:00:00Z' },
+ { work_id: 'sj_c', kind: 'shell', title: 'new', state: 'running' },
+ ]
+ const merged = mergeBackgroundWork(existing, incoming)
+ assert.deepEqual(merged.map(i => i.work_id), ['sj_a', 'sj_c', 'sj_b'])
+ assert.equal(merged[2].state, 'succeeded', '新数据覆盖旧状态')
+})
+
+test('poller retries after a list error instead of stopping', async () => {
+ let calls = 0
+ const seen: BackgroundWorkItem[][] = []
+ const poller = createBackgroundWorkPoller({
+ sessionId: 's1',
+ intervalMs: 15,
+ list: async () => {
+ calls += 1
+ if (calls === 1) throw new Error('unavailable')
+ return [{ work_id: 'sj_a', kind: 'shell', title: 'build', state: 'running' }]
+ },
+ onUpdate: (items) => { seen.push(items) },
+ onError: () => {},
+ })
+ poller.start()
+ await new Promise((resolve) => setTimeout(resolve, 80))
+ poller.stop()
+ assert.ok(calls >= 2, '5xx 之后还要再拉')
+ assert.ok(seen.some((items) => items.some((item) => item.work_id === 'sj_a')))
+})
+
+test('poller stops on empty list and ignores stale session responses', async () => {
+ const calls: string[] = []
+ let respond: (v: any) => void = () => {}
+ const gate = new Promise<void>(resolve => { respond = resolve })
+ const fakeGet = (url: string) => {
+ calls.push(url)
+ return gate.then(() => ({ data: [] }))
+ }
+ const poller = createBackgroundWorkPoller({
+ sessionId: 's1',
+ intervalMs: 10,
+ list: fakeGet,
+ onUpdate: () => {},
+ })
+ poller.start()
+ poller.stop() // 切会话:立即停
+ respond(undefined)
+ await new Promise(r => setTimeout(r, 30))
+ assert.equal(calls.length, 1, '停止后不再发请求')
+})
+
+function deferred<T>() {
+ let resolve!: (value: T) => void
+ let reject!: (reason: Error) => void
+ const promise = new Promise<T>((yes, no) => { resolve = yes; reject = no })
+ return { promise, resolve, reject }
+}
+
+for (const staleItems of [[], [{ work_id: 'sj_old', kind: 'shell', title: 'old', state: 'running' }]] as BackgroundWorkItem[][]) {
+ test(`manual refresh cannot replace another session or stop its polling (${staleItems.length} old items)`, async () => {
+ let session = 'A'
+ const requests: { session: string; response: ReturnType<typeof deferred<BackgroundWorkItem[]>> }[] = []
+ const seen: BackgroundWorkItem[][] = []
+ const poller = createBackgroundWorkPoller({
+ sessionId: () => session,
+ intervalMs: 60_000,
+ list: (id) => {
+ const response = deferred<BackgroundWorkItem[]>()
+ requests.push({ session: id, response })
+ return response.promise // Deliberately ignore AbortSignal, like a late network response.
+ },
+ onUpdate: items => seen.push(items),
+ })
+ try {
+ const refreshA = poller.refresh()
+ session = 'B'
+ poller.stop()
+ poller.start()
+ const current: BackgroundWorkItem[] = [{ work_id: 'sj_new', kind: 'shell', state: 'running', title: 'new' }]
+ requests[1].response.resolve(current)
+ await Promise.resolve()
+ requests[0].response.resolve(staleItems)
+ await refreshA
+ assert.deepEqual(seen, [current])
+ const count = requests.length
+ poller.start()
+ assert.equal(requests.length, count, 'stale empty response must not stop B polling')
+ } finally { poller.dispose() }
+ })
+}
+
+test('manual refresh is invalidated on switch away and back, unmount, and newer refresh', async () => {
+ const requests: ReturnType<typeof deferred<BackgroundWorkItem[]>>[] = []
+ const seen: BackgroundWorkItem[][] = []
+ const poller = createBackgroundWorkPoller({
+ sessionId: 'A',
+ list: () => { const response = deferred<BackgroundWorkItem[]>(); requests.push(response); return response.promise },
+ onUpdate: items => seen.push(items),
+ })
+ const first = poller.refresh()
+ poller.stop() // A -> B -> A must invalidate the original A request too.
+ const second = poller.refresh()
+ const third = poller.refresh()
+ requests[1].resolve([{ work_id: 'sj_old', kind: 'shell', state: 'running', title: '' }])
+ requests[0].resolve([])
+ await Promise.all([first, second])
+ assert.deepEqual(seen, [])
+ poller.dispose()
+ requests[2].resolve([{ work_id: 'sj_late', kind: 'shell', state: 'running', title: '' }])
+ await third
+ await poller.refresh() // A cancellation completing after unmount must not restart polling.
+ poller.start()
+ assert.equal(requests.length, 3)
+ assert.deepEqual(seen, [])
+})
diff --git a/frontend/src/utils/backgroundWork.ts b/frontend/src/utils/backgroundWork.ts
new file mode 100644
index 000000000..33b46bfab
--- /dev/null
+++ b/frontend/src/utils/backgroundWork.ts
@@ -0,0 +1,140 @@
+import type { BackgroundWorkItem } from '../api/chat/background-work'
+
+function itemTime(item: BackgroundWorkItem): number {
+ const parsed = Date.parse(item.started_at ?? item.finished_at ?? '')
+ return Number.isFinite(parsed) ? parsed : 0
+}
+
+function sameWorkItem(a: BackgroundWorkItem, b: BackgroundWorkItem): boolean {
+ return a.work_id === b.work_id
+ && a.kind === b.kind
+ && a.title === b.title
+ && a.state === b.state
+ && a.started_at === b.started_at
+ && a.finished_at === b.finished_at
+}
+
+function compareWork(a: BackgroundWorkItem, b: BackgroundWorkItem): number {
+ const rank = (item: BackgroundWorkItem) => item.state === 'running' ? 0 : 1
+ const byState = rank(a) - rank(b)
+ if (byState !== 0) return byState
+ return itemTime(a) - itemTime(b)
+}
+
+/**
+ * Merge by work_id. Incoming wins. Unchanged items keep their object identity,
+ * and an unchanged list keeps the same array, so a poll tick does not rerender.
+ * Running rows stay above terminal ones; within a group, earlier times come first.
+ */
+export function mergeBackgroundWork(
+ existing: BackgroundWorkItem[],
+ incoming: BackgroundWorkItem[],
+): BackgroundWorkItem[] {
+ const byID = new Map<string, BackgroundWorkItem>()
+ for (const item of existing) byID.set(item.work_id, item)
+ for (const item of incoming) {
+ const prev = byID.get(item.work_id)
+ if (prev && sameWorkItem(prev, item)) continue
+ byID.set(item.work_id, item)
+ }
+ const sorted = [...byID.values()].sort(compareWork)
+ if (sorted.length === existing.length && sorted.every((item, index) => item === existing[index])) {
+ return existing
+ }
+ return sorted
+}
+
+export interface PollerOptions {
+ /** Current session, or a getter so a switch can drop the in-flight response. */
+ sessionId: string | (() => string)
+ intervalMs?: number
+ list: (
+ sessionId: string,
+ signal: AbortSignal,
+ ) => Promise<BackgroundWorkItem[] | { data?: BackgroundWorkItem[] | null }>
+ onUpdate: (items: BackgroundWorkItem[]) => void
+ onError?: (message: string) => void
+}
+
+function readSessionId(sessionId: PollerOptions['sessionId']): string {
+ return typeof sessionId === 'function' ? sessionId() : sessionId
+}
+
+function asWorkItems(
+ value: BackgroundWorkItem[] | { data?: BackgroundWorkItem[] | null } | null | undefined,
+): BackgroundWorkItem[] {
+ if (Array.isArray(value)) return value
+ if (value && Array.isArray(value.data)) return value.data
+ return []
+}
+
+export function createBackgroundWorkPoller(options: PollerOptions): {
+ start(): void; stop(): void; refresh(): Promise<void>; dispose(): void
+} {
+ const intervalMs = options.intervalMs ?? 5000
+ let timer: ReturnType<typeof setTimeout> | null = null
+ let abort: AbortController | null = null
+ let running = false
+ let disposed = false
+
+ const stop = () => {
+ running = false
+ if (timer) {
+ clearTimeout(timer)
+ timer = null
+ }
+ abort?.abort()
+ abort = null
+ }
+
+ const tick = async () => {
+ if (!running || disposed) return
+ const sessionId = readSessionId(options.sessionId)
+ const controller = new AbortController()
+ abort?.abort()
+ abort = controller
+ try {
+ const raw = await options.list(sessionId, controller.signal)
+ if (!running || controller.signal.aborted) return
+ if (readSessionId(options.sessionId) !== sessionId) {
+ void tick()
+ return
+ }
+ const items = asWorkItems(raw)
+ options.onUpdate(items)
+ if (!running) return
+ if (items.length === 0) {
+ running = false
+ timer = null
+ return
+ }
+ timer = setTimeout(() => { void tick() }, intervalMs)
+ } catch (err) {
+ if (!running || controller.signal.aborted) return
+ options.onError?.(err instanceof Error ? err.message : String(err))
+ if (!running) return
+ timer = setTimeout(() => { void tick() }, intervalMs)
+ }
+ }
+
+ return {
+ start() {
+ if (running || disposed) return
+ running = true
+ void tick()
+ },
+ stop,
+ async refresh() {
+ if (disposed) return
+ // Manual refreshes use the same abort/generation boundary as polling.
+ // A newer request, session switch or unmount invalidates their results.
+ stop()
+ running = true
+ await tick()
+ },
+ dispose() {
+ disposed = true
+ stop()
+ },
+ }
+}
diff --git a/frontend/src/utils/shellExecResult.test.ts b/frontend/src/utils/shellExecResult.test.ts
index 6183d1613..c5c7e4af9 100644
--- a/frontend/src/utils/shellExecResult.test.ts
+++ b/frontend/src/utils/shellExecResult.test.ts
@@ -44,6 +44,19 @@ test('buildShellExecView parses legacy markdown output when streams were strippe
assert.equal(view.stderr, 'warn')
})
+test('shell job payload renders like a shell result with work metadata', () => {
+ const view = buildShellExecView({
+ display_type: 'shell_exec', work_id: 'sj_0123', state: 'succeeded',
+ command: 'make build', work_dir: '/workspace', exit_code: 0,
+ duration_ms: 1200, stdout: 'ok', stderr: '', truncated: false,
+ from_bytes: { stdout: 0, stderr: 0 }, total_bytes: { stdout: 2, stderr: 0 },
+ omitted_bytes: { stdout: 0, stderr: 0 },
+ }, {})
+ assert.equal(view.command, 'make build')
+ assert.equal(view.exitCode, 0)
+ assert.equal(view.stdout, 'ok')
+})
+
test('previewShellCommand collapses whitespace and truncates', () => {
assert.equal(previewShellCommand('ls -la'), 'ls -la')
assert.equal(previewShellCommand('abcdefghij', 8), 'abcdefg…')
diff --git a/frontend/src/utils/shellProgress.test.ts b/frontend/src/utils/shellProgress.test.ts
new file mode 100644
index 000000000..370c843e4
--- /dev/null
+++ b/frontend/src/utils/shellProgress.test.ts
@@ -0,0 +1,21 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { showLiveShellOutput } from './shellProgress.ts'
+
+test('async shell progress stays visible after the tool call has returned', () => {
+ assert.equal(showLiveShellOutput({
+ tool_name: 'shell_exec_async',
+ pending: false,
+ command_output: { done: false },
+ }), true)
+ assert.equal(showLiveShellOutput({
+ tool_name: 'shell_exec',
+ pending: false,
+ command_output: { done: false },
+ }), false, '同步命令只在工具还在跑时显示进度')
+ assert.equal(showLiveShellOutput({
+ tool_name: 'shell_exec_async',
+ pending: false,
+ command_output: { done: true },
+ }), false)
+})
diff --git a/frontend/src/utils/shellProgress.ts b/frontend/src/utils/shellProgress.ts
new file mode 100644
index 000000000..948ed12dc
--- /dev/null
+++ b/frontend/src/utils/shellProgress.ts
@@ -0,0 +1,11 @@
+/** Live tail under a shell card. Async jobs keep streaming after the tool returns. */
+export function showLiveShellOutput(event: {
+ tool_name?: string
+ pending?: boolean
+ command_output?: { done?: boolean } | null
+} | null | undefined): boolean {
+ const output = event?.command_output
+ if (!output || output.done) return false
+ if (event?.tool_name === 'shell_exec_async') return true
+ return event?.tool_name === 'shell_exec' && event.pending === true
+}
diff --git a/frontend/src/views/chat/automationConversation.test.ts b/frontend/src/views/chat/automationConversation.test.ts
index f22d4fc07..ed2ebf177 100644
--- a/frontend/src/views/chat/automationConversation.test.ts
+++ b/frontend/src/views/chat/automationConversation.test.ts
@@ -39,7 +39,7 @@ function render(kind: string | null, embeddedMode = false) {
embeddedMode, userHasScrolledUp: false, onClickScrollToBottom() {}, t: (key: string) => key,
focusComposerOnMount: false, isReplying: false, composerLocked: false, session_id: 'session',
currentAssistantMessageId: '', modelId: '', defaultModelId: '', steerQueue: [],
- isAgentStreamSession: () => true, $slots: {},
+ isAgentStreamSession: () => true, $slots: {}, backgroundItems: [],
}, []))
}
diff --git a/frontend/src/views/chat/backgroundWorkPolling.test.ts b/frontend/src/views/chat/backgroundWorkPolling.test.ts
new file mode 100644
index 000000000..d30f8f1b5
--- /dev/null
+++ b/frontend/src/views/chat/backgroundWorkPolling.test.ts
@@ -0,0 +1,40 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { runInNewContext } from 'node:vm'
+import test from 'node:test'
+import { createBackgroundWorkPoller } from '../../utils/backgroundWork.ts'
+
+test('chat page retries its first failed background query and stops after a confirmed empty list', async (t) => {
+ const source = readFileSync(new URL('./index.vue', import.meta.url), 'utf8')
+ const start = source.indexOf('const backgroundPoller = createBackgroundWorkPoller(')
+ const end = source.indexOf('\nfunction startBackgroundWork()', start)
+ assert.ok(start >= 0 && end > start)
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ let calls = 0
+ const backgroundItems = { value: [] as any[] }
+ const running = [{ work_id: 'sj_0123', kind: 'shell', title: 'build', state: 'running' }]
+ const poller = runInNewContext(`${source.slice(start, end)}\nbackgroundPoller`, {
+ session_id: { value: 's' }, backgroundItems,
+ createBackgroundWorkPoller: (options: Parameters<typeof createBackgroundWorkPoller>[0]) =>
+ createBackgroundWorkPoller({ ...options, intervalMs: 10 }),
+ listBackgroundWork: async () => {
+ calls++
+ if (calls === 1) throw new Error('HTTP 503')
+ return calls === 2 ? running : []
+ },
+ replaceBackgroundItems: (items: any[]) => { backgroundItems.value = items },
+ })
+ try {
+ poller.start()
+ await Promise.resolve()
+ t.mock.timers.tick(10)
+ await Promise.resolve()
+ assert.equal(calls, 2, 'the real page callback must not stop retries while its initial list is empty')
+ assert.deepEqual(backgroundItems.value, running)
+ t.mock.timers.tick(10)
+ await Promise.resolve()
+ assert.deepEqual(backgroundItems.value, [])
+ t.mock.timers.tick(100)
+ assert.equal(calls, 3, 'a successful empty response ends polling')
+ } finally { poller.dispose() }
+})
diff --git a/frontend/src/views/chat/components/AgentStreamDisplay.vue b/frontend/src/views/chat/components/AgentStreamDisplay.vue
index 2f1ba26dc..6d73129db 100644
--- a/frontend/src/views/chat/components/AgentStreamDisplay.vue
+++ b/frontend/src/views/chat/components/AgentStreamDisplay.vue
@@ -242,7 +242,7 @@
</div>
<SandboxCommandProgress
- v-if="event.tool_name === 'shell_exec' && event.pending && event.command_output && !event.command_output.done"
+ v-if="showLiveShellOutput(event)"
:progress="event.command_output"
/>
@@ -627,7 +627,7 @@
</div>
<SandboxCommandProgress
- v-if="event.tool_name === 'shell_exec' && event.pending && event.command_output && !event.command_output.done"
+ v-if="showLiveShellOutput(event)"
:progress="event.command_output"
/>
@@ -807,6 +807,7 @@ import {
skillScriptTitleCommand,
} from '@/utils/skillToolDisplay';
import { previewShellCommand } from '@/utils/shellExecResult';
+import { showLiveShellOutput } from '@/utils/shellProgress';
import type { DisplayType } from '@/types/tool-results';
import { parseWikiToolReferences } from '@/utils/wikiToolReferences';
import {
@@ -891,6 +892,7 @@ const TOOL_NAME_KEYS: Record<string, string> = {
write_sandbox_file: 'agentStream.tools.writeSandboxFile',
edit_sandbox_file: 'agentStream.tools.editSandboxFile',
shell_exec: 'agentStream.tools.shellExec',
+ shell_exec_async: 'agentStream.tools.shellExec',
local_browser: 'localBrowser.local',
data_analysis: 'agentStream.tools.dataAnalysis',
data_schema: 'agentStream.tools.dataSchema',
@@ -1388,9 +1390,11 @@ const resolveToolDisplayType = (event: any): DisplayType | undefined => {
const catalogType = getCatalogToolDisplayType(event?.tool_name)
if (catalogType) return catalogType
if (event?.display_type) return event.display_type as DisplayType
- if (event?.tool_name === 'shell_exec' || event?.tool_name === 'execute_skill_script') {
+ if (event?.tool_name === 'shell_exec' || event?.tool_name === 'shell_exec_async' ||
+ event?.tool_name === 'execute_skill_script') {
return 'shell_exec'
}
+ if (event?.tool_name === 'work_wait') return 'work_wait'
if (event?.tool_name === 'list_sandbox_files' && event?.success !== false) {
return 'list_sandbox_files'
}
@@ -3148,7 +3152,7 @@ const getToolTitle = (event: any): string => {
const name = getLocalizedToolName(event.tool_name);
return `${formatToolTitleWithDetail(name, getSandboxToolDetail(event))}...`;
}
- if (event.tool_name === 'shell_exec') {
+ if (event.tool_name === 'shell_exec' || event.tool_name === 'shell_exec_async') {
return t('agentStream.toolStatus.shellExecRunning');
}
const localizedName = getLocalizedToolName(event.tool_name);
@@ -3310,7 +3314,7 @@ const getToolDescription = (event: any): string => {
const name = getLocalizedToolName(event.tool_name);
return `${formatToolTitleWithDetail(name, getSandboxToolDetail(event))}...`;
}
- if (event.tool_name === 'shell_exec') {
+ if (event.tool_name === 'shell_exec' || event.tool_name === 'shell_exec_async') {
return t('agentStream.toolStatus.shellExecRunning');
}
const localizedName = getLocalizedToolName(event.tool_name);
diff --git a/frontend/src/views/chat/components/BackgroundWorkChip.test.ts b/frontend/src/views/chat/components/BackgroundWorkChip.test.ts
new file mode 100644
index 000000000..90acc3792
--- /dev/null
+++ b/frontend/src/views/chat/components/BackgroundWorkChip.test.ts
@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { runInNewContext } from 'node:vm'
+import test from 'node:test'
+import { compileScript, parse } from '@vue/compiler-sfc'
+import * as vue from 'vue'
+import ts from 'typescript'
+
+const source = readFileSync(new URL('./BackgroundWorkChip.vue', import.meta.url), 'utf8')
+const { descriptor } = parse(source)
+const compiled = compileScript(descriptor, { id: 'background-work-chip' })
+const exports: any = {}
+runInNewContext(ts.transpileModule(compiled.content, {
+ compilerOptions: { module: ts.ModuleKind.CommonJS },
+}).outputText, {
+ exports, Date, Set, setInterval, clearInterval,
+ require: () => ({ ...vue, onUnmounted() {} }),
+})
+
+for (const rejected of [false, true]) {
+ test(`cancel button re-enables when request settles (${rejected ? 'failure' : 'success'})`, async () => {
+ let finish!: () => void
+ let fail!: (reason: Error) => void
+ let calls = 0
+ const props = vue.reactive({
+ items: [{ work_id: 'sj_0123', kind: 'shell', state: 'running', title: 'build' }],
+ cancelWork: () => {
+ calls++
+ return new Promise<void>((resolve, reject) => { finish = resolve; fail = reject })
+ },
+ })
+ const scope = vue.effectScope()
+ const state = scope.run(() => exports.default.setup(props, { expose() {}, emit() {} }))
+ try {
+ const pending = state.onCancel('sj_0123')
+ assert.equal(state.cancelling.value.has('sj_0123'), true)
+ await state.onCancel('sj_0123')
+ assert.equal(calls, 1, 'duplicate clicks are suppressed while pending')
+ if (rejected) {
+ fail(new Error('HTTP 503'))
+ await assert.rejects(pending, /503/)
+ } else { finish(); await pending }
+ assert.equal(props.items[0].state, 'running', 'poll may still show a running job')
+ assert.equal(state.cancelling.value.has('sj_0123'), false)
+ const retry = state.onCancel('sj_0123')
+ assert.equal(calls, 2)
+ finish()
+ await retry
+ } finally { scope.stop() }
+ })
+}
diff --git a/frontend/src/views/chat/components/BackgroundWorkChip.vue b/frontend/src/views/chat/components/BackgroundWorkChip.vue
new file mode 100644
index 000000000..0adb8416c
--- /dev/null
+++ b/frontend/src/views/chat/components/BackgroundWorkChip.vue
@@ -0,0 +1,218 @@
+<template>
+ <t-popup
+ v-if="running.length"
+ v-model:visible="open"
+ trigger="click"
+ placement="top-left"
+ :overlay-inner-style="{ padding: '6px' }"
+ >
+ <button
+ type="button"
+ class="chip-count"
+ :aria-expanded="open"
+ :aria-label="$t('chat.backgroundWork.running', { count: running.length }, running.length)"
+ >
+ <span class="chip-badge" aria-hidden="true" />
+ <span>{{ $t('chat.backgroundWork.running', { count: running.length }, running.length) }}</span>
+ </button>
+ <template #content>
+ <div class="work-card" role="list">
+ <div v-for="item in running" :key="item.work_id" class="work-row" role="listitem">
+ <button type="button" class="work-main" @click="openOutput(item.work_id)">
+ <span class="work-title">{{ item.title || item.work_id }}</span>
+ <span v-if="elapsed(item)" class="work-duration">{{ elapsed(item) }}</span>
+ </button>
+ <button
+ type="button"
+ class="work-stop"
+ :disabled="cancelling.has(item.work_id)"
+ @click="onCancel(item.work_id)"
+ >{{ $t('chat.backgroundWork.stop') }}</button>
+ </div>
+ </div>
+ </template>
+ </t-popup>
+</template>
+
+<script setup lang="ts">
+import { computed, onUnmounted, ref, watch } from 'vue'
+import type { BackgroundWorkItem } from '@/api/chat/background-work'
+
+const props = defineProps<{
+ items: BackgroundWorkItem[]
+ cancelWork: (workId: string) => Promise<void>
+}>()
+const emit = defineEmits<{ view: [workId: string] }>()
+const cancelling = ref(new Set<string>())
+const open = ref(false)
+const now = ref(Date.now())
+let clock: ReturnType<typeof setInterval> | undefined
+
+const running = computed(() => props.items.filter((item) => item.state === 'running'))
+
+function syncClock() {
+ const live = running.value.some((item) => item.started_at)
+ if (live && !clock) {
+ clock = setInterval(() => { now.value = Date.now() }, 1000)
+ } else if (!live && clock) {
+ clearInterval(clock)
+ clock = undefined
+ }
+}
+
+watch(running, (items) => {
+ syncClock()
+ if (!items.length) open.value = false
+ const ids = new Set(items.map((item) => item.work_id))
+ const next = new Set([...cancelling.value].filter((id) => ids.has(id)))
+ if (next.size !== cancelling.value.size) cancelling.value = next
+}, { deep: true, immediate: true })
+
+onUnmounted(() => {
+ if (clock) clearInterval(clock)
+})
+
+function openOutput(workId: string) {
+ open.value = false
+ emit('view', workId)
+}
+
+async function onCancel(workId: string) {
+ if (cancelling.value.has(workId)) return
+ const next = new Set(cancelling.value)
+ next.add(workId)
+ cancelling.value = next
+ try {
+ await props.cancelWork(workId)
+ } finally {
+ // The request may fail while the job remains running. Always permit retry
+ // after it settles; only an in-flight request should disable the button.
+ const remaining = new Set(cancelling.value)
+ remaining.delete(workId)
+ cancelling.value = remaining
+ }
+}
+
+function formatDuration(ms: number): string {
+ const seconds = Math.max(0, Math.floor(ms / 1000))
+ if (seconds < 60) return `${seconds}s`
+ const minutes = Math.floor(seconds / 60)
+ if (minutes < 60) return `${minutes}m ${seconds % 60}s`
+ const hours = Math.floor(minutes / 60)
+ return `${hours}h ${minutes % 60}m`
+}
+
+function elapsed(item: BackgroundWorkItem): string {
+ const start = Date.parse(item.started_at ?? '')
+ if (!Number.isFinite(start)) return ''
+ return formatDuration(now.value - start)
+}
+</script>
+
+<style lang="less" scoped>
+.chip-count {
+ display: inline-flex;
+ align-items: center;
+ gap: 6px;
+ flex-shrink: 0;
+ height: 28px;
+ margin: 0;
+ padding: 0 10px;
+ border: 0;
+ border-radius: var(--app-radius-sm);
+ background: transparent;
+ color: var(--td-text-color-secondary);
+ font: inherit;
+ font-size: var(--app-text-md);
+ font-weight: 500;
+ line-height: 18px;
+ cursor: pointer;
+ transition: background var(--app-motion-instant), color var(--app-motion-instant);
+
+ &:hover,
+ &[aria-expanded='true'] {
+ background: var(--td-bg-color-secondarycontainer-hover);
+ color: var(--td-text-color-primary);
+ }
+}
+
+.chip-badge {
+ width: 6px;
+ height: 6px;
+ border-radius: 50%;
+ background: currentColor;
+}
+
+.work-card {
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+ width: min(320px, 72vw);
+ max-height: 280px;
+ overflow: auto;
+}
+
+.work-row {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+}
+
+.work-main {
+ display: flex;
+ align-items: baseline;
+ gap: 8px;
+ flex: 1;
+ min-width: 0;
+ margin: 0;
+ padding: 6px 8px;
+ border: 0;
+ border-radius: var(--td-radius-medium);
+ background: transparent;
+ color: var(--td-text-color-primary);
+ font: inherit;
+ font-size: var(--app-text-sm);
+ line-height: 18px;
+ text-align: left;
+ cursor: pointer;
+
+ &:hover {
+ background: var(--td-bg-color-secondarycontainer-hover);
+ }
+}
+
+.work-title {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.work-duration {
+ flex-shrink: 0;
+ color: var(--td-text-color-placeholder);
+ font-variant-numeric: tabular-nums;
+}
+
+.work-stop {
+ flex-shrink: 0;
+ margin: 0;
+ padding: 4px 8px;
+ border: 0;
+ border-radius: var(--td-radius-medium);
+ background: transparent;
+ color: var(--td-text-color-secondary);
+ font: inherit;
+ font-size: var(--app-text-sm);
+ line-height: 18px;
+ cursor: pointer;
+
+ &:hover {
+ background: var(--td-bg-color-secondarycontainer-hover);
+ }
+
+ &:disabled {
+ cursor: default;
+ opacity: 0.45;
+ }
+}
+</style>
diff --git a/frontend/src/views/chat/components/BackgroundWorkResults.vue b/frontend/src/views/chat/components/BackgroundWorkResults.vue
new file mode 100644
index 000000000..c45989b25
--- /dev/null
+++ b/frontend/src/views/chat/components/BackgroundWorkResults.vue
@@ -0,0 +1,48 @@
+<template>
+ <div class="background-work-results">
+ <div v-if="loading" class="results-loading">{{ $t('chat.backgroundWork.loading') }}</div>
+ <ShellExecResult v-else-if="detail?.result" :data="detail.result" />
+ <div v-else class="results-error">{{ $t('chat.backgroundWork.unavailable') }}</div>
+ </div>
+</template>
+
+<script setup lang="ts">
+import { ref, watch } from 'vue'
+import { getBackgroundWork, type BackgroundWorkDetail } from '@/api/chat/background-work'
+import ShellExecResult from './tool-results/ShellExecResult.vue'
+
+const props = defineProps<{ sessionId: string; workId: string }>()
+const detail = ref<BackgroundWorkDetail | null>(null)
+const loading = ref(false)
+
+let request = 0
+watch(() => [props.sessionId, props.workId] as const, async ([sessionId, id]) => {
+ if (!sessionId || !id) return
+ const current = ++request
+ loading.value = true
+ detail.value = null
+ try {
+ const next = await getBackgroundWork(sessionId, id)
+ if (current !== request) return
+ detail.value = next
+ } catch {
+ if (current !== request) return
+ detail.value = null
+ } finally {
+ if (current === request) loading.value = false
+ }
+}, { immediate: true })
+</script>
+
+<style lang="less" scoped>
+.background-work-results {
+ min-width: 0;
+}
+
+.results-loading,
+.results-error {
+ font-size: var(--app-text-sm);
+ line-height: 1.4;
+ color: var(--td-text-color-placeholder);
+}
+</style>
diff --git a/frontend/src/views/chat/components/ToolResultRenderer.vue b/frontend/src/views/chat/components/ToolResultRenderer.vue
index 5987237af..6474a0262 100644
--- a/frontend/src/views/chat/components/ToolResultRenderer.vue
+++ b/frontend/src/views/chat/components/ToolResultRenderer.vue
@@ -48,11 +48,28 @@
<ShellExecResult
v-else-if="displayType === 'shell_exec'"
- :data="toolData as ShellExecData"
- :output="output"
+ :data="shellExecData"
+ :output="shellExecOutput"
:arguments="toolArguments"
/>
+ <div v-else-if="displayType === 'work_wait'" class="work-wait-results">
+ <p v-if="workNote" class="work-wait-note">{{ workNote }}</p>
+ <ShellExecResult
+ v-for="(item, index) in workItems"
+ :key="item.work_id || index"
+ :data="shellItemResult(item)"
+ />
+ <div v-if="!workItems.length" class="fallback-output">
+ <div class="fallback-header">
+ <span class="fallback-label">{{ $t('chat.rawOutputLabel') }}</span>
+ </div>
+ <div class="detail-output-wrapper">
+ <div class="detail-output">{{ output }}</div>
+ </div>
+ </div>
+ </div>
+
<SandboxFilesResult
v-else-if="displayType === 'list_sandbox_files'"
:data="toolData as ListSandboxFilesData"
@@ -91,6 +108,16 @@
:output="output" :data="toolData" :success="success"
/>
+ <!-- Unmatched display types still reuse the shell card when the payload is a work list. -->
+ <div v-else-if="workItems.length" class="work-wait-results">
+ <p v-if="workNote" class="work-wait-note">{{ workNote }}</p>
+ <ShellExecResult
+ v-for="(item, index) in workItems"
+ :key="item.work_id || index"
+ :data="shellItemResult(item)"
+ />
+ </div>
+
<!-- Fallback: Display raw output -->
<div v-else class="fallback-output">
<div class="fallback-header">
@@ -121,7 +148,6 @@ import type {
GrepResultsData,
KnowledgeChunksListData,
WikiEditData,
- ShellExecData,
ListSandboxFilesData,
WriteSandboxFileData,
ReadSkillData,
@@ -165,6 +191,36 @@ const displayType = computed(() => props.displayType);
const toolData = computed(() => props.toolData || {});
const output = computed(() => props.output || '');
const toolArguments = computed(() => props.arguments || {});
+
+const shellExecData = computed(() => {
+ const nested = toolData.value.shell_job
+ if (nested && typeof nested === 'object' && !Array.isArray(nested)) return nested
+ return toolData.value
+})
+const shellExecOutput = computed(() => (
+ shellExecData.value === toolData.value ? output.value : ''
+))
+
+const workItems = computed(() => {
+ const data = toolData.value
+ if (Array.isArray(data.items)) return data.items
+ const nested = data.work
+ if (nested && typeof nested === 'object' && Array.isArray(nested.items)) return nested.items
+ return []
+})
+const workNote = computed(() => {
+ const data = toolData.value
+ if (typeof data.note === 'string' && data.note) return data.note
+ const nested = data.work
+ if (nested && typeof nested === 'object' && typeof nested.note === 'string') return nested.note
+ return ''
+})
+
+function shellItemResult(item: Record<string, any>) {
+ const result = item?.result
+ if (result && typeof result === 'object') return result
+ return {}
+}
</script>
<style lang="less" scoped>
@@ -172,6 +228,19 @@ const toolArguments = computed(() => props.arguments || {});
margin: 0;
}
+.work-wait-results {
+ display: flex;
+ flex-direction: column;
+ gap: 12px;
+}
+
+.work-wait-note {
+ margin: 0;
+ font-size: var(--app-text-sm);
+ line-height: 1.4;
+ color: var(--td-text-color-placeholder);
+}
+
.fallback-output {
margin: 12px 0;
padding: 0;
diff --git a/frontend/src/views/chat/components/tool-results/ShellExecResult.vue b/frontend/src/views/chat/components/tool-results/ShellExecResult.vue
index f5aabccae..2eee65d8d 100644
--- a/frontend/src/views/chat/components/tool-results/ShellExecResult.vue
+++ b/frontend/src/views/chat/components/tool-results/ShellExecResult.vue
@@ -13,8 +13,12 @@
<span v-if="durationLabel">{{ durationLabel }}</span>
<span v-if="view.killed">{{ $t('agentStream.shellExec.killed') }}</span>
<span v-if="view.truncated">{{ $t('agentStream.shellExec.truncated') }}</span>
+ <span v-if="workId">{{ workId }}</span>
+ <span v-if="workState">{{ workState }}</span>
</div>
+ <div v-if="note" class="shell-exec-note">{{ note }}</div>
+
<div v-if="view.stdoutBinary || view.stderrBinary" class="shell-exec-note">
{{ $t('agentStream.shellExec.binarySuppressed') }}
</div>
@@ -37,26 +41,44 @@
<script setup lang="ts">
import { computed } from 'vue'
+import type { ShellJobPayload } from '@/api/chat/background-work'
import type { ShellExecData } from '@/types/tool-results'
import { buildShellExecView } from '@/utils/shellExecResult'
const props = defineProps<{
- data: ShellExecData | Record<string, unknown>
+ data?: ShellExecData | ShellJobPayload | Record<string, unknown> | null
output?: string
arguments?: Record<string, unknown>
}>()
+const source = computed(() => {
+ const value = props.data
+ if (value && typeof value === 'object') return value as unknown as Record<string, unknown>
+ return null
+})
+
const view = computed(() =>
- buildShellExecView(props.data as Record<string, unknown>, props.arguments, props.output),
+ buildShellExecView(source.value, props.arguments, props.output),
)
+function textField(key: string): string {
+ const value = source.value?.[key]
+ return typeof value === 'string' ? value : ''
+}
+
+const workId = computed(() => textField('work_id'))
+const workState = computed(() => textField('state'))
+const note = computed(() => textField('note'))
+
const hasMeta = computed(() =>
Boolean(
view.value.workDir ||
view.value.exitCode !== null ||
view.value.durationMs !== null ||
view.value.killed ||
- view.value.truncated,
+ view.value.truncated ||
+ workId.value ||
+ workState.value,
),
)
diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue
index 28048d306..73fa06a29 100644
--- a/frontend/src/views/chat/index.vue
+++ b/frontend/src/views/chat/index.vue
@@ -182,7 +182,12 @@
:model-id="modelId" :default-model-id="defaultModelId"
@update:model-id="(id) => emit('update:modelId', id)"
:queuedSteers="steerQueue.filter(item => item.delivery === 'after')" :canSteer="isAgentStreamSession()">
- <template v-if="$slots['composer-chips']" #chips><slot name="composer-chips" /></template>
+ <template v-if="$slots['composer-chips']" #chips>
+ <slot name="composer-chips" />
+ </template>
+ <template v-if="backgroundItems.length" #background-work>
+ <BackgroundWorkChip :items="backgroundItems" :cancel-work="onCancelWork" @view="openWorkOutput" />
+ </template>
<template v-if="$slots['composer-controls']" #controls><slot name="composer-controls" /></template>
</InputField>
</div>
@@ -209,6 +214,10 @@
:shift-width="referencesPanelWidth"
:artifacts="sessionArtifacts" :artifacts-collecting="sessionArtifactsCollecting"
@artifact-deleted="handleArtifactDeleted" />
+ <t-dialog v-model:visible="workOutputVisible" :header="viewingWorkTitle" :footer="false" width="720px">
+ <BackgroundWorkResults v-if="workOutputVisible && viewingWorkId && session_id" :session-id="String(session_id)"
+ :work-id="viewingWorkId" />
+ </t-dialog>
</template>
<script setup>
import ChildAgentTabBar from './components/ChildAgentTabBar.vue';
@@ -269,6 +278,10 @@ import { provideChatSandboxPanel } from '@/composables/useChatSandboxPanel';
import SandboxSidePanel from '@/components/chat/SandboxSidePanel.vue';
import BrowserTaskPreview from './components/BrowserTaskPreview.vue';
import PendingPromptDock from './components/PendingPromptDock.vue';
+import BackgroundWorkChip from './components/BackgroundWorkChip.vue';
+import BackgroundWorkResults from './components/BackgroundWorkResults.vue';
+import { cancelBackgroundWork, listBackgroundWork } from '@/api/chat/background-work';
+import { createBackgroundWorkPoller, mergeBackgroundWork } from '@/utils/backgroundWork';
import { providePromptDock } from '@/composables/usePromptDock';
import { collectSessionArtifacts, markSessionArtifactDeleted } from '@/utils/sessionArtifacts';
import { isCollectingSkillArtifacts } from '@/utils/skillArtifacts';
@@ -346,6 +359,73 @@ const session_id = ref(props.session_id || route.params.chatid);
watch(session_id, () => childTabs.reset(), { flush: 'sync' });
const currentSession = ref(null);
const isAutomationSession = computed(() => currentSession.value?.kind === 'automation');
+const backgroundItems = ref([]);
+const viewingWorkId = ref('');
+const workOutputVisible = ref(false);
+const viewingWorkTitle = computed(() => {
+ const item = backgroundItems.value.find((row) => row.work_id === viewingWorkId.value);
+ return item?.title || viewingWorkId.value || '';
+});
+
+function replaceBackgroundItems(incoming) {
+ const merged = mergeBackgroundWork(backgroundItems.value, incoming);
+ const ids = new Set(incoming.map((item) => item.work_id));
+ const next = merged.filter((item) => ids.has(item.work_id));
+ if (
+ next.length === backgroundItems.value.length
+ && next.every((item, index) => item === backgroundItems.value[index])
+ ) {
+ return;
+ }
+ backgroundItems.value = next;
+}
+
+const backgroundPoller = createBackgroundWorkPoller({
+ sessionId: () => String(session_id.value || ''),
+ list: (id, signal) => listBackgroundWork(id, signal),
+ onUpdate: replaceBackgroundItems,
+});
+
+function startBackgroundWork() {
+ if (!session_id.value) {
+ backgroundItems.value = [];
+ return;
+ }
+ backgroundPoller.start();
+}
+
+async function refreshBackgroundWork() {
+ if (!session_id.value) return;
+ await backgroundPoller.refresh();
+}
+
+function openWorkOutput(workId) {
+ viewingWorkId.value = workId;
+ workOutputVisible.value = true;
+}
+
+async function onCancelWork(workId) {
+ const id = String(session_id.value || '');
+ if (!id || !workId) return;
+ try {
+ await cancelBackgroundWork(id, workId);
+ } catch {
+ // Refresh anyway so a rejected stop can leave the row running.
+ }
+ try {
+ if (String(session_id.value || '') === id) await refreshBackgroundWork();
+ } catch {
+ // Keep the last chip. A non-empty list is still on the 5s poller.
+ }
+}
+
+watch(session_id, (id) => {
+ backgroundPoller.stop();
+ backgroundItems.value = [];
+ viewingWorkId.value = '';
+ workOutputVisible.value = false;
+ if (id) backgroundPoller.start();
+});
// 拉 session 详情,并按其 last_request_state 把输入栏状态恢复到当时的发起态。
// 嵌入式(embeddedMode)由宿主页面注入 agent/KB,所以跳过整套恢复逻辑,
@@ -389,6 +469,16 @@ function handleAskAssistant(messageId) {
assistant.open();
}
+const backgroundWorkStamp = computed(() => messagesList.map((message) => {
+ const work = message?.background_work;
+ if (!work || typeof work !== 'object') return '';
+ return Object.entries(work).map(([id, row]) => `${id}:${row?.state || ''}`).join(',');
+}).join('|'));
+watch(backgroundWorkStamp, (stamp, previous) => {
+ if (!stamp || stamp === previous) return;
+ refreshBackgroundWork().catch(() => {});
+});
+
function forkAffordanceOf(messageId) {
if (isAutomationSession.value || !messageId) return { canFork: false }
return resolveForkAffordance(messagesList, messageId)
@@ -1807,6 +1897,7 @@ onMounted(async () => {
// 初始化状态:加载历史消息时不应显示loading
loading.value = false;
isReplying.value = false;
+ startBackgroundWork();
if (props.embeddedMode && props.initialQuery) {
scrollLock.value = true;
@@ -1852,6 +1943,7 @@ onUnmounted(() => {
activitySessionId.value = '';
window.removeEventListener(SESSION_MUTATION_EVENT, handleSessionMutation);
if (recoverPollTimer) { clearTimeout(recoverPollTimer); recoverPollTimer = null; }
+ backgroundPoller.dispose();
});
// An embedded view outlives the page behind it: a route change must not stop
// its stream or restore the main chat's defaults.
diff --git a/internal/agent/tools/output_links.go b/internal/agent/tools/output_links.go
index 8eb2c8bb8..703901354 100644
--- a/internal/agent/tools/output_links.go
+++ b/internal/agent/tools/output_links.go
@@ -48,6 +48,13 @@ func outputEntriesSnapshot(entries []sandbox.RemoteDirEntry) map[string]sandbox.
return files
}
+// ChangedOutputFileLinks reports deliverable links for files that appeared or
+// changed between two output-directory listings. An empty non-nil slice means
+// the listings were compared and nothing changed.
+func ChangedOutputFileLinks(outputDir string, before, after []sandbox.RemoteDirEntry) []string {
+ return changedOutputLinks(outputDir, outputEntriesSnapshot(before), outputEntriesSnapshot(after))
+}
+
func changedOutputLinks(outputDir string, before, after map[string]sandbox.RemoteDirEntry) []string {
var paths []string
for filePath, next := range after {
diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.go
index de054d014..91841e2ac 100644
--- a/internal/agent/tools/registry.go
+++ b/internal/agent/tools/registry.go
@@ -338,13 +338,27 @@ func executeRecovered(
return tool.Execute(ctx, args)
}
-// Cleanup cleans up all registered tools that implement the types.Cleanable interface.
-// This is called at the end of agent sessions to release tool-specific resources.
+// errorCleanable is Cleanup that reports a failure. types.Cleanable takes no
+// error, and one method cannot satisfy both signatures. shell_exec_async uses
+// the error form so a failed cursor update is visible.
+type errorCleanable interface {
+ Cleanup(ctx context.Context) error
+}
+
+// Cleanup cleans up all registered tools that implement types.Cleanable or
+// errorCleanable. This is called at the end of agent sessions to release
+// tool-specific resources, including shell-job output cursors.
func (r *ToolRegistry) Cleanup(ctx context.Context) {
for name, tool := range r.tools {
- if cleanable, ok := tool.(types.Cleanable); ok {
+ switch cleanable := tool.(type) {
+ case types.Cleanable:
logger.Infof(ctx, "[ToolRegistry] Cleaning up tool: %s", name)
cleanable.Cleanup(ctx)
+ case errorCleanable:
+ logger.Infof(ctx, "[ToolRegistry] Cleaning up tool: %s", name)
+ if err := cleanable.Cleanup(ctx); err != nil {
+ logger.Warnf(ctx, "[ToolRegistry] cleanup %s: %v", name, err)
+ }
}
}
}
diff --git a/internal/agent/tools/shell_exec.go b/internal/agent/tools/shell_exec.go
index 6e01c7059..50c9c406b 100644
--- a/internal/agent/tools/shell_exec.go
+++ b/internal/agent/tools/shell_exec.go
@@ -147,12 +147,12 @@ The sandbox belongs to this session alone; nothing here runs on the host.`
- Use ls/find to discover files, grep/awk to search, and cat/head/tail/sed to inspect text. Read known paths directly; no mandatory discovery call.
- Use write_sandbox_file for scripts or large text; edit_sandbox_file for precise changes. Commands are limited to 8192 bytes. Execution is synchronous (no nohup or trailing &).
- skill_name selects a listed skill for this call. Installed skills use their Python virtualenv and Node modules;
- host resources are staged automatically and use the system runtime until a local .venv is created.
+ host-staged resources use the system runtime until a local .venv exists.
Scoped credentials apply to both. Example: skill_name="pdf", command="python3 report.py".
Run bundled scripts via "$WEKNORA_SKILL_DIR/scripts/...". Omit skill_name for system commands.
- /workspace/input contains user attachments: preserve originals. /workspace/output is the only directory collected
for download, so it takes finished deliverables only; keep scratch and intermediate files elsewhere under /workspace.
- apt-get is available when the sandbox network policy allows it; permanent dependencies belong in the skill installer.
+ apt-get works when the network policy allows it; permanent dependencies belong in the skill installer.
- Install extras with skill_name set and the default work_dir:
` + "`" + skillPythonPackageInstallCommand + "`" + ` (no pip needed), or
` + "`" + skillNodePackageInstallCommand + "`" + `.
@@ -161,7 +161,9 @@ The sandbox belongs to this session alone; nothing here runs on the host.`
%s
- Non-zero exit_code is a command result: inspect stderr before deciding whether a corrected call is useful. Transport failures/timeouts are tool failures. Changing tools does not change permissions; do not repeat a denied operation through another tool.
- stdout/stderr have independent byte limits, preserving head and tail when truncated. Full output is not automatically saved; redirect verbose commands to a workspace log when it must be retained. Binary bytes are suppressed.
-- Use ![description](sandbox:<file name>) with exact links from Output files, never stdout or temporary paths.`
+- Use ![description](sandbox:<file name>) with exact links from Output files, never stdout or temporary paths.
+- For anything that may run longer than a couple of minutes use ` +
+ "`shell_exec_async`" + ` instead: short commands still return their result directly.`
)
func shellExecDescription(l sandbox.WorkspaceLayout) string {
@@ -187,7 +189,9 @@ const hostShellExecDescription = "Execute a command in %s. The process is OS-san
"- Non-zero exit_code is a command result: inspect stderr before deciding whether a corrected call is useful. " +
"Do not bypass permission or policy denials through another tool.\n" +
"- stdout/stderr have independent byte limits. " +
- "Redirect verbose commands to a workspace log when output must be kept."
+ "Redirect verbose commands to a workspace log when output must be kept.\n" +
+ "- If you are not sure the command finishes within a couple of minutes, use " +
+ "`shell_exec_async` instead: short commands still return their result directly.\n"
// ShellExecInput defines the input parameters for shell_exec.
type ShellExecInput struct {
@@ -753,17 +757,25 @@ func (t *ShellExecTool) Execute(ctx context.Context, args json.RawMessage) (*typ
func (t *ShellExecTool) maybeCaptureSkillEnv(
ctx context.Context, skillName string, pairs map[string]string, res *sandbox.ExecuteResult,
) {
- if t == nil || t.envCapture == nil || t.isInstallMode() {
+ if t == nil || t.isInstallMode() {
return
}
- if res == nil || res.ExitCode != 0 {
+ captureUsedSkillEnv(ctx, t.envCapture, skillName, pairs, res != nil && res.ExitCode == 0)
+}
+
+// captureUsedSkillEnv persists credentials a successful skill command brought
+// with it. pairs must already exclude names that were stored before the call.
+func captureUsedSkillEnv(
+ ctx context.Context, capture SkillEnvCapture, skillName string, pairs map[string]string, succeeded bool,
+) {
+ if !succeeded || capture == nil {
return
}
skillName = strings.TrimSpace(skillName)
if !sandbox.IsValidSkillName(skillName) || len(pairs) == 0 {
return
}
- t.envCapture(ctx, skillName, pairs)
+ capture(ctx, skillName, pairs)
}
// stillMissing removes from missing every name this call supplied itself.
diff --git a/internal/agent/tools/shell_exec_async.go b/internal/agent/tools/shell_exec_async.go
new file mode 100644
index 000000000..159b75560
--- /dev/null
+++ b/internal/agent/tools/shell_exec_async.go
@@ -0,0 +1,500 @@
+package tools
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/skills"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+const shellExecAsyncDescription = `Run a shell command that may take longer than a couple of minutes: builds, test
+suites, data processing, large downloads, install software. shell_exec is killed at 600s; this has no
+time limit. It first waits up to yield_time_ms (default 10000, max 30000): if the
+command finishes by then you get its result directly, like shell_exec. Otherwise you
+get a work_id plus the output so far, and the command keeps running.
+Use this instead of &, nohup, setsid or disown inside shell_exec.
+Do not pipe the command through tail or head. The job log records only what the
+command itself writes, and tail waits until the command exits, so the log stays
+empty until then. Head and tail of that log are kept for you.
+Run non-interactively. There is no TTY and stdin stays closed, so a prompt waits
+forever. Pass the flags that skip questions (apt-get -y, DEBIAN_FRONTEND=noninteractive,
+--yes, --non-interactive). Do not use editors, pagers, or commands that read a reply.
+Arguments: same as shell_exec, plus description (short label shown to the user) and
+yield_time_ms.`
+
+// shellExecAsyncOutputLimit keeps a finished job's stdout, stderr, original
+// command and JSON envelope intact, including six-character control-byte escapes.
+// The generic 24000-rune cap would split that envelope.
+const shellExecAsyncOutputLimit = ((16+8)*1024+shellExecMaxCommandBytes)*6 + 8*1024
+
+// ShellExecAsyncInput is the model-facing argument object.
+type ShellExecAsyncInput struct {
+ Command string `json:"command" jsonschema:"The shell one-liner to run"`
+ WorkDir string `json:"work_dir,omitempty" jsonschema:"Working directory; defaults like shell_exec"`
+ Stdin string `json:"stdin,omitempty" jsonschema:"Text written once at start, max 65536 bytes"`
+ Env map[string]string `json:"env,omitempty" jsonschema:"Extra environment variables"`
+ SkillName string `json:"skill_name,omitempty" jsonschema:"Pull this skill's env into the command"`
+ Description string `json:"description" jsonschema:"Short label shown to the user"`
+ YieldTimeMS int `json:"yield_time_ms,omitempty" jsonschema:"Wait ms; default 10000, max 30000"`
+}
+
+// ShellJobSubmitter starts one shell job. *service.ShellJobService implements it.
+// EndShellJobRun is optional and checked separately at Cleanup.
+type ShellJobSubmitter interface {
+ SubmitShellJob(ctx context.Context, s interfaces.SessionRef, req ShellJobSubmitRequest) (ShellJobSubmission, error)
+}
+
+// ShellExecAsyncTool validates a command the same way shell_exec does, then
+// hands it to ShellJobSubmitter. It does not run the command itself.
+type ShellExecAsyncTool struct {
+ BaseTool
+ sessionBound
+ submitter ShellJobSubmitter
+ executor SandboxCommandExecutor
+ envResolver skills.SkillEnvResolver
+ skillEnv *skills.Manager
+ envCapture SkillEnvCapture
+ backgroundRef interfaces.SessionRef
+ backgroundRun string
+ marks interface{ MarkStarted(string) }
+}
+
+var (
+ _ types.Tool = (*ShellExecAsyncTool)(nil)
+ _ interface{ Cleanup(context.Context) error } = (*ShellExecAsyncTool)(nil)
+)
+
+func init() {
+ backgroundSubmissionTools = append(backgroundSubmissionTools, ToolShellExecAsync)
+}
+
+// NewShellExecAsyncTool builds shell_exec_async. submitter starts a background
+// job, executor runs a command that finishes inside the yield window, and
+// envResolver supplies skill credentials.
+func NewShellExecAsyncTool(submitter ShellJobSubmitter, executor SandboxCommandExecutor,
+ envResolver skills.SkillEnvResolver,
+) *ShellExecAsyncTool {
+ return &ShellExecAsyncTool{
+ BaseTool: NewBaseTool(
+ ToolShellExecAsync, shellExecAsyncDescription, utils.GenerateSchema[ShellExecAsyncInput](),
+ ),
+ submitter: submitter,
+ executor: executor,
+ envResolver: envResolver,
+ }
+}
+
+// WithSkillEnvironment attaches the skill runtime used by shell_exec.
+func (t *ShellExecAsyncTool) WithSkillEnvironment(manager *skills.Manager) *ShellExecAsyncTool {
+ if t == nil {
+ return nil
+ }
+ t.skillEnv = manager
+ return t
+}
+
+// WithEnvCapture attaches the same credential write-back shell_exec uses.
+// A nil hook does nothing. Values stay in memory until the job succeeds.
+func (t *ShellExecAsyncTool) WithEnvCapture(capture SkillEnvCapture) *ShellExecAsyncTool {
+ if t != nil {
+ t.envCapture = capture
+ }
+ return t
+}
+
+// BindBackgroundRun records the run that owns this tool. A handle that is still
+// running is marked started on that run; a direct result is not. Cursor expiry
+// uses the same run id at Cleanup.
+func (t *ShellExecAsyncTool) BindBackgroundRun(
+ s interfaces.SessionRef, runID string, marks interface{ MarkStarted(string) },
+) {
+ if t == nil {
+ return
+ }
+ t.backgroundRef, t.backgroundRun, t.marks = s, runID, marks
+}
+
+// Cleanup tells the submitter the run ended so its output cursors can expire.
+func (t *ShellExecAsyncTool) Cleanup(context.Context) error {
+ if t == nil || t.backgroundRun == "" || t.submitter == nil {
+ return nil
+ }
+ ender, ok := t.submitter.(interface {
+ EndShellJobRun(interfaces.SessionRef, string)
+ })
+ if ok {
+ ender.EndShellJobRun(t.backgroundRef, t.backgroundRun)
+ }
+ return nil
+}
+
+// OutputLimitChars is the rune budget for the model-facing JSON envelope.
+func (t *ShellExecAsyncTool) OutputLimitChars(json.RawMessage) int {
+ return shellExecAsyncOutputLimit
+}
+
+// Execute validates the command, wraps stdin the way shell_exec does, and submits.
+func (t *ShellExecAsyncTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) {
+ logger.Infof(ctx, "[Tool][ShellExecAsync] Execute started")
+
+ var input ShellExecAsyncInput
+ if err := json.Unmarshal(args, &input); err != nil {
+ return &types.ToolResult{Success: false, Error: "failed to parse args: " + err.Error()}, nil
+ }
+ if t.submitter == nil {
+ return &types.ToolResult{Success: false, Error: "shell_exec_async is not available in this deployment"}, nil
+ }
+ if input.SkillName != "" && t.skillEnv == nil {
+ return &types.ToolResult{Success: false, Error: "no skill environment is available for this call; " +
+ "omit skill_name for system commands"}, nil
+ }
+ if len(input.Stdin) > 65536 {
+ return &types.ToolResult{Success: false, Error: "stdin exceeds 65536 bytes; " +
+ "write the input to a workspace file and redirect from it"}, nil
+ }
+ command := strings.TrimSpace(input.Command)
+ if command == "" {
+ return &types.ToolResult{Success: false, Error: "command is required"}, nil
+ }
+ if len(command) > shellExecMaxCommandBytes {
+ return &types.ToolResult{Success: false, Error: fmt.Sprintf(
+ "command too long (%d bytes; max %d). Put the file in write_sandbox_file, then run it",
+ len(command), shellExecMaxCommandBytes)}, nil
+ }
+ if reason := checkShellExecBlacklist(command); reason != "" {
+ logger.Warnf(ctx, "[Tool][ShellExecAsync] rejected by blacklist: %s command=%q",
+ reason, maskCommandAssignments(command))
+ return &types.ToolResult{
+ Success: false,
+ Error: "command rejected by shell_exec safety guard: " + reason,
+ }, nil
+ }
+ if reason := rejectExecutableStdin(command, input.Stdin); reason != "" {
+ logger.Warnf(ctx, "[Tool][ShellExecAsync] rejected executable stdin: %s command=%q",
+ reason, maskCommandAssignments(command))
+ return &types.ToolResult{Success: false, Error: reason}, nil
+ }
+ sessionID := resolveSessionID(ctx)
+ if sessionID == "" {
+ return &types.ToolResult{Success: false, Error: "no session ID in context"}, nil
+ }
+ ref, refErr := t.submitRef(ctx, sessionID)
+ if refErr != nil {
+ return refErr, nil
+ }
+ layoutCtx, layoutSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.layout",
+ Input: map[string]interface{}{"session_id": sessionID},
+ })
+ layout, layoutErr := executeWorkspaceLayout(layoutCtx, sessionID, t.executor)
+ layoutSpan.Finish(spanToolResult(layoutErr, map[string]interface{}{"root": layout.Root}), nil, nil)
+ if layoutErr != nil {
+ return layoutErr, nil
+ }
+ workDir, dirErr := resolveAsyncWorkDir(layout, input.WorkDir)
+ if dirErr != nil {
+ return dirErr, nil
+ }
+ baselineCtx, baselineSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.output_baseline",
+ Input: map[string]interface{}{"session_id": sessionID, "output_dir": layoutOutputDir(layout)},
+ })
+ outputDir, outputSeen, outputBefore := asyncOutputBaseline(baselineCtx, t.executor, sessionID, layout)
+ baselineSpan.Finish(map[string]interface{}{"seen": outputSeen, "files": len(outputBefore)}, nil, nil)
+
+ envCtx, envSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.env_resolve",
+ Input: map[string]interface{}{"skill_name": input.SkillName},
+ })
+ env, supplied, envErr := t.resolveAsyncEnv(envCtx, input, command)
+ envSpan.Finish(spanToolResult(envErr, nil), nil, nil)
+ if envErr != nil {
+ return envErr, nil
+ }
+ execCommand := command
+ if input.SkillName != "" && t.skillEnv != nil {
+ var prepErr error
+ execCommand, env, prepErr = t.skillEnv.PrepareShellEnvironment(
+ ctx, sessionID, input.SkillName, command, env)
+ if prepErr != nil {
+ return &types.ToolResult{Success: false, Error: prepErr.Error()}, nil
+ }
+ }
+ if input.Stdin != "" {
+ // Same wrapping as shell_exec: the payload is applied to the whole
+ // command, and base64 keeps quotes and trailing newlines out of the shell.
+ encoded := base64.StdEncoding.EncodeToString([]byte(input.Stdin))
+ execCommand = "printf %s " + sandbox.ShellQuote(encoded) +
+ " | base64 -d | /bin/bash --noprofile --norc -c " + sandbox.ShellQuote(execCommand)
+ }
+ // The watcher does not cd to meta.work_dir before starting: the command
+ // itself is wrapped in its own cd, and the exec cwd is the session layout
+ // root the manager picks (never "/"; shared machines reject that).
+ // A brace group keeps a later ";" (skill PATH setup) from running when cd fails.
+ execCommand = commandInWorkDir(workDir, execCommand)
+
+ request := ShellJobSubmitRequest{
+ Command: execCommand,
+ DisplayCommand: command,
+ WorkDir: workDir,
+ Env: env,
+ Description: shellJobLabel(input.Description, command),
+ YieldTime: time.Duration(clampYieldMS(input.YieldTimeMS)) * time.Millisecond,
+ RunID: t.backgroundRun,
+ OutputDir: outputDir,
+ OutputSeen: outputSeen,
+ OutputBefore: outputBefore,
+ }
+ if meta, ok := ToolExecFromContext(ctx); ok && meta != nil {
+ request.ParentMessageID = meta.AssistantMessageID
+ request.ToolCallID = meta.ToolCallID
+ }
+ if preview, finish := shellCommandOutput(ctx, command); preview != nil {
+ request.OnOutput = preview
+ defer finish()
+ }
+
+ submission, err := t.submitter.SubmitShellJob(ctx, ref, request)
+ if err != nil {
+ if errors.Is(err, interfaces.ErrActivityLimit) ||
+ strings.Contains(err.Error(), interfaces.ErrActivityLimit.Error()) {
+ return &types.ToolResult{Success: false, Error: "ERR_WORK_LIMIT: this session already runs " +
+ "64 background shell jobs; work_wait for one to finish or work_cancel one first"}, nil
+ }
+ return &types.ToolResult{Success: false, Error: "shell_exec_async failed: " + err.Error()}, nil
+ }
+ // A direct result was already shown. Only a still-running handle belongs
+ // in this run's "started" set.
+ if submission.Background && t.marks != nil {
+ t.marks.MarkStarted(submission.Result.ID)
+ }
+ t.captureSkillEnv(ctx, ref, submission, input.SkillName, supplied)
+ body := shellJobResultMap(submission.Result)
+ raw, err := json.Marshal(body)
+ if err != nil {
+ return &types.ToolResult{Success: false, Error: "encode shell job result: " + err.Error()}, nil
+ }
+ return &types.ToolResult{
+ Success: true,
+ Output: string(raw),
+ OutputFiles: submission.Result.OutputFiles,
+ Data: map[string]interface{}{"shell_job": body},
+ }, nil
+}
+
+func asyncOutputBaseline(
+ ctx context.Context, executor SandboxCommandExecutor, sessionID string, layout sandbox.WorkspaceLayout,
+) (dir string, seen bool, before []sandbox.OutputFileStamp) {
+ dir = layoutOutputDir(layout)
+ if dir == "" || executor == nil {
+ return dir, false, nil
+ }
+ listed, ok := sandboxOutputSnapshot(ctx, executor, sessionID, dir)
+ if !ok {
+ return dir, false, nil
+ }
+ entries := make([]sandbox.RemoteDirEntry, 0, len(listed))
+ for _, entry := range listed {
+ entries = append(entries, entry)
+ }
+ stamps, ok := sandbox.StampsFromEntries(entries)
+ if !ok {
+ return dir, false, nil
+ }
+ return dir, true, stamps
+}
+
+func (t *ShellExecAsyncTool) submitRef(
+ ctx context.Context, sessionID string,
+) (interfaces.SessionRef, *types.ToolResult) {
+ ref := t.backgroundRef
+ if strings.TrimSpace(ref.SessionID) == "" {
+ ref.SessionID = sessionID
+ }
+ if ref.TenantID != 0 {
+ return ref, nil
+ }
+ tenantID, ok := types.SandboxTenantIDFromContext(ctx)
+ if !ok || tenantID == 0 {
+ return interfaces.SessionRef{}, &types.ToolResult{Success: false, Error: "no session tenant in context"}
+ }
+ ref.TenantID = tenantID
+ return ref, nil
+}
+
+func (t *ShellExecAsyncTool) captureSkillEnv(
+ ctx context.Context, ref interfaces.SessionRef, submission ShellJobSubmission,
+ skillName string, pairs map[string]string,
+) {
+ succeeded := submission.Result.State == interfaces.WorkStateSucceeded
+ if !submission.Background {
+ captureUsedSkillEnv(ctx, t.envCapture, skillName, pairs, succeeded)
+ return
+ }
+ if t.envCapture == nil || len(pairs) == 0 || !sandbox.IsValidSkillName(strings.TrimSpace(skillName)) {
+ return
+ }
+ holder, ok := t.submitter.(interface {
+ HoldSkillEnvCapture(context.Context, interfaces.SessionRef, string, func(context.Context))
+ })
+ if !ok || submission.Result.ID == "" {
+ return
+ }
+ name := strings.TrimSpace(skillName)
+ held := copySkillPairs(pairs)
+ capture := t.envCapture
+ holder.HoldSkillEnvCapture(ctx, ref, submission.Result.ID, func(ctx context.Context) {
+ captureUsedSkillEnv(ctx, capture, name, held, true)
+ })
+}
+
+func copySkillPairs(pairs map[string]string) map[string]string {
+ out := make(map[string]string, len(pairs))
+ for name, value := range pairs {
+ out[name] = value
+ }
+ return out
+}
+
+func (t *ShellExecAsyncTool) resolveAsyncEnv(
+ ctx context.Context, input ShellExecAsyncInput, command string,
+) (map[string]string, map[string]string, *types.ToolResult) {
+ supplied := collectUsedSkillEnv(command, input.Env)
+ env := input.Env
+ if t.envResolver == nil {
+ return env, supplied, nil
+ }
+ // Empty skill name still asks for the caller's config-wide variables,
+ // matching shell_exec. Model-supplied env wins over resolved values.
+ resolved, missing, err := t.envResolver.ResolveEnv(ctx, input.SkillName)
+ if err != nil {
+ return nil, nil, &types.ToolResult{Success: false, Error: fmt.Sprintf(
+ "failed to resolve environment variables: %v", err)}
+ }
+ missing = stillMissing(missing, supplied)
+ if len(missing) > 0 {
+ return nil, nil, &types.ToolResult{Success: false, Error: fmt.Sprintf(
+ "skill %q needs the environment variable(s) %s, which nobody has set yet. "+
+ "Ask the user for them and pass them in this call's env, "+
+ "or have them set the values under Settings → Sandbox secrets.",
+ input.SkillName, strings.Join(missing, ", "))}
+ }
+ if len(resolved) > 0 && env == nil {
+ env = make(map[string]string)
+ }
+ skills.ApplyResolvedEnv(env, resolved)
+ // Stored names must not be written again, matching shell_exec.
+ return env, dropResolvedNames(supplied, resolved), nil
+}
+
+func resolveAsyncWorkDir(layout sandbox.WorkspaceLayout, requested string) (string, *types.ToolResult) {
+ root := strings.TrimSpace(layout.Root)
+ if root == "" {
+ root = sandbox.RemoteWorkspaceLayout().Root
+ }
+ workDir := strings.TrimSpace(requested)
+ if workDir == "" {
+ workDir = root
+ }
+ if !path.IsAbs(workDir) {
+ workDir = path.Join(root, workDir)
+ }
+ clean := path.Clean(workDir)
+ roots := asyncWorkDirRoots(layout)
+ for _, allowed := range roots {
+ if isUnderRoot(clean, allowed) {
+ return clean, nil
+ }
+ }
+ return "", &types.ToolResult{Success: false, Error: fmt.Sprintf(
+ "work_dir %q is outside the allowed sandbox roots %s",
+ requested, strings.Join(roots, ", "))}
+}
+
+func asyncWorkDirRoots(layout sandbox.WorkspaceLayout) []string {
+ if !layout.IsHost() {
+ return []string{"/"}
+ }
+ if len(layout.WriteRoots) > 0 {
+ return layout.WriteRoots
+ }
+ if strings.TrimSpace(layout.Root) != "" {
+ return []string{layout.Root}
+ }
+ return []string{"/"}
+}
+
+func shellJobResultMap(result interfaces.WorkResult) map[string]any {
+ body := map[string]any{}
+ switch payload := result.Payload.(type) {
+ case map[string]any:
+ for key, value := range payload {
+ body[key] = value
+ }
+ case nil:
+ default:
+ raw, err := json.Marshal(payload)
+ if err == nil {
+ _ = json.Unmarshal(raw, &body)
+ }
+ }
+ if body == nil {
+ body = map[string]any{}
+ }
+ if id, _ := body["work_id"].(string); id == "" {
+ body["work_id"] = result.ID
+ }
+ if state, _ := body["state"].(string); state == "" {
+ body["state"] = result.State
+ }
+ return body
+}
+
+func clampYieldMS(ms int) int {
+ if ms <= 0 {
+ return 10000
+ }
+ return min(max(ms, 250), 30000)
+}
+
+func commandInWorkDir(workDir, command string) string {
+ if workDir == "" {
+ return command
+ }
+ // A newline terminates comments and heredoc delimiters without changing
+ // the command's shell grammar (an appended semicolon does not).
+ return "cd " + sandbox.ShellQuote(workDir) + " && {\n" + command + "\n}"
+}
+
+func shellJobLabel(description, command string) string {
+ if label := strings.TrimSpace(description); label != "" {
+ return label
+ }
+ return command
+}
+
+// spanToolResult reports a *types.ToolResult failure on a span without the
+// error channel: ToolResult is a tool-domain outcome, not a Go error, and
+// keeping it in output keeps the span status clean.
+func spanToolResult(result *types.ToolResult, ok map[string]interface{}) map[string]interface{} {
+ out := map[string]interface{}{}
+ for k, v := range ok {
+ out[k] = v
+ }
+ if result != nil && !result.Success {
+ out["error"] = result.Error
+ }
+ return out
+}
diff --git a/internal/agent/tools/shell_exec_async_regression_test.go b/internal/agent/tools/shell_exec_async_regression_test.go
new file mode 100644
index 000000000..9276af2a0
--- /dev/null
+++ b/internal/agent/tools/shell_exec_async_regression_test.go
@@ -0,0 +1,53 @@
+package tools
+
+import (
+ "encoding/json"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCommandInWorkDirPreservesCommentsAndHeredocs(t *testing.T) {
+ for _, command := range []string{"printf ok # comment", "cat <<'EOF'\nok\nEOF"} {
+ t.Run(command, func(t *testing.T) {
+ plain, err := exec.Command("/bin/bash", "-c", command).CombinedOutput()
+ require.NoError(t, err)
+ wrapped, err := exec.Command("/bin/bash", "-c", commandInWorkDir(t.TempDir(), command)).CombinedOutput()
+ require.NoError(t, err, "valid shell input broke after wrapping: %s", wrapped)
+ require.Equal(t, string(plain), string(wrapped))
+ })
+ }
+}
+
+func TestCommandInWorkDirDoesNotRunAfterCDReturnsFailure(t *testing.T) {
+ root := t.TempDir()
+ marker := filepath.Join(root, "must-not-exist")
+ command := "printf ran > " + sandbox.ShellQuote(marker)
+ _, err := exec.Command("/bin/bash", "-c", commandInWorkDir(filepath.Join(root, "absent"), command)).CombinedOutput()
+ require.Error(t, err)
+ _, err = os.Stat(marker)
+ require.True(t, os.IsNotExist(err))
+}
+
+func TestShellExecAsyncPreservesJSONWithEscapedOutput(t *testing.T) {
+ submitter := &fakeSubmitter{response: ShellJobSubmission{Result: interfaces.WorkResult{
+ ID: "sj_0123", State: interfaces.WorkStateSucceeded, Terminal: true,
+ Payload: map[string]any{"stdout": strings.Repeat("\x01", 16*1024), "stderr": strings.Repeat("\x02", 8*1024)},
+ }}}
+ registry := NewToolRegistry()
+ registry.RegisterTool(NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil))
+ result, err := registry.ExecuteTool(
+ asyncTestCtx(),
+ "shell_exec_async",
+ json.RawMessage(`{"command":"true","description":"build"}`),
+ )
+ require.NoError(t, err)
+ require.True(t, result.Success, result.Error)
+ require.True(t, json.Valid([]byte(result.Output)), "the registry must preserve the structured result")
+}
diff --git a/internal/agent/tools/shell_exec_async_test.go b/internal/agent/tools/shell_exec_async_test.go
new file mode 100644
index 000000000..1b8038a87
--- /dev/null
+++ b/internal/agent/tools/shell_exec_async_test.go
@@ -0,0 +1,313 @@
+package tools
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type fakeSubmitter struct {
+ requests []ShellJobSubmitRequest
+ refs []interfaces.SessionRef
+ response ShellJobSubmission
+ err error
+ ended int
+}
+
+func (f *fakeSubmitter) SubmitShellJob(_ context.Context, ref interfaces.SessionRef,
+ req ShellJobSubmitRequest,
+) (ShellJobSubmission, error) {
+ f.requests = append(f.requests, req)
+ f.refs = append(f.refs, ref)
+ return f.response, f.err
+}
+func (f *fakeSubmitter) EndShellJobRun(interfaces.SessionRef, string) { f.ended++ }
+
+type holdingSubmitter struct {
+ fakeSubmitter
+ held func(context.Context)
+}
+
+func (h *holdingSubmitter) HoldSkillEnvCapture(
+ _ context.Context, _ interfaces.SessionRef, _ string, run func(context.Context),
+) {
+ h.held = run
+}
+
+func asyncTestCtx() context.Context {
+ // The turn's session tenant is SandboxTenantIDFromContext, not a tool-context helper.
+ ctx := types.WithSandboxTenantID(context.Background(), 7)
+ return WithToolExecContext(ctx, &ToolExecContext{
+ SessionID: "session-1", AssistantMessageID: "m1", ToolCallID: "tc1",
+ })
+}
+
+func runAsyncTool(t *testing.T, submitter ShellJobSubmitter, args string) (*types.ToolResult, error) {
+ t.Helper()
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil)
+ return tool.Execute(asyncTestCtx(), json.RawMessage(args))
+}
+
+func TestShellExecAsyncRejectsBlacklistedAndMissingCommand(t *testing.T) {
+ submitter := &fakeSubmitter{}
+ for _, args := range []string{
+ `{"command":"sleep 5 &"}`,
+ `{"command":"nohup make build"}`,
+ `{"command":" "}`,
+ } {
+ result, err := runAsyncTool(t, submitter, args)
+ require.NoError(t, err)
+ require.False(t, result.Success, args)
+ require.Empty(t, submitter.requests, args)
+ }
+}
+
+func TestShellExecAsyncClampsYieldAndPassesContext(t *testing.T) {
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateRunning},
+ Background: true,
+ }}
+ result, err := runAsyncTool(t, submitter,
+ `{"command":"make build","description":"build","yield_time_ms":999999,"skill_name":""}`)
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ require.Len(t, submitter.requests, 1)
+ require.Equal(t, 30*time.Second, submitter.requests[0].YieldTime, "夹到 30000ms 上限")
+ require.Equal(t, "m1", submitter.requests[0].ParentMessageID)
+ require.Equal(t, "tc1", submitter.requests[0].ToolCallID)
+ require.Equal(t, "build", submitter.requests[0].Description)
+ require.Equal(t, "cd /workspace && {\nmake build\n}", submitter.requests[0].Command)
+ require.Equal(t, "make build", submitter.requests[0].DisplayCommand)
+ require.Equal(t, "/workspace", submitter.requests[0].WorkDir)
+ require.Equal(t, interfaces.SessionRef{TenantID: 7, SessionID: "session-1"}, submitter.refs[0])
+}
+
+func TestShellExecAsyncMarksStartedOnRunningHandle(t *testing.T) {
+ marks := &recordingMarks{}
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateRunning},
+ Background: true,
+ }}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "session-1"}
+ tool.BindBackgroundRun(ref, "m1", marks)
+ _, err := tool.Execute(asyncTestCtx(), json.RawMessage(`{"command":"make build"}`))
+ require.NoError(t, err)
+ require.Equal(t, []string{"sj_0123"}, marks.started)
+ require.NoError(t, tool.Cleanup(context.Background()))
+ require.Equal(t, 1, submitter.ended, "Cleanup 通知 run 结束(cursor 一小时后过期)")
+}
+
+func TestShellExecAsyncShortResultCarriesPayload(t *testing.T) {
+ exit := 0
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{
+ ID: "sj_0123", State: interfaces.WorkStateSucceeded,
+ Terminal: true, Payload: map[string]any{"exit_code": exit, "stdout": "ok"},
+ },
+ Background: false,
+ }}
+ result, err := runAsyncTool(t, submitter, `{"command":"true"}`)
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ data := result.Data["shell_job"].(map[string]any)
+ require.Equal(t, "sj_0123", data["work_id"])
+ require.Equal(t, "succeeded", data["state"])
+ require.Equal(t, "ok", data["stdout"])
+}
+
+func TestShellExecAsyncDirectResultDoesNotMarkStarted(t *testing.T) {
+ marks := &recordingMarks{}
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{
+ ID: "sj_0123", State: interfaces.WorkStateSucceeded,
+ Terminal: true, Payload: map[string]any{"stdout": "ok"},
+ },
+ Background: false,
+ }}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "session-1"}
+ tool.BindBackgroundRun(ref, "m1", marks)
+ ctx := types.WithSandboxTenantID(context.Background(), 99)
+ ctx = WithToolExecContext(ctx, &ToolExecContext{
+ SessionID: "other", AssistantMessageID: "m-other", ToolCallID: "tc-other",
+ })
+ _, err := tool.Execute(ctx, json.RawMessage(`{"command":"true"}`))
+ require.NoError(t, err)
+ require.Empty(t, marks.started)
+ require.Equal(t, ref, submitter.refs[0])
+ require.Equal(t, "m1", submitter.requests[0].RunID)
+}
+
+func TestShellExecAsyncWrapsStdinBeforeSubmit(t *testing.T) {
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateSucceeded, Terminal: true},
+ }}
+ stdin := "hello 'world'\n"
+ args, err := json.Marshal(ShellExecAsyncInput{Command: "cat", Stdin: stdin})
+ require.NoError(t, err)
+ result, err := runAsyncTool(t, submitter, string(args))
+ require.NoError(t, err)
+ require.True(t, result.Success, result.Error)
+ require.Len(t, submitter.requests, 1)
+ encoded := base64.StdEncoding.EncodeToString([]byte(stdin))
+ want := "cd " + sandbox.ShellQuote(sandbox.SessionWorkspaceRoot) + " && {\n" +
+ "printf %s " + sandbox.ShellQuote(encoded) +
+ " | base64 -d | /bin/bash --noprofile --norc -c " + sandbox.ShellQuote("cat") + "\n}"
+ require.Equal(t, want, submitter.requests[0].Command)
+ require.Equal(t, "cat", submitter.requests[0].DisplayCommand)
+ require.Equal(t, sandbox.SessionWorkspaceRoot, submitter.requests[0].WorkDir)
+}
+
+func TestShellExecAsyncRejectsOversizeAndExecutableStdin(t *testing.T) {
+ submitter := &fakeSubmitter{}
+ oversize, err := json.Marshal(ShellExecAsyncInput{Command: "cat", Stdin: strings.Repeat("x", 65537)})
+ require.NoError(t, err)
+ result, err := runAsyncTool(t, submitter, string(oversize))
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Contains(t, result.Error, "65536")
+ require.Empty(t, submitter.requests)
+
+ bomb, err := json.Marshal(ShellExecAsyncInput{Command: "bash -s", Stdin: ":(){ :|:& };:"})
+ require.NoError(t, err)
+ result, err = runAsyncTool(t, submitter, string(bomb))
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Contains(t, result.Error, "fork_bomb")
+ require.Empty(t, submitter.requests)
+}
+
+func TestShellExecAsyncReportsActivityLimit(t *testing.T) {
+ submitter := &fakeSubmitter{err: interfaces.ErrActivityLimit}
+ result, err := runAsyncTool(t, submitter, `{"command":"make build"}`)
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Contains(t, result.Error, "ERR_WORK_LIMIT")
+}
+
+func TestShellExecAsyncRejectsSkillWithoutEnvironment(t *testing.T) {
+ submitter := &fakeSubmitter{}
+ result, err := runAsyncTool(t, submitter, `{"command":"python3 report.py","skill_name":"pdf"}`)
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Contains(t, result.Error, "skill environment")
+ require.Empty(t, submitter.requests)
+}
+
+func TestShellExecAsyncRequiresSessionTenant(t *testing.T) {
+ submitter := &fakeSubmitter{}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil)
+ ctx := WithToolExecContext(context.Background(), &ToolExecContext{
+ SessionID: "session-1", AssistantMessageID: "m1", ToolCallID: "tc1",
+ })
+ result, err := tool.Execute(ctx, json.RawMessage(`{"command":"true"}`))
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Contains(t, result.Error, "tenant")
+ require.Empty(t, submitter.requests)
+}
+
+func TestShellExecAsyncCapturesSkillEnvWhenTheJobFinishesInTheWindow(t *testing.T) {
+ recorder := &recordedCapture{}
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateSucceeded},
+ }}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil).
+ WithSkillEnvironment(shellTestSkillEnvironment(t)).
+ WithEnvCapture(recorder.capture)
+ result, err := tool.Execute(asyncTestCtx(), json.RawMessage(
+ `{"command":"export USER_TOKEN=from-command; python x.py",`+
+ `"skill_name":"pdf-tools","env":{"EXTRA_TOKEN":"from-tool"}}`,
+ ))
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ require.Equal(t, 1, recorder.calls)
+ require.Equal(t, "pdf-tools", recorder.skillName)
+ require.Equal(t, "from-command", recorder.pairs["USER_TOKEN"])
+ require.Equal(t, "from-tool", recorder.pairs["EXTRA_TOKEN"])
+}
+
+func TestShellExecAsyncHoldsSkillEnvUntilTheBackgroundJobSucceeds(t *testing.T) {
+ recorder := &recordedCapture{}
+ submitter := &holdingSubmitter{fakeSubmitter: fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateRunning},
+ Background: true,
+ }}}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil).
+ WithSkillEnvironment(shellTestSkillEnvironment(t)).
+ WithEnvCapture(recorder.capture)
+ result, err := tool.Execute(asyncTestCtx(), json.RawMessage(
+ `{"command":"export USER_TOKEN=from-command; python x.py","skill_name":"pdf-tools"}`,
+ ))
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ require.Zero(t, recorder.calls)
+ require.NotNil(t, submitter.held)
+ submitter.held(context.Background())
+ require.Equal(t, 1, recorder.calls)
+ require.Equal(t, "from-command", recorder.pairs["USER_TOKEN"])
+}
+
+func TestShellExecAsyncDoesNotCaptureAFailedInWindowJob(t *testing.T) {
+ recorder := &recordedCapture{}
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateFailed},
+ }}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil).
+ WithSkillEnvironment(shellTestSkillEnvironment(t)).
+ WithEnvCapture(recorder.capture)
+ result, err := tool.Execute(asyncTestCtx(), json.RawMessage(
+ `{"command":"export USER_TOKEN=from-command; python x.py","skill_name":"pdf-tools"}`,
+ ))
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ require.Zero(t, recorder.calls)
+}
+
+func TestShellExecAsyncDoesNotRewriteResolvedSkillEnv(t *testing.T) {
+ recorder := &recordedCapture{}
+ submitter := &fakeSubmitter{response: ShellJobSubmission{
+ Result: interfaces.WorkResult{ID: "sj_0123", State: interfaces.WorkStateSucceeded},
+ }}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, stubEnvResolver{
+ resolved: map[string]string{"USER_TOKEN": "already-stored"},
+ }).
+ WithSkillEnvironment(shellTestSkillEnvironment(t)).
+ WithEnvCapture(recorder.capture)
+ result, err := tool.Execute(asyncTestCtx(), json.RawMessage(
+ `{"command":"export USER_TOKEN=from-command; python x.py",`+
+ `"skill_name":"pdf-tools","env":{"EXTRA_TOKEN":"from-tool"}}`,
+ ))
+ require.NoError(t, err)
+ require.True(t, result.Success)
+ require.Equal(t, 1, recorder.calls)
+ require.NotContains(t, recorder.pairs, "USER_TOKEN")
+ require.Equal(t, "from-tool", recorder.pairs["EXTRA_TOKEN"])
+}
+
+func TestShellExecDescriptionPointsAtAsync(t *testing.T) {
+ needle := "use `shell_exec_async` instead: short commands still return their result directly."
+ require.Contains(t, shellExecDescription(sandbox.RemoteWorkspaceLayout()), needle)
+ require.Contains(t, shellExecDescription(sandbox.WorkspaceLayout{
+ Origin: sandbox.WorkspaceOriginHost, Root: "/Users/dev/proj",
+ }), needle)
+}
+
+func TestShellExecAsyncCleanupRunsFromRegistry(t *testing.T) {
+ submitter := &fakeSubmitter{}
+ tool := NewShellExecAsyncTool(submitter, &fakeShellExecutor{}, nil)
+ tool.BindBackgroundRun(interfaces.SessionRef{TenantID: 7, SessionID: "session-1"}, "m1", &recordingMarks{})
+ registry := NewToolRegistry()
+ registry.RegisterTool(tool)
+ registry.Cleanup(context.Background())
+ require.Equal(t, 1, submitter.ended)
+}
diff --git a/internal/agent/tools/shell_exec_test.go b/internal/agent/tools/shell_exec_test.go
index 9cf684a11..cc47864d2 100644
--- a/internal/agent/tools/shell_exec_test.go
+++ b/internal/agent/tools/shell_exec_test.go
@@ -231,7 +231,7 @@ func TestShellExecDescriptionDefinesOneExecutionEntry(t *testing.T) {
require.Contains(t, description, fact)
}
require.NotContains(t, description, "execute_skill_script")
- require.Less(t, len(description), 2500)
+ require.Less(t, len(description), 2800)
}
func TestShellExecBoundsStdoutStderrErrorAndTotal(t *testing.T) {
diff --git a/internal/agent/tools/shell_job_request.go b/internal/agent/tools/shell_job_request.go
new file mode 100644
index 000000000..1920a341c
--- /dev/null
+++ b/internal/agent/tools/shell_job_request.go
@@ -0,0 +1,47 @@
+package tools
+
+import (
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+// ShellJobSubmitRequest is the frozen submit payload shared by the remote
+// shell-job service and the async shell tool.
+type ShellJobSubmitRequest struct {
+ Command string
+ // DisplayCommand is the original command, before stdin, cwd and skill
+ // wrappers. Only this command belongs in persistent metadata and results.
+ DisplayCommand string
+ WorkDir string
+ Env map[string]string
+ Description string
+ YieldTime time.Duration
+ ParentMessageID string
+ ToolCallID string
+ RunID string
+ OnOutput func(string, []byte)
+ // OutputDir is the deliverable directory, same as shell_exec. OutputSeen
+ // reports that OutputBefore is a complete listing taken before the command.
+ OutputDir string
+ OutputSeen bool
+ OutputBefore []sandbox.OutputFileStamp
+}
+
+// MaskedCommand keeps execution-only wrappers out of metadata. Direct service
+// callers without a separate display command retain the existing fallback.
+func (r ShellJobSubmitRequest) MaskedCommand() string {
+ command := r.DisplayCommand
+ if command == "" {
+ command = r.Command
+ }
+ return utils.MaskCommandAssignments(command)
+}
+
+// ShellJobSubmission is the direct result of one SubmitShellJob call.
+type ShellJobSubmission struct {
+ Result interfaces.WorkResult
+ Background bool
+}
diff --git a/internal/application/service/agent_service.go b/internal/application/service/agent_service.go
index 8a5d56eaa..bfd973410 100644
--- a/internal/application/service/agent_service.go
+++ b/internal/application/service/agent_service.go
@@ -17,6 +17,7 @@ import (
"github.com/Tencent/WeKnora/internal/application/access"
"github.com/Tencent/WeKnora/internal/application/attachment"
"github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/background"
"github.com/Tencent/WeKnora/internal/browserskill"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/event"
@@ -131,6 +132,18 @@ type agentService struct {
hostSkillInstaller sandbox.Manager
hostSkillsRoot string
hostSkillVersionsRoot string
+ background *BackgroundWork
+ // shellJobs is an interface so registration can assert ShellJobSubmitter.
+ // *ShellJobService is the production value. && binds tighter than ||, so
+ // the availability check groups RemoteAvailable and LocalAvailable.
+ shellJobs shellJobSource
+}
+
+// shellJobSource is the part of *ShellJobService the agent asks about
+// availability. Submit is asserted to tools.ShellJobSubmitter at registration.
+type shellJobSource interface {
+ RemoteAvailable() bool
+ LocalAvailable() bool
}
// NewAgentService creates a new agent service
@@ -161,6 +174,8 @@ func NewAgentService(
browserSkill *browserskill.Manager,
userRepo interfaces.UserRepository,
graphRepo interfaces.RetrieveGraphRepository,
+ backgroundWork *BackgroundWork,
+ shellJobs *ShellJobService,
operationDispatcher *operations.HandlerDispatcher,
customAgentService interfaces.CustomAgentService,
collaboration *AgentCollaboration,
@@ -195,6 +210,8 @@ func NewAgentService(
sandboxPolicy: sandboxPolicy,
hostSandbox: hostSandbox.Manager,
hostDesktop: hostSandbox.Desktop,
+ background: backgroundWork,
+ shellJobs: shellJobs,
}
if hostSandbox.SkillsAvailable() {
svc.hostSkillInstaller = hostSandbox.SkillInstaller
@@ -344,12 +361,61 @@ func (s *agentService) CreateAgentEngine(
return nil, err
}
}
+ s.attachBackgroundWork(ctx, engine, toolRegistry, sessionID, assistantMessageID)
toolRegistry.BindSession(sessionID)
engine.SetWorkspaceLayout(s.lookupSessionWorkspaceLayout(ctx, sessionID, config))
return engine, nil
}
+// attachBackgroundWork gives the run its background-notice tracker and, when
+// the run can submit background work, the work_wait and work_cancel tools.
+// Tools are decided here, once per run, and never change mid-run (D10).
+func (s *agentService) attachBackgroundWork(
+ ctx context.Context, engine *agent.AgentEngine, registry *tools.ToolRegistry,
+ sessionID, assistantMessageID string,
+) {
+ if !s.background.Enabled() {
+ return
+ }
+ tenantID, ok := types.SandboxTenantIDFromContext(ctx)
+ if !ok || tenantID == 0 {
+ return
+ }
+ ref := interfaces.SessionRef{TenantID: tenantID, SessionID: sessionID}
+ run := background.NewRunNotices(ref, s.background.Registry, s.background.Inbox)
+ engine.SetBackgroundRun(run)
+
+ submitters := tools.RegisteredBackgroundSubmitters(registry)
+ if len(submitters) == 0 {
+ return
+ }
+ for _, name := range submitters {
+ tool, err := registry.GetTool(name)
+ if err != nil {
+ continue
+ }
+ if async, ok := tool.(*tools.ShellExecAsyncTool); ok {
+ async.BindBackgroundRun(ref, assistantMessageID, run)
+ continue
+ }
+ // Tests wrap the real tool. The method is the same one: the run's
+ // SessionRef and RunNotices, whose MarkStarted records a running handle.
+ if binder, ok := tool.(interface {
+ BindBackgroundRun(interfaces.SessionRef, string, interface{ MarkStarted(string) })
+ }); ok {
+ binder.BindBackgroundRun(ref, assistantMessageID, run)
+ }
+ }
+ pending := func(ctx context.Context) bool {
+ return engine.HasPendingInput(ctx, sessionID, assistantMessageID)
+ }
+ registry.RegisterTool(tools.NewWorkWaitTool(
+ s.background.Handler, ref, assistantMessageID, run, pending, submitters,
+ ))
+ registry.RegisterTool(tools.NewWorkCancelTool(s.background.Handler, ref, run))
+}
+
// registerMCPTools registers MCP tools from enabled services for this tenant.
func (s *agentService) registerMCPTools(
ctx context.Context,
@@ -827,6 +893,11 @@ func (s *agentService) initializeSkillsManager(
shellEnabled = true
}
}
+ if tool, err := toolRegistry.GetTool(tools.ToolShellExecAsync); err == nil {
+ if async, ok := tool.(*tools.ShellExecAsyncTool); ok {
+ async.WithSkillEnvironment(skillsManager)
+ }
+ }
if config.SkillsEnabled {
var reader *tools.ReadFileTool
if tool, err := toolRegistry.GetTool(tools.ToolReadFile); err == nil {
@@ -925,7 +996,7 @@ func (s *agentService) loadInstalledSkillBundle(
// config-wide variables are injected into every execution, skills or not.
//
// The repository is constructed from s.db here rather than injected:
-// NewAgentService already takes 23 parameters, and this is the only place in
+// NewAgentService already takes 24 parameters, and this is the only place in
// the agent path that touches the user env table.
func (s *agentService) userEnvResolver(
ctx context.Context, config *types.AgentConfig,
@@ -1109,11 +1180,33 @@ func (s *agentService) registerSandboxShellTool(
tools.NewShellExecTool(executor, resolver).WithEnvCapture(s.skillEnvCapture(config)),
)
logger.Infof(ctx, "Registered shell_exec tool")
+ s.registerShellExecAsync(ctx, toolRegistry, executor, resolver, config)
} else {
logger.Infof(ctx, "Sandbox backend does not advertise remote shell capability; shell_exec not registered")
}
}
+// registerShellExecAsync registers the background shell when this process can
+// submit a job. The submitter assertion and the availability check are both
+// required. && binds tighter than ||, so the availability disjunction is grouped.
+func (s *agentService) registerShellExecAsync(
+ ctx context.Context,
+ toolRegistry *tools.ToolRegistry,
+ executor tools.SandboxCommandExecutor,
+ resolver skills.SkillEnvResolver,
+ config *types.AgentConfig,
+) {
+ if s == nil || !s.background.Enabled() || s.shellJobs == nil {
+ return
+ }
+ if submitter, ok := s.shellJobs.(tools.ShellJobSubmitter); ok &&
+ (s.shellJobs.RemoteAvailable() || s.shellJobs.LocalAvailable()) {
+ toolRegistry.RegisterTool(tools.NewShellExecAsyncTool(submitter, executor, resolver).
+ WithEnvCapture(s.skillEnvCapture(config)))
+ logger.Infof(ctx, "Registered shell_exec_async tool")
+ }
+}
+
// registerTools registers tools based on the agent configuration
func (s *agentService) registerTools(
ctx context.Context,
diff --git a/internal/application/service/agent_service_background_test.go b/internal/application/service/agent_service_background_test.go
new file mode 100644
index 000000000..314d8767b
--- /dev/null
+++ b/internal/application/service/agent_service_background_test.go
@@ -0,0 +1,113 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent"
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type stubWorkHandler struct{ interfaces.WorkHandler }
+
+func newBackgroundTestEngine(t *testing.T, registry *tools.ToolRegistry) *agent.AgentEngine {
+ t.Helper()
+ engine := agent.NewAgentEngine(&types.AgentConfig{MaxIterations: 1}, nil, registry, nil, nil, nil, "s", "")
+ require.NotNil(t, engine)
+ return engine
+}
+
+func TestBackgroundWorkIsDisabledWithoutAHandler(t *testing.T) {
+ var nilWork *BackgroundWork
+ require.False(t, nilWork.Enabled())
+ work := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ require.False(t, work.Enabled())
+ require.NotNil(t, work.Listeners)
+ work.Handler = stubWorkHandler{}
+ require.True(t, work.Enabled())
+}
+
+func TestAttachBackgroundWorkRegistersNoWorkToolsWithoutASubmitter(t *testing.T) {
+ work := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ work.Handler = stubWorkHandler{}
+ s := &agentService{background: work}
+ registry := tools.NewToolRegistry()
+ engine := newBackgroundTestEngine(t, registry)
+ ctx := types.WithSandboxTenantID(context.Background(), 7)
+
+ s.attachBackgroundWork(ctx, engine, registry, "s", "m")
+
+ _, err := registry.GetTool(tools.ToolWorkWait)
+ require.Error(t, err, "no submission tool is registered, so the run cannot start background work")
+ _, err = registry.GetTool(tools.ToolWorkCancel)
+ require.Error(t, err)
+}
+
+var (
+ stubAsyncToolBound bool
+ stubAsyncBoundRef interfaces.SessionRef
+ stubAsyncBoundRun string
+ stubAsyncMarks interface{ MarkStarted(string) }
+)
+
+// stubAsyncTool wraps a real async shell tool. attachBackgroundWork binds by
+// the BindBackgroundRun method, which this wrapper records.
+type stubAsyncTool struct {
+ *tools.ShellExecAsyncTool
+}
+
+func (s *stubAsyncTool) BindBackgroundRun(
+ ref interfaces.SessionRef, runID string, marks interface{ MarkStarted(string) },
+) {
+ stubAsyncToolBound = true
+ stubAsyncBoundRef = ref
+ stubAsyncBoundRun = runID
+ stubAsyncMarks = marks
+ s.ShellExecAsyncTool.BindBackgroundRun(ref, runID, marks)
+}
+
+func TestAttachBackgroundWorkBindsAsyncSubmitter(t *testing.T) {
+ stubAsyncToolBound = false
+ stubAsyncBoundRef = interfaces.SessionRef{}
+ stubAsyncBoundRun = ""
+ stubAsyncMarks = nil
+
+ work := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ work.Handler = stubWorkHandler{}
+ s := &agentService{background: work}
+ registry := tools.NewToolRegistry()
+ engine := newBackgroundTestEngine(t, registry)
+ registry.RegisterTool(&stubAsyncTool{ShellExecAsyncTool: tools.NewShellExecAsyncTool(nil, nil, nil)})
+ ctx := types.WithSandboxTenantID(context.Background(), 7)
+
+ s.attachBackgroundWork(ctx, engine, registry, "s", "m")
+
+ _, err := registry.GetTool(tools.ToolWorkWait)
+ require.NoError(t, err, "有提交工具时 work_* 必须注册(D10 静态)")
+ require.True(t, stubAsyncToolBound, "BindBackgroundRun 被调用(D11)")
+ require.Equal(t, interfaces.SessionRef{TenantID: 7, SessionID: "s"}, stubAsyncBoundRef)
+ require.Equal(t, "m", stubAsyncBoundRun)
+ notices, ok := stubAsyncMarks.(*background.RunNotices)
+ require.True(t, ok, "BindBackgroundRun receives the run's RunNotices")
+ require.NoError(t, work.Registry.Add(ctx, stubAsyncBoundRef, interfaces.BackgroundActivity{
+ ID: "sj_probe", Kind: "shell", Title: "probe", StartedAt: time.Now(),
+ }, time.Minute, 0))
+ notices.MarkStarted("sj_probe")
+ delivery, err := notices.BeforeFinal(ctx)
+ require.NoError(t, err)
+ require.Contains(t, delivery.Text, "sj_probe")
+}
+
+func TestAttachBackgroundWorkIsANoOpWhenDisabled(t *testing.T) {
+ registry := tools.NewToolRegistry()
+ engine := newBackgroundTestEngine(t, registry)
+ for _, s := range []*agentService{{}, {background: NewBackgroundWork(nil, nil)}} {
+ s.attachBackgroundWork(context.Background(), engine, registry, "s", "m")
+ }
+ require.Empty(t, registry.ListTools())
+}
diff --git a/internal/application/service/background_events.go b/internal/application/service/background_events.go
new file mode 100644
index 000000000..22c201c52
--- /dev/null
+++ b/internal/application/service/background_events.go
@@ -0,0 +1,125 @@
+package service
+
+import (
+ "context"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/event"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// BackgroundEventEmitter publishes the background work events of shared
+// design §7. Features call it; they never pick the stream themselves.
+type BackgroundEventEmitter struct {
+ streams interfaces.StreamManager
+}
+
+// NewBackgroundEventEmitter returns an emitter that publishes background work
+// events on the stream selected by the shared live-run rules.
+func NewBackgroundEventEmitter(streams interfaces.StreamManager) *BackgroundEventEmitter {
+ return &BackgroundEventEmitter{streams: streams}
+}
+
+// ParentLive reports whether parentMessageID is the session's live run.
+func (e *BackgroundEventEmitter) ParentLive(
+ ctx context.Context, s interfaces.SessionRef, parentMessageID string,
+) (bool, error) {
+ if e == nil || e.streams == nil {
+ return false, nil
+ }
+ live, _, err := e.streams.GetLiveRun(ctx, s.SessionID)
+ if err != nil {
+ return false, err
+ }
+ return live != "" && live == parentMessageID, nil
+}
+
+// EmitCommandOutput previews running output on the parent's stream. When the
+// parent run is no longer live the event is dropped: the process can be
+// replayed from the feature's own storage, and appending after complete would
+// corrupt replay.
+func (e *BackgroundEventEmitter) EmitCommandOutput(
+ ctx context.Context, s interfaces.SessionRef, parentMessageID, workID string,
+ data event.CommandOutputData,
+) error {
+ if e == nil || e.streams == nil {
+ return nil
+ }
+ live, err := e.ParentLive(ctx, s, parentMessageID)
+ if err != nil {
+ return err
+ }
+ if !live {
+ return nil
+ }
+ return e.streams.AppendEvent(ctx, s.SessionID, parentMessageID, interfaces.StreamEvent{
+ Type: types.ResponseTypeWorkEvent, Timestamp: time.Now(),
+ Data: map[string]interface{}{
+ "work_id": workID, "kind": "shell",
+ "inner": map[string]interface{}{
+ "response_type": "command_output",
+ "data": map[string]interface{}{
+ "tool_call_id": data.ToolCallID, "command": data.Command,
+ "started_at": data.StartedAt, "output": data.Output, "done": data.Done,
+ },
+ },
+ },
+ })
+}
+
+// EmitWorkStarted routes to the parent stream when live, else to the session's
+// current live run, else nowhere.
+func (e *BackgroundEventEmitter) EmitWorkStarted(
+ ctx context.Context, s interfaces.SessionRef, parentMessageID string,
+ a interfaces.BackgroundActivity, toolCallID string,
+) error {
+ target, err := e.route(ctx, s, parentMessageID)
+ if err != nil || target == "" {
+ return err
+ }
+ return e.streams.AppendEvent(ctx, s.SessionID, target, interfaces.StreamEvent{
+ Type: types.ResponseTypeWorkStarted, Timestamp: time.Now(),
+ Data: map[string]interface{}{
+ "work_id": a.ID, "kind": a.Kind, "title": a.Title,
+ "parent_tool_call_id": toolCallID, "started_at": a.StartedAt,
+ },
+ })
+}
+
+// EmitWorkFinished publishes work_finished. The parent stream is used when
+// that run is live; otherwise the session's current live run is used.
+func (e *BackgroundEventEmitter) EmitWorkFinished(
+ ctx context.Context, s interfaces.SessionRef, parentMessageID string,
+ r interfaces.WorkResult,
+) error {
+ target, err := e.route(ctx, s, parentMessageID)
+ if err != nil || target == "" {
+ return err
+ }
+ return e.streams.AppendEvent(ctx, s.SessionID, target, interfaces.StreamEvent{
+ Type: types.ResponseTypeWorkFinished, Timestamp: time.Now(),
+ Data: map[string]interface{}{
+ "work_id": r.ID, "kind": "shell", "state": r.State, "payload": r.Payload,
+ },
+ })
+}
+
+func (e *BackgroundEventEmitter) route(
+ ctx context.Context, s interfaces.SessionRef, parentMessageID string,
+) (string, error) {
+ if e == nil || e.streams == nil {
+ return "", nil
+ }
+ live, _, err := e.streams.GetLiveRun(ctx, s.SessionID)
+ if err != nil {
+ return "", err
+ }
+ if live == "" {
+ return "", nil
+ }
+ if live == parentMessageID {
+ return parentMessageID, nil
+ }
+ return live, nil
+}
diff --git a/internal/application/service/background_events_test.go b/internal/application/service/background_events_test.go
new file mode 100644
index 000000000..4b2eac708
--- /dev/null
+++ b/internal/application/service/background_events_test.go
@@ -0,0 +1,92 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/event"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type fakeStreamManager struct {
+ interfaces.StreamManager
+ appended []struct {
+ session, message string
+ ev interfaces.StreamEvent
+ }
+ live map[string]string // session -> assistant message
+}
+
+func (f *fakeStreamManager) AppendEvent(_ context.Context, session, message string, ev interfaces.StreamEvent) error {
+ f.appended = append(f.appended, struct {
+ session string
+ message string
+ ev interfaces.StreamEvent
+ }{session, message, ev})
+ return nil
+}
+
+func (f *fakeStreamManager) GetLiveRun(_ context.Context, session string) (string, string, error) {
+ if id, ok := f.live[session]; ok {
+ return id, "r", nil
+ }
+ return "", "", nil
+}
+
+func TestCommandOutputOnlyWhenParentLive(t *testing.T) {
+ streams := &fakeStreamManager{live: map[string]string{"s": "parent"}}
+ emitter := NewBackgroundEventEmitter(streams)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, emitter.EmitCommandOutput(context.Background(), ref, "parent", "sj_0123",
+ event.CommandOutputData{ToolCallID: "tc1", Output: "half"}))
+ require.Len(t, streams.appended, 1)
+ require.Equal(t, types.ResponseTypeWorkEvent, streams.appended[0].ev.Type)
+ require.Equal(t, "sj_0123", streams.appended[0].ev.Data["work_id"])
+ inner := streams.appended[0].ev.Data["inner"].(map[string]any)
+ require.Equal(t, "command_output", inner["response_type"])
+
+ delete(streams.live, "s")
+ require.NoError(t, emitter.EmitCommandOutput(context.Background(), ref, "parent", "sj_0123",
+ event.CommandOutputData{Output: "more"}))
+ require.Len(t, streams.appended, 1, "父 run 不 live 时丢弃 work_event")
+}
+
+func TestWorkStartedFallsBackToCurrentLiveRun(t *testing.T) {
+ streams := &fakeStreamManager{live: map[string]string{"s": "other"}}
+ emitter := NewBackgroundEventEmitter(streams)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ activity := interfaces.BackgroundActivity{
+ ID: "sj_0123", Kind: "shell",
+ Title: "build", StartedAt: time.Unix(100, 0),
+ }
+ require.NoError(t, emitter.EmitWorkStarted(context.Background(), ref, "parent", activity, "tc1"))
+ require.Len(t, streams.appended, 1)
+ require.Equal(t, "other", streams.appended[0].message, "路由到当前 live run")
+ require.Equal(t, types.ResponseTypeWorkStarted, streams.appended[0].ev.Type)
+ require.Equal(t, "tc1", streams.appended[0].ev.Data["parent_tool_call_id"])
+
+ streams.live = map[string]string{}
+ require.NoError(t, emitter.EmitWorkStarted(context.Background(), ref, "parent", activity, "tc1"))
+ require.Len(t, streams.appended, 1, "没有任何 live run 时不写事件")
+}
+
+func TestWorkFinishedCarriesStateAndPayload(t *testing.T) {
+ streams := &fakeStreamManager{live: map[string]string{"s": "parent"}}
+ emitter := NewBackgroundEventEmitter(streams)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ result := interfaces.WorkResult{
+ ID: "sj_0123", State: interfaces.WorkStateSucceeded,
+ Terminal: true, Payload: map[string]any{"exit_code": 0},
+ }
+ require.NoError(t, emitter.EmitWorkFinished(context.Background(), ref, "parent", result))
+ require.Len(t, streams.appended, 1)
+ require.Equal(t, types.ResponseTypeWorkFinished, streams.appended[0].ev.Type)
+ require.Equal(t, interfaces.WorkStateSucceeded, streams.appended[0].ev.Data["state"])
+
+ live, err := emitter.ParentLive(context.Background(), ref, "parent")
+ require.NoError(t, err)
+ require.True(t, live)
+}
diff --git a/internal/application/service/local_shell_job_drain_test.go b/internal/application/service/local_shell_job_drain_test.go
new file mode 100644
index 000000000..0ee690379
--- /dev/null
+++ b/internal/application/service/local_shell_job_drain_test.go
@@ -0,0 +1,105 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "io"
+ "os/exec"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/localsandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type drainingShellProcess struct {
+ cmd *exec.Cmd
+ stdout, stderr io.ReadCloser
+}
+
+func (p *drainingShellProcess) Stdout() io.Reader { return p.stdout }
+func (p *drainingShellProcess) Stderr() io.Reader { return p.stderr }
+func (p *drainingShellProcess) PID() int { return p.cmd.Process.Pid }
+func (p *drainingShellProcess) Kill() error { return p.cmd.Process.Kill() }
+func (p *drainingShellProcess) Wait(context.Context) (localsandbox.ExitStatus, error) {
+ err := p.cmd.Wait()
+ return localsandbox.ExitStatus{Code: p.cmd.ProcessState.ExitCode()}, err
+}
+
+type drainingShellStarter struct{ root string }
+
+func (s drainingShellStarter) GuardForSession(
+ context.Context,
+ string,
+) (*localsandbox.PathGuard, localsandbox.Workspace, error) {
+ return nil, localsandbox.Workspace{Root: s.root}, nil
+}
+
+func (s drainingShellStarter) StartForSession(
+ ctx context.Context,
+ r localsandbox.RunRequest,
+) (localsandbox.Process, error) {
+ cmd := exec.CommandContext(ctx, "/bin/sh", "-c", r.Command)
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return nil, err
+ }
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return nil, err
+ }
+ if err = cmd.Start(); err != nil {
+ return nil, err
+ }
+ return &drainingShellProcess{cmd: cmd, stdout: stdout, stderr: stderr}, nil
+}
+
+func TestLocalShellJobDrainsPipesBeforeWait(t *testing.T) {
+ base := t.TempDir()
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc, err := newLocalShellJobService(drainingShellStarter{root: base}, bg, nil, filepath.Join(base, "jobs"),
+ func(context.Context, int) (int64, error) { return 1, nil })
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, svc.Shutdown(context.Background())) })
+ result, err := svc.SubmitShellJob(
+ context.Background(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ tools.ShellJobSubmitRequest{
+ Command: "printf BEGIN; printf BEGIN-ERR >&2; sleep 0.1; printf FINAL-DIAGNOSTIC; printf FINAL-ERROR >&2",
+ YieldTime: 2 * time.Second,
+ OnOutput: func(string, []byte) { time.Sleep(400 * time.Millisecond) },
+ },
+ )
+ require.NoError(t, err)
+ p := result.Result.Payload.(ShellJobPayload)
+ require.Equal(t, interfaces.WorkStateSucceeded, p.State)
+ require.Contains(t, p.Stderr, "FINAL-ERROR")
+ require.Contains(
+ t,
+ p.Stdout,
+ "FINAL-DIAGNOSTIC",
+ "Wait closes the pipes while the output copier is still forwarding an earlier chunk",
+ )
+}
+
+func TestLocalShellJobFailedDeletionReopensSubmissions(t *testing.T) {
+ svc, proc, _ := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, svc.OnDelete(t.Context(), ref))
+ _, err := svc.SubmitShellJob(t.Context(), ref, tools.ShellJobSubmitRequest{Command: "true"})
+ require.Error(t, err)
+ require.NoError(t, svc.OnDeleteAborted(t.Context(), ref))
+ proc.finish()
+ result, err := svc.SubmitShellJob(
+ t.Context(),
+ ref,
+ tools.ShellJobSubmitRequest{Command: "true", YieldTime: time.Second},
+ )
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateSucceeded, result.Result.State)
+}
diff --git a/internal/application/service/local_shell_job_followup_test.go b/internal/application/service/local_shell_job_followup_test.go
new file mode 100644
index 000000000..6b2f1e717
--- /dev/null
+++ b/internal/application/service/local_shell_job_followup_test.go
@@ -0,0 +1,128 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+// Layout falls back to the remote layout; the explicit working directory below
+// selects the test workspace. The async tool never executes through this stub.
+type shellJobToolExecutor struct{ tools.SandboxCommandExecutor }
+
+func TestLocalShellJobLargeStdinPreservesResultJSON(t *testing.T) {
+ for _, command := range []string{"cat", "tee /dev/stderr #" + strings.Repeat("\x01", 8192-17)} {
+ t.Run(map[bool]string{false: "stdin", true: "escaped-command"}[len(command) > 3], func(t *testing.T) {
+ root := t.TempDir()
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc, err := newLocalShellJobService(drainingShellStarter{root: root}, bg, nil,
+ filepath.Join(root, "jobs"), func(context.Context, int) (int64, error) { return 1, nil })
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, svc.Shutdown(context.Background())) })
+ registry := tools.NewToolRegistry()
+ registry.RegisterTool(tools.NewShellExecAsyncTool(svc, shellJobToolExecutor{}, nil))
+ ctx := tools.WithToolExecContext(types.WithSandboxTenantID(t.Context(), 7),
+ &tools.ToolExecContext{SessionID: "s"})
+ input, err := json.Marshal(tools.ShellExecAsyncInput{
+ Command: command, Stdin: strings.Repeat("\x01", 65536), WorkDir: root, Description: "copy input",
+ })
+ require.NoError(t, err)
+ result, err := registry.ExecuteTool(ctx, tools.ToolShellExecAsync, input)
+ require.NoError(t, err)
+ require.True(t, result.Success, result.Error)
+ require.True(t, json.Valid([]byte(result.Output)), "the registry truncated a valid large-stdin result")
+ var payload ShellJobPayload
+ require.NoError(t, json.Unmarshal([]byte(result.Output), &payload))
+ require.Equal(t, interfaces.WorkStateSucceeded, payload.State)
+ require.Equal(t, strings.Repeat("\x01", 16*1024), payload.Stdout)
+ require.Equal(t, int64(65536), payload.TotalBytes.Stdout)
+ require.Equal(t, command, payload.Command, "display the user's command, not the stdin transport")
+ var meta sandbox.ShellJobMeta
+ require.NoError(t, readLocalJobJSON(filepath.Join(svc.root, payload.WorkID, "meta.json"), &meta))
+ require.Equal(t, command, meta.Command, "the stdin transport must not be persisted either")
+ if command != "cat" {
+ require.Equal(t, strings.Repeat("\x01", 8*1024), payload.Stderr)
+ }
+ })
+ }
+}
+
+func TestLocalShellJobRecoveryReadsLargeLegacyMetadata(t *testing.T) {
+ root := t.TempDir()
+ ended := time.Now().UTC()
+ command := "printf %s " + base64.StdEncoding.EncodeToString([]byte(strings.Repeat("x", 65536))) +
+ " | base64 -d | cat"
+ var signalled []int
+ original := signalRecoveredProcess
+ signalRecoveredProcess = func(_ context.Context, pid int, _ int64) error {
+ signalled = append(signalled, pid)
+ return nil
+ }
+ t.Cleanup(func() { signalRecoveredProcess = original })
+ for i, state := range []string{interfaces.WorkStateRunning, interfaces.WorkStateSucceeded} {
+ id := []string{"sj_b001", "sj_b002"}[i]
+ dir := writeLocalJob(t, root, id, sandbox.ShellJobMeta{
+ ID: id, TenantID: 7, SessionID: "s", Command: command,
+ }, sandbox.ShellJobStatus{
+ State: state, PGID: 4321, ProcStartedAt: 1, StartedAt: ended, EndedAt: &ended,
+ })
+ info, err := os.Stat(filepath.Join(dir, "meta.json"))
+ require.NoError(t, err)
+ require.Greater(t, info.Size(), int64(65536))
+ }
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := &LocalShellJobService{root: root, bg: bg, jobs: map[string]*localJob{}}
+ require.NoError(t, svc.recoverJobs(t.Context()))
+ require.Len(t, svc.jobs, 2)
+ require.Equal(t, []int{4321}, signalled, "the orphan must still pass through identity-checked cleanup")
+ require.Equal(t, interfaces.WorkStateLost, svc.jobs["sj_b001"].status.State)
+ notices, err := bg.Inbox.Pending(t.Context(), interfaces.SessionRef{TenantID: 7, SessionID: "s"})
+ require.NoError(t, err)
+ require.Len(t, notices, 2)
+}
+
+func TestLocalShellJobCaptureRegisteredAfterReportSettles(t *testing.T) {
+ for _, code := range []int{0, 1} {
+ t.Run(map[int]string{0: "success", 1: "failure"}[code], func(t *testing.T) {
+ svc, proc, _ := localFixture(t)
+ proc.exit.Code = code
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(t.Context(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background)
+ job, err := svc.ownedJob(ref, sub.Result.ID)
+ require.NoError(t, err)
+ proc.finish()
+ select {
+ case <-job.reported:
+ case <-time.After(time.Second):
+ t.Fatal("watcher did not finish reporting")
+ }
+ var calls atomic.Int32
+ svc.HoldSkillEnvCapture(ref, sub.Result.ID, func(context.Context) { calls.Add(1) })
+ want := int32(0)
+ if code == 0 {
+ want = 1
+ }
+ require.Equal(t, want, calls.Load(), "a completed success must not lose the late write-back")
+ require.Empty(t, svc.skillCapture.pending(), "terminal failures must drop late credentials too")
+ })
+ }
+}
diff --git a/internal/application/service/local_shell_job_identity_darwin.go b/internal/application/service/local_shell_job_identity_darwin.go
new file mode 100644
index 000000000..8fafb228a
--- /dev/null
+++ b/internal/application/service/local_shell_job_identity_darwin.go
@@ -0,0 +1,48 @@
+//go:build desktop && darwin
+
+package service
+
+import (
+ "context"
+ "errors"
+ "syscall"
+
+ "github.com/shirou/gopsutil/v3/process"
+)
+
+func localProcessStartedAt(ctx context.Context, pid int) (int64, error) {
+ proc, err := process.NewProcess(int32(pid))
+ if err != nil {
+ return 0, err
+ }
+ return proc.CreateTimeWithContext(ctx)
+}
+
+// localRecoverProcess kills a leftover job process group only when the process
+// at pgid still is the one the dead watcher recorded (PID-reuse guard).
+func localRecoverProcess(ctx context.Context, pgid int, expected int64) error {
+ actual, err := localProcessStartedAt(ctx, pgid)
+ if errors.Is(err, process.ErrorProcessNotRunning) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if actual != expected {
+ return nil // reused PID: do not signal
+ }
+ group, err := syscall.Getpgid(pgid)
+ if errors.Is(err, syscall.ESRCH) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if group != pgid {
+ return nil
+ }
+ if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) {
+ return err
+ }
+ return nil
+}
diff --git a/internal/application/service/local_shell_job_identity_darwin_test.go b/internal/application/service/local_shell_job_identity_darwin_test.go
new file mode 100644
index 000000000..b3a35c5a1
--- /dev/null
+++ b/internal/application/service/local_shell_job_identity_darwin_test.go
@@ -0,0 +1,20 @@
+//go:build desktop && darwin
+
+package service
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestRecoveryDoesNotSignalReusedPID(t *testing.T) {
+ stamp, err := localProcessStartedAt(context.Background(), os.Getpid())
+ require.NoError(t, err)
+ require.NotZero(t, stamp)
+ // 启动时间不匹配(PID 被复用)→ 不发信号。本测试用错误时间戳,永远到不了 kill。
+ require.NoError(t, localRecoverProcess(context.Background(), os.Getpid(), stamp+1))
+ require.NoError(t, localRecoverProcess(context.Background(), os.Getpid(), 1))
+}
diff --git a/internal/application/service/local_shell_job_identity_other.go b/internal/application/service/local_shell_job_identity_other.go
new file mode 100644
index 000000000..23cb987fa
--- /dev/null
+++ b/internal/application/service/local_shell_job_identity_other.go
@@ -0,0 +1,12 @@
+//go:build desktop && !darwin
+
+package service
+
+import "context"
+
+// This platform cannot stamp a process start time, so a crash cannot tell a
+// reused pid from the original child. Leave the stamp at 0: recovery then
+// does not signal, and submit does not kill a command that is already running.
+func localProcessStartedAt(context.Context, int) (int64, error) { return 0, nil }
+
+func localRecoverProcess(context.Context, int, int64) error { return nil }
diff --git a/internal/application/service/local_shell_job_log.go b/internal/application/service/local_shell_job_log.go
new file mode 100644
index 000000000..5e0b3b4c7
--- /dev/null
+++ b/internal/application/service/local_shell_job_log.go
@@ -0,0 +1,110 @@
+package service
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strconv"
+)
+
+const localJobSegmentBytes = 16 * 1024 * 1024
+
+// localJobLog appends one output stream to disk with the remote rotation
+// protocol: at 16 MiB keep an 8 KiB .head once, move the segment to .1
+// (overwriting), accumulate .rotated, reopen fresh. O_APPEND keeps the
+// truncate-and-reopen free of sparse holes (shell 稿 §9.2 wrapper 步骤 4)。
+type localJobLog struct {
+ path string
+ file *os.File
+ size int64
+ rotated int64
+}
+
+func newLocalJobLog(path string) (*localJobLog, error) {
+ f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+ if err != nil {
+ return nil, err
+ }
+ return &localJobLog{path: path, file: f}, nil
+}
+
+func (l *localJobLog) Write(p []byte) (int, error) {
+ total := 0
+ for len(p) > 0 {
+ if l.size >= localJobSegmentBytes {
+ if err := l.rotate(); err != nil {
+ return total, err
+ }
+ }
+ room := int64(localJobSegmentBytes) - l.size
+ n, err := l.file.Write(p[:min(int64(len(p)), room)])
+ l.size += int64(n)
+ total += n
+ p = p[n:]
+ if err != nil {
+ return total, err
+ }
+ if n == 0 {
+ return total, io.ErrShortWrite
+ }
+ }
+ return total, nil
+}
+
+func (l *localJobLog) rotate() error {
+ _, statErr := os.Stat(l.path + ".head")
+ if statErr != nil && !os.IsNotExist(statErr) {
+ return statErr
+ }
+ if os.IsNotExist(statErr) {
+ in, err := os.Open(l.path)
+ if err != nil {
+ return err
+ }
+ head := make([]byte, min(l.size, int64(8192)))
+ _, rerr := io.ReadFull(in, head)
+ _ = in.Close()
+ if rerr != nil {
+ return rerr
+ }
+ if err := os.WriteFile(l.path+".head", head, 0o600); err != nil {
+ return err
+ }
+ }
+ if err := l.file.Close(); err != nil {
+ return err
+ }
+ if err := os.Rename(l.path, l.path+".1"); err != nil {
+ return err
+ }
+ l.rotated += l.size
+ if err := writeLocalRotated(l.path+".rotated", l.rotated); err != nil {
+ return err
+ }
+ f, err := os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+ if err != nil {
+ return err
+ }
+ l.file, l.size = f, 0
+ return nil
+}
+
+func writeLocalRotated(path string, value int64) error {
+ f, err := os.CreateTemp(filepath.Dir(path), ".rotated-")
+ if err != nil {
+ return err
+ }
+ name := f.Name()
+ defer func() { _ = os.Remove(name) }()
+ if _, err := fmt.Fprint(f, strconv.FormatInt(value, 10)); err != nil {
+ _ = f.Close()
+ return err
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ return os.Rename(name, path)
+}
+
+func (l *localJobLog) Close() error { return l.file.Close() }
diff --git a/internal/application/service/local_shell_job_log_test.go b/internal/application/service/local_shell_job_log_test.go
new file mode 100644
index 000000000..0024d709e
--- /dev/null
+++ b/internal/application/service/local_shell_job_log_test.go
@@ -0,0 +1,46 @@
+package service
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestLocalJobLogRotatesKeepsHeadAndPrevious(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "stdout")
+ log, err := newLocalJobLog(path)
+ require.NoError(t, err)
+ _, err = log.Write(bytes.Repeat([]byte("a"), localJobSegmentBytes))
+ require.NoError(t, err)
+ _, err = log.Write(bytes.Repeat([]byte("b"), localJobSegmentBytes+7))
+ require.NoError(t, err)
+ require.NoError(t, log.Close())
+
+ head, err := os.ReadFile(path + ".head")
+ require.NoError(t, err)
+ require.Equal(t, bytes.Repeat([]byte("a"), 8192), head)
+ prev, err := os.ReadFile(path + ".1")
+ require.NoError(t, err)
+ require.Equal(t, bytes.Repeat([]byte("b"), localJobSegmentBytes), prev)
+ cur, err := os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, []byte("bbbbbbb"), cur)
+ rot, err := os.ReadFile(path + ".rotated")
+ require.NoError(t, err)
+ require.Equal(t, "33554432", string(rot))
+}
+
+func TestLocalJobLogAppendNoSparseHoles(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "stderr")
+ log, err := newLocalJobLog(path)
+ require.NoError(t, err)
+ _, err = log.Write(bytes.Repeat([]byte("x"), localJobSegmentBytes+3))
+ require.NoError(t, err)
+ require.NoError(t, log.Close())
+ cur, err := os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, []byte("xxx"), cur, "追加写 + 截断重开,无空洞")
+}
diff --git a/internal/application/service/local_shell_job_recovery.go b/internal/application/service/local_shell_job_recovery.go
new file mode 100644
index 000000000..1080b1acc
--- /dev/null
+++ b/internal/application/service/local_shell_job_recovery.go
@@ -0,0 +1,131 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// signalRecoveredProcess kills a leftover process group after an identity
+// check. Tests replace it. A failure must not stop recovery of other jobs:
+// the watcher that owned the job is already gone, and a failed identity
+// check is not permission to guess which process to signal.
+var signalRecoveredProcess = localRecoverProcess
+
+func localJobNotice(meta sandbox.ShellJobMeta, result interfaces.WorkResult,
+ finished time.Time,
+) interfaces.BackgroundNotice {
+ title := strings.Join(strings.Fields(meta.Title), " ")
+ if len(title) > 160 {
+ title = strings.ToValidUTF8(title[:160], "")
+ }
+ return interfaces.BackgroundNotice{
+ ID: meta.ID, Kind: "shell", FinishedAt: finished,
+ Text: fmt.Sprintf("Background shell %s %s: %s. Use work_wait(wait_sec=0) to read its output.",
+ meta.ID, result.State, title),
+ }
+}
+
+func readLocalJobJSON(path string, target any) error {
+ file, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = file.Close() }()
+ // The private job directory contains metadata written by this service.
+ // Legacy stdin wrappers and output baselines can exceed 64 KiB; recovery
+ // must accept the same metadata as the normal ReadShellJob path.
+ return json.NewDecoder(file).Decode(target)
+}
+
+// recoverJobs is the ONLY writer allowed besides the watcher: the previous
+// owner crashed. Terminal jobs within NoticeRetention re-push their notice
+// (the in-memory inbox was lost); older directories are deleted (D5). A
+// running job is identity-checked, killed, and recorded lost.
+func (s *LocalShellJobService) recoverJobs(ctx context.Context) error {
+ entries, err := os.ReadDir(s.root)
+ if err != nil {
+ return err
+ }
+ now := time.Now().UTC()
+ for _, entry := range entries {
+ if !entry.IsDir() || !sandbox.ValidShellJobID(entry.Name()) {
+ continue
+ }
+ if err := s.recoverJob(ctx, entry.Name(), now); err != nil {
+ logger.Warnf(ctx, "[LocalShellJob] skip recovery of %s: %v", entry.Name(), err)
+ }
+ }
+ return nil
+}
+
+func (s *LocalShellJobService) recoverJob(ctx context.Context, name string, now time.Time) error {
+ directory := filepath.Join(s.root, name)
+ var meta sandbox.ShellJobMeta
+ var status sandbox.ShellJobStatus
+ if err := readLocalJobJSON(filepath.Join(directory, "meta.json"), &meta); err != nil {
+ return err
+ }
+ if meta.ID != name || meta.TenantID == 0 || meta.SessionID == "" {
+ return fmt.Errorf("invalid local job metadata in %s", name)
+ }
+ if err := readLocalJobJSON(filepath.Join(directory, "status.json"), &status); err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ status = sandbox.ShellJobStatus{State: interfaces.WorkStateRunning, StartedAt: meta.CreatedAt}
+ }
+ if status.State == interfaces.WorkStateRunning {
+ if status.PGID > 0 && status.ProcStartedAt > 0 {
+ if kerr := signalRecoveredProcess(ctx, status.PGID, status.ProcStartedAt); kerr != nil {
+ logger.Warnf(ctx, "[LocalShellJob] could not signal %s pid=%d: %v", name, status.PGID, kerr)
+ }
+ }
+ ended := now
+ status.State, status.EndedAt = interfaces.WorkStateLost, &ended
+ if werr := sandbox.WriteShellJobJSON(filepath.Join(directory, "status.json"), status); werr != nil {
+ return werr
+ }
+ }
+ if !interfaces.IsTerminalWorkState(status.State) || status.EndedAt == nil {
+ return fmt.Errorf("invalid local terminal status in %s", name)
+ }
+ if status.EndedAt.Before(now.Add(-interfaces.NoticeRetention)) {
+ if rerr := os.RemoveAll(directory); rerr != nil {
+ return rerr
+ }
+ return nil
+ }
+ j := &localJob{
+ meta: meta, status: status,
+ done: make(chan struct{}), reported: make(chan struct{}),
+ }
+ close(j.done)
+ close(j.reported)
+ s.jobs[meta.ID] = j
+ result := ShellJobResult(sandbox.ShellJobSnapshot{
+ Meta: meta, Status: status, WatcherAlive: false,
+ }, true)
+ // Pending lists the oldest FinishedAt first. A recovered lost job uses
+ // its start time while that start is still inside retention.
+ finished := *status.EndedAt
+ if status.State == interfaces.WorkStateLost && !status.StartedAt.IsZero() &&
+ !status.StartedAt.Before(now.Add(-interfaces.NoticeRetention)) {
+ finished = status.StartedAt
+ }
+ if _, perr := s.bg.Inbox.Push(ctx, localRef(meta),
+ localJobNotice(meta, result, finished)); perr != nil {
+ return perr
+ }
+ return nil
+}
diff --git a/internal/application/service/local_shell_job_recovery_test.go b/internal/application/service/local_shell_job_recovery_test.go
new file mode 100644
index 000000000..9be13d976
--- /dev/null
+++ b/internal/application/service/local_shell_job_recovery_test.go
@@ -0,0 +1,126 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func writeLocalJob(t *testing.T, root, id string, meta sandbox.ShellJobMeta,
+ status sandbox.ShellJobStatus,
+) string {
+ t.Helper()
+ dir := filepath.Join(root, id)
+ require.NoError(t, os.MkdirAll(dir, 0o700))
+ require.NoError(t, sandbox.WriteShellJobJSON(filepath.Join(dir, "meta.json"), meta))
+ require.NoError(t, sandbox.WriteShellJobJSON(filepath.Join(dir, "status.json"), status))
+ return dir
+}
+
+func TestRecoveryMarksLostRepushesNoticePrunesOld(t *testing.T) {
+ root := t.TempDir()
+ recent := time.Now().UTC().Add(-time.Hour)
+ old := time.Now().UTC().Add(-25 * time.Hour)
+ meta := sandbox.ShellJobMeta{TenantID: 7, SessionID: "s", Title: "build"}
+ running := meta
+ running.ID = "sj_0123"
+ writeLocalJob(t, root, "sj_0123", running, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, PGID: 999999, ProcStartedAt: 1,
+ StartedAt: recent,
+ })
+ succeeded := meta
+ succeeded.ID = "sj_ab01"
+ ended := recent
+ writeLocalJob(t, root, "sj_ab01", succeeded, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateSucceeded, StartedAt: recent, EndedAt: &ended,
+ })
+ ancient := meta
+ ancient.ID = "sj_fedc"
+ writeLocalJob(t, root, "sj_fedc", ancient, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateFailed, StartedAt: old, EndedAt: &old,
+ })
+
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := &LocalShellJobService{
+ bg: bg, root: root, jobs: make(map[string]*localJob),
+ cursors: make(map[string]localCursor), blocked: make(map[interfaces.SessionRef]bool),
+ }
+ require.NoError(t, svc.recoverJobs(context.Background()))
+
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 2, "running→lost 与 24h 内的 succeeded 各补推一条;25h 前的直接清理")
+ require.Contains(t, notices[0].Text, "lost")
+ require.True(t, interfaces.IsTerminalWorkState(svc.jobs["sj_0123"].status.State))
+ _, err = os.Stat(filepath.Join(root, "sj_fedc"))
+ require.True(t, os.IsNotExist(err), "ended_at 超过 24h 的终态目录被删除(D5)")
+}
+
+func TestRecoverySkipsCorruptMetaAndContinues(t *testing.T) {
+ root := t.TempDir()
+ recent := time.Now().UTC().Add(-time.Hour)
+ meta := sandbox.ShellJobMeta{ID: "sj_ab01", TenantID: 7, SessionID: "s", Title: "build"}
+ ended := recent
+ writeLocalJob(t, root, "sj_ab01", meta, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateSucceeded, StartedAt: recent, EndedAt: &ended,
+ })
+ bad := filepath.Join(root, "sj_bad1")
+ require.NoError(t, os.MkdirAll(bad, 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(bad, "meta.json"), []byte("{"), 0o600))
+
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := &LocalShellJobService{
+ bg: bg, root: root, jobs: make(map[string]*localJob),
+ }
+ require.NoError(t, svc.recoverJobs(context.Background()))
+ require.Contains(t, svc.jobs, "sj_ab01")
+ require.NotContains(t, svc.jobs, "sj_bad1")
+ notices, err := bg.Inbox.Pending(context.Background(), interfaces.SessionRef{TenantID: 7, SessionID: "s"})
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+}
+
+func TestRecoveryContinuesWhenProcessSignalFails(t *testing.T) {
+ root := t.TempDir()
+ recent := time.Now().UTC()
+ meta := sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", Title: "build"}
+ writeLocalJob(t, root, "sj_0123", meta, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, PGID: 4321, ProcStartedAt: recent.UnixNano(),
+ StartedAt: recent,
+ })
+ orig := signalRecoveredProcess
+ signalRecoveredProcess = func(context.Context, int, int64) error {
+ return errors.New("process table busy")
+ }
+ t.Cleanup(func() { signalRecoveredProcess = orig })
+
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := &LocalShellJobService{
+ bg: bg, root: root, jobs: make(map[string]*localJob),
+ }
+ require.NoError(t, svc.recoverJobs(context.Background()))
+ require.Equal(t, interfaces.WorkStateLost, svc.jobs["sj_0123"].status.State)
+}
+
+func TestRecoveryDoesNotTouchRunningWithoutPGID(t *testing.T) {
+ root := t.TempDir()
+ meta := sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s"}
+ writeLocalJob(t, root, "sj_0123", meta, sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, StartedAt: time.Now().UTC(),
+ })
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := &LocalShellJobService{bg: bg, root: root, jobs: make(map[string]*localJob)}
+ require.NoError(t, svc.recoverJobs(context.Background()))
+ require.Equal(t, interfaces.WorkStateLost, svc.jobs["sj_0123"].status.State)
+}
diff --git a/internal/application/service/local_shell_job_service.go b/internal/application/service/local_shell_job_service.go
new file mode 100644
index 000000000..d5604cc3f
--- /dev/null
+++ b/internal/application/service/local_shell_job_service.go
@@ -0,0 +1,970 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+ "unicode/utf8"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/event"
+ "github.com/Tencent/WeKnora/internal/localsandbox"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+const (
+ // localShellJobLease is the Lite registration lease. Remote jobs renew for
+ // 30s; the in-process watcher heartbeats this shorter lease itself.
+ localShellJobLease = 15 * time.Second
+ localShellJobHeartbeat = 5 * time.Second
+ localShellJobStopWait = 5 * time.Second
+ localPreviewBytes = 8 * 1024
+ localPreviewEvery = 500 * time.Millisecond
+)
+
+type localJobStarter interface {
+ StartForSession(context.Context, localsandbox.RunRequest) (localsandbox.Process, error)
+ GuardForSession(context.Context, string) (*localsandbox.PathGuard, localsandbox.Workspace, error)
+}
+
+type localJob struct {
+ meta sandbox.ShellJobMeta
+ proc localsandbox.Process
+ done chan struct{} // closed by watcher after terminal status is persisted
+ decision chan bool // submitter's single verdict: backgrounded?
+ release chan struct{} // closed after work_started was attempted
+ reported chan struct{} // closed after the watcher's report path finished
+ requests chan string // cancel / lost wakeups (non-blocking send)
+ reportMu sync.Mutex // guards suppress vs the watcher's report section
+ mu sync.Mutex
+ status sandbox.ShellJobStatus
+ background bool
+ cause string
+ preview string
+ dirty bool
+ suppress bool
+ originRun string
+ lastPreview time.Time
+ killOnce sync.Once
+}
+
+type localCursor struct {
+ from sandbox.ShellJobOffsets
+ expires time.Time
+}
+
+// LocalShellJobService is Lite's watcher runtime. It implements the plan-4
+// LocalShellJobs contract; ShellJobService delegates host sessions here.
+type LocalShellJobService struct {
+ starter localJobStarter
+ bg *BackgroundWork
+ events *BackgroundEventEmitter
+ lifecycle background.Lifecycle
+ root string
+ appCtx context.Context
+ cancel context.CancelFunc
+ identity func(context.Context, int) (int64, error)
+ submitMu sync.Mutex
+ mu sync.Mutex
+ closing bool
+ blocked map[interfaces.SessionRef]bool
+ jobs map[string]*localJob
+ cursors map[string]localCursor
+ skillCapture skillEnvCaptures
+}
+
+var _ LocalShellJobs = (*LocalShellJobService)(nil)
+
+// HoldSkillEnvCapture keeps a credential write-back until the host watcher
+// sees this job succeed. Secrets stay in this process only.
+func (s *LocalShellJobService) HoldSkillEnvCapture(
+ ref interfaces.SessionRef, jobID string, run func(context.Context),
+) {
+ if s == nil || run == nil || jobID == "" {
+ return
+ }
+ s.mu.Lock()
+ j := s.jobs[jobID]
+ if s.closing || s.blocked[ref] || j == nil || localRef(j.meta) != ref {
+ s.mu.Unlock()
+ return
+ }
+ s.mu.Unlock()
+ // Serialize registration with reporting and suppression. The watcher may
+ // have reported before SubmitShellJob returned its background handle.
+ j.reportMu.Lock()
+ defer j.reportMu.Unlock()
+ s.mu.Lock()
+ blocked := s.closing || s.blocked[ref]
+ s.mu.Unlock()
+ if blocked || j.suppress {
+ return
+ }
+ s.skillCapture.put(ref, jobID, run)
+ j.mu.Lock()
+ state := j.status.State
+ j.mu.Unlock()
+ if interfaces.IsTerminalWorkState(state) {
+ s.skillCapture.settle(context.Background(), ref, jobID, state)
+ }
+}
+
+// NewLocalShellJobService starts the host watcher runtime. root is the absolute
+// jobs directory. The process-identity function is supplied by crash recovery.
+func NewLocalShellJobService(
+ local *localsandbox.Service, bg *BackgroundWork, events *BackgroundEventEmitter, root string,
+) (*LocalShellJobService, error) {
+ if local == nil {
+ return nil, errors.New("local shell is unavailable")
+ }
+ return newLocalShellJobService(local, bg, events, root, localProcessStartedAt)
+}
+
+func newLocalShellJobService(
+ starter localJobStarter, bg *BackgroundWork, events *BackgroundEventEmitter, root string,
+ identity func(context.Context, int) (int64, error),
+) (*LocalShellJobService, error) {
+ if bg == nil || bg.Registry == nil || bg.Inbox == nil {
+ return nil, errors.New("background stores unavailable")
+ }
+ if !filepath.IsAbs(root) {
+ return nil, errors.New("jobs root must be absolute")
+ }
+ if err := os.MkdirAll(root, 0o700); err != nil {
+ return nil, err
+ }
+ if err := os.Chmod(root, 0o700); err != nil {
+ return nil, err
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ svc := &LocalShellJobService{
+ starter: starter, bg: bg, events: events, root: root,
+ appCtx: ctx, cancel: cancel, identity: identity,
+ blocked: make(map[interfaces.SessionRef]bool),
+ jobs: make(map[string]*localJob),
+ cursors: make(map[string]localCursor),
+ lifecycle: background.Lifecycle{
+ Registry: bg.Registry, Inbox: bg.Inbox, Logf: logger.Warnf,
+ },
+ }
+ if events != nil {
+ svc.lifecycle.Events = events
+ }
+ if err := svc.recoverJobs(ctx); err != nil {
+ cancel()
+ return nil, err
+ }
+ return svc, nil
+}
+
+func localRef(meta sandbox.ShellJobMeta) interfaces.SessionRef {
+ return interfaces.SessionRef{TenantID: meta.TenantID, SessionID: meta.SessionID}
+}
+
+// SubmitShellJob starts one host command and waits out the yield window.
+// The watcher and the child use the application context: cancelling ctx only
+// ends this wait.
+func (s *LocalShellJobService) SubmitShellJob(
+ ctx context.Context, ref interfaces.SessionRef, req tools.ShellJobSubmitRequest,
+) (tools.ShellJobSubmission, error) {
+ s.submitMu.Lock()
+ unlockSubmit := true
+ defer func() {
+ if unlockSubmit {
+ s.submitMu.Unlock()
+ }
+ }()
+ s.mu.Lock()
+ unavailable := s.closing || s.blocked[ref]
+ s.mu.Unlock()
+ if unavailable {
+ return tools.ShellJobSubmission{}, errors.New("session is closing or deleted")
+ }
+ if ref.TenantID == 0 || ref.SessionID == "" {
+ return tools.ShellJobSubmission{}, errors.New("session owner is required")
+ }
+ id, err := allocateShellJobID(func(candidate string) bool {
+ if _, statErr := os.Stat(filepath.Join(s.root, candidate)); statErr == nil {
+ return true
+ }
+ s.mu.Lock()
+ _, live := s.jobs[candidate]
+ s.mu.Unlock()
+ return live
+ })
+ if err != nil {
+ return tools.ShellJobSubmission{}, err
+ }
+ requestCtx := types.WithSandboxTenantID(
+ types.WithExecutionTenant(context.WithoutCancel(ctx), ref.TenantID), ref.TenantID)
+ _, ws, err := s.starter.GuardForSession(requestCtx, ref.SessionID)
+ if err != nil {
+ return tools.ShellJobSubmission{}, err
+ }
+ now := time.Now().UTC()
+ j := &localJob{
+ meta: sandbox.ShellJobMeta{
+ ID: id, TenantID: ref.TenantID, SandboxTenantID: ref.TenantID,
+ SessionID: ref.SessionID, Root: ws.Root,
+ Command: req.MaskedCommand(), WorkDir: req.WorkDir,
+ CreatedAt: now, YieldUntil: now.Add(req.YieldTime),
+ ParentMessageID: req.ParentMessageID, ToolCallID: req.ToolCallID,
+ Title: req.Description,
+ OutputDir: req.OutputDir, OutputSeen: req.OutputSeen, OutputBefore: req.OutputBefore,
+ },
+ done: make(chan struct{}), decision: make(chan bool, 1),
+ release: make(chan struct{}),
+ reported: make(chan struct{}),
+ requests: make(chan string, 1),
+ originRun: req.RunID,
+ }
+ if j.meta.WorkDir == "" {
+ j.meta.WorkDir = ws.Root
+ }
+ directory := filepath.Join(s.root, id)
+ var out, errLog *localJobLog
+ activity := interfaces.BackgroundActivity{
+ ID: id, Kind: "shell", Title: req.Description, StartedAt: now,
+ }
+ err = s.lifecycle.Begin(requestCtx, ref, activity, localShellJobLease, shellJobKindCap,
+ func(context.Context) error {
+ if err := os.Mkdir(directory, 0o700); err != nil {
+ return err
+ }
+ if err := sandbox.WriteShellJobJSON(filepath.Join(directory, "meta.json"), j.meta); err != nil {
+ return err
+ }
+ var lerr error
+ out, lerr = newLocalJobLog(filepath.Join(directory, "stdout"))
+ if lerr != nil {
+ return lerr
+ }
+ errLog, lerr = newLocalJobLog(filepath.Join(directory, "stderr"))
+ if lerr != nil {
+ _ = out.Close()
+ return lerr
+ }
+ // Application ctx (plus the session tenant): a /stop run ctx never reaches Spawn.
+ processCtx := types.WithSandboxTenantID(
+ types.WithExecutionTenant(s.appCtx, ref.TenantID), ref.TenantID)
+ j.proc, lerr = s.starter.StartForSession(processCtx, localsandbox.RunRequest{
+ SessionID: ref.SessionID, Command: req.Command,
+ WorkDir: req.WorkDir, Env: req.Env,
+ })
+ if lerr != nil {
+ _ = out.Close()
+ _ = errLog.Close()
+ return lerr
+ }
+ startedAt, serr := s.identity(processCtx, j.proc.PID())
+ if errors.Is(serr, localsandbox.ErrUnsupportedPlatform) {
+ startedAt = 0
+ serr = nil
+ }
+ if serr != nil {
+ _ = j.proc.Kill()
+ localDrainAndReap(j.proc)
+ _ = out.Close()
+ _ = errLog.Close()
+ return fmt.Errorf("record job process identity: %w", serr)
+ }
+ j.status = sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, WatcherPID: os.Getpid(),
+ PGID: j.proc.PID(), ProcStartedAt: startedAt, StartedAt: now,
+ }
+ if serr := sandbox.WriteShellJobJSON(filepath.Join(directory, "status.json"), j.status); serr != nil {
+ _ = j.proc.Kill()
+ localDrainAndReap(j.proc)
+ _ = out.Close()
+ _ = errLog.Close()
+ return serr
+ }
+ s.mu.Lock()
+ s.jobs[id] = j
+ s.mu.Unlock()
+ go s.watch(j, out, errLog, req.OnOutput)
+ return nil
+ })
+ unlockSubmit = false
+ s.submitMu.Unlock()
+ if err != nil {
+ // The watcher is started only after the directory is published. A failed
+ // start must not leave a running-looking directory for crash recovery.
+ _ = os.RemoveAll(directory)
+ if errors.Is(err, interfaces.ErrActivityLimit) {
+ return tools.ShellJobSubmission{}, errors.New(
+ "ERR_WORK_LIMIT: this session already runs 64 background shell jobs")
+ }
+ return tools.ShellJobSubmission{}, err
+ }
+ timer := time.NewTimer(max(req.YieldTime, time.Nanosecond))
+ defer timer.Stop()
+ var stop <-chan struct{}
+ if ctx != nil {
+ stop = ctx.Done()
+ }
+ // /stop ends the wait and does not kill the command. A command that
+ // finished in the same instant still returns inline: done wins over the
+ // timer and the cancel when more than one is ready.
+ backgrounded := backgroundedAfterYield(j.done, stop, timer.C)
+ j.mu.Lock()
+ j.background = backgrounded
+ j.mu.Unlock()
+ j.decision <- backgrounded
+ if backgrounded && s.events != nil {
+ _ = s.events.EmitWorkStarted(context.WithoutCancel(ctx), ref,
+ req.ParentMessageID, activity, req.ToolCallID)
+ }
+ close(j.release)
+ if !backgrounded {
+ if derr := s.lifecycle.Discard(context.WithoutCancel(ctx), ref, id); derr != nil {
+ logger.Warnf(ctx, "[LocalShellJob] discard id=%s: %v", id, derr)
+ }
+ }
+ result, serr := s.Status(ctx, ref, []string{id}, interfaces.StatusOptions{
+ Detailed: true, RunID: req.RunID,
+ })
+ if serr != nil {
+ return tools.ShellJobSubmission{}, serr
+ }
+ return tools.ShellJobSubmission{Result: result[0], Background: backgrounded}, nil
+}
+
+// backgroundedAfterYield reports whether the yield wait ended while the
+// command was still running. done is closed when the watcher has persisted a
+// terminal status. stop is the submitter's context. When done and another
+// channel are both ready, select would choose at random and a finished
+// command would also be reported as background work.
+func backgroundedAfterYield(done, stop <-chan struct{}, timer <-chan time.Time) bool {
+ select {
+ case <-done:
+ return false
+ case <-timer:
+ case <-stop:
+ }
+ select {
+ case <-done:
+ return false
+ default:
+ return true
+ }
+}
+
+func localDrainAndReap(proc localsandbox.Process) {
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ _, _ = io.Copy(io.Discard, proc.Stdout())
+ }()
+ go func() {
+ defer wg.Done()
+ _, _ = io.Copy(io.Discard, proc.Stderr())
+ }()
+ wg.Wait()
+ _, _ = proc.Wait(context.Background())
+}
+
+// requestStop records why the watcher should kill the child. lost wins over
+// cancelled so a shutdown cannot be rewritten as a user cancel.
+func (j *localJob) requestStop(cause string) {
+ j.mu.Lock()
+ if j.cause == "lost" || (j.cause != "" && cause != "lost") {
+ j.mu.Unlock()
+ return
+ }
+ j.cause = cause
+ j.mu.Unlock()
+ select {
+ case j.requests <- cause:
+ default:
+ }
+}
+
+func (j *localJob) killProc() {
+ j.killOnce.Do(func() {
+ if j.proc != nil {
+ _ = j.proc.Kill()
+ }
+ })
+}
+
+func (s *LocalShellJobService) watch(j *localJob, out, errLog *localJobLog, onOutput func(string, []byte)) {
+ defer close(j.reported)
+ defer func() { _ = out.Close() }()
+ defer func() { _ = errLog.Close() }()
+
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ s.copyStream(j, j.proc.Stdout(), out, "stdout", onOutput)
+ }()
+ go func() {
+ defer wg.Done()
+ s.copyStream(j, j.proc.Stderr(), errLog, "stderr", onOutput)
+ }()
+ copiesDone := make(chan struct{})
+ go func() {
+ wg.Wait()
+ close(copiesDone)
+ }()
+ exitCh := make(chan localsandbox.ExitStatus, 1)
+ go func(drained <-chan struct{}) {
+ // exec.Cmd.Wait closes StdoutPipe/StderrPipe. Drain both first, even
+ // when forwarding output is slower than the command. The select below
+ // continues to handle cancellation and renew the lease while draining.
+ <-drained
+ // Wait's ctx is the application ctx. Cancelling it kills a real process
+ // tree; the submitting run's ctx is never passed here.
+ st, _ := j.proc.Wait(s.appCtx)
+ exitCh <- st
+ }(copiesDone)
+
+ var exit localsandbox.ExitStatus
+ copiesClosed := false
+ waitClosed := false
+ appDone := s.appCtx.Done()
+ ticker := time.NewTicker(localShellJobHeartbeat)
+ defer ticker.Stop()
+ for !copiesClosed || !waitClosed {
+ select {
+ case <-copiesDone:
+ copiesClosed = true
+ copiesDone = nil
+ case st := <-exitCh:
+ exit = st
+ waitClosed = true
+ exitCh = nil
+ case <-j.requests:
+ j.killProc()
+ case <-appDone:
+ appDone = nil
+ j.requestStop("lost")
+ j.killProc()
+ case <-ticker.C:
+ s.heartbeat(j)
+ }
+ }
+ s.writeTerminal(j, exit)
+ close(j.done)
+ backgrounded := <-j.decision
+ <-j.release
+ if !backgrounded {
+ return
+ }
+ s.flushPreview(j, true)
+ s.report(j)
+}
+
+func (s *LocalShellJobService) copyStream(
+ j *localJob, r io.Reader, log *localJobLog, stream string, onOutput func(string, []byte),
+) {
+ if r == nil {
+ return
+ }
+ buf := make([]byte, 32*1024)
+ for {
+ n, err := r.Read(buf)
+ if n > 0 {
+ if _, werr := log.Write(buf[:n]); werr != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] write %s id=%s: %v", stream, j.meta.ID, werr)
+ }
+ body := append([]byte(nil), buf[:n]...)
+ s.emitPreview(j, stream, body, onOutput)
+ }
+ if err != nil {
+ return
+ }
+ }
+}
+
+// emitPreview forwards yield-window bytes to the submitter and, once the job
+// has been backgrounded, publishes a throttled 8 KiB tail.
+func (s *LocalShellJobService) emitPreview(
+ j *localJob, stream string, chunk []byte, onOutput func(string, []byte),
+) {
+ if len(chunk) == 0 {
+ return
+ }
+ j.mu.Lock()
+ j.preview = appendPreview(j.preview, string(chunk), localPreviewBytes)
+ backgrounded := j.background
+ due := j.lastPreview.IsZero() || time.Since(j.lastPreview) >= localPreviewEvery
+ output := j.preview
+ if backgrounded && due {
+ j.lastPreview = time.Now()
+ j.dirty = false
+ } else if backgrounded {
+ j.dirty = true
+ }
+ data := event.CommandOutputData{
+ ToolCallID: j.meta.ToolCallID, Command: j.meta.Command,
+ StartedAt: j.status.StartedAt, Output: output,
+ }
+ ref := localRef(j.meta)
+ parent := j.meta.ParentMessageID
+ id := j.meta.ID
+ j.mu.Unlock()
+ if !backgrounded {
+ if onOutput != nil {
+ onOutput(stream, chunk)
+ }
+ return
+ }
+ if !due || s.events == nil {
+ return
+ }
+ if err := s.events.EmitCommandOutput(s.appCtx, ref, parent, id, data); err != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] preview id=%s: %v", id, err)
+ }
+}
+
+func (s *LocalShellJobService) flushPreview(j *localJob, done bool) {
+ if s.events == nil {
+ return
+ }
+ j.mu.Lock()
+ if !j.dirty && !done {
+ j.mu.Unlock()
+ return
+ }
+ j.dirty = false
+ j.lastPreview = time.Now()
+ data := event.CommandOutputData{
+ ToolCallID: j.meta.ToolCallID, Command: j.meta.Command,
+ StartedAt: j.status.StartedAt, Output: j.preview, Done: done,
+ }
+ ref := localRef(j.meta)
+ parent := j.meta.ParentMessageID
+ id := j.meta.ID
+ j.mu.Unlock()
+ if err := s.events.EmitCommandOutput(s.appCtx, ref, parent, id, data); err != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] preview id=%s: %v", id, err)
+ }
+}
+
+func appendPreview(existing, chunk string, limit int) string {
+ next := existing + chunk
+ if len(next) <= limit {
+ return next
+ }
+ next = next[len(next)-limit:]
+ for len(next) > 0 && !utf8.RuneStart(next[0]) {
+ next = next[1:]
+ }
+ return strings.ToValidUTF8(next, "")
+}
+
+func (s *LocalShellJobService) heartbeat(j *localJob) {
+ if s.appCtx.Err() != nil || jobTerminal(j) {
+ return
+ }
+ ref := localRef(j.meta)
+ found, err := s.lifecycle.Alive(s.appCtx, ref, j.meta.ID, localShellJobLease)
+ if err != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] renew id=%s: %v", j.meta.ID, err)
+ return
+ }
+ if found {
+ return
+ }
+ j.mu.Lock()
+ started := j.status.StartedAt
+ j.mu.Unlock()
+ if started.IsZero() {
+ started = j.meta.CreatedAt
+ }
+ activity := interfaces.BackgroundActivity{
+ ID: j.meta.ID, Kind: "shell", Title: j.meta.Title, StartedAt: started,
+ }
+ if err := s.bg.Registry.Add(s.appCtx, ref, activity, localShellJobLease, 0); err != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] re-add id=%s: %v", j.meta.ID, err)
+ }
+}
+
+func jobTerminal(j *localJob) bool {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ return interfaces.IsTerminalWorkState(j.status.State)
+}
+
+func (s *LocalShellJobService) writeTerminal(j *localJob, exit localsandbox.ExitStatus) {
+ j.mu.Lock()
+ status := j.status
+ cause := j.cause
+ j.mu.Unlock()
+ ended := time.Now().UTC()
+ status.EndedAt = &ended
+ switch {
+ case cause == "lost":
+ status.State = interfaces.WorkStateLost
+ status.ExitCode = nil
+ case cause == "cancelled" || exit.Killed:
+ status.State = interfaces.WorkStateCancelled
+ code := exit.Code
+ if code == 0 {
+ code = 143
+ }
+ status.ExitCode = &code
+ case exit.Code == 0:
+ status.State = interfaces.WorkStateSucceeded
+ code := 0
+ status.ExitCode = &code
+ default:
+ status.State = interfaces.WorkStateFailed
+ code := exit.Code
+ status.ExitCode = &code
+ }
+ path := filepath.Join(s.root, j.meta.ID, "status.json")
+ if err := sandbox.WriteShellJobJSON(path, status); err != nil {
+ logger.Warnf(s.appCtx, "[LocalShellJob] status id=%s: %v", j.meta.ID, err)
+ }
+ j.mu.Lock()
+ j.status = status
+ j.mu.Unlock()
+}
+
+func (s *LocalShellJobService) report(j *localJob) {
+ j.reportMu.Lock()
+ defer j.reportMu.Unlock()
+ if j.suppress {
+ return
+ }
+ snap := s.snapshot(j)
+ result := applyHostOutputFiles(ShellJobResult(snap, true), snap.Meta)
+ if result.ID == "" {
+ result.ID = j.meta.ID
+ }
+ s.skillCapture.settle(context.Background(), localRef(j.meta), j.meta.ID, result.State)
+ notice := shellJobNotice(j.meta.ID, snap, result.State, result.OutputFiles)
+ ctx := context.WithoutCancel(s.appCtx)
+ if err := s.lifecycle.Finish(ctx, localRef(j.meta), notice, result, j.meta.ParentMessageID); err != nil {
+ logger.Warnf(ctx, "[LocalShellJob] finish id=%s: %v", j.meta.ID, err)
+ }
+}
+
+func (s *LocalShellJobService) snapshot(j *localJob) sandbox.ShellJobSnapshot {
+ snap, err := sandbox.ReadShellJob(s.root, sandbox.ShellJobRead{ID: j.meta.ID})
+ if err != nil || snap.Meta.ID == "" {
+ snap.Meta = j.meta
+ }
+ if err != nil || snap.Status.State == "" {
+ j.mu.Lock()
+ snap.Status = j.status
+ j.mu.Unlock()
+ }
+ return snap
+}
+
+// Status returns one result per id. An empty RunID reads head plus tail
+// and does not move the submitting run's cursor.
+func (s *LocalShellJobService) Status(
+ _ context.Context, ref interfaces.SessionRef, ids []string, opts interfaces.StatusOptions,
+) ([]interfaces.WorkResult, error) {
+ if err := validateShellJobIDs(ids); err != nil {
+ return nil, err
+ }
+ out := make([]interfaces.WorkResult, len(ids))
+ for i, id := range ids {
+ j, err := s.ownedJob(ref, id)
+ if err != nil {
+ out[i] = lostResult(sandbox.ShellJobMeta{ID: id}, "job not found")
+ continue
+ }
+ if !opts.Detailed {
+ out[i] = s.memoryResult(j)
+ continue
+ }
+ from := s.cursorFrom(ref, opts.RunID, id)
+ snap, err := sandbox.ReadShellJob(s.root, sandbox.ShellJobRead{
+ ID: id, From: from, OutputBudget: opts.OutputBudget,
+ })
+ if err != nil || snap.Missing || belongsToOtherSession(snap.Meta, ref) {
+ out[i] = lostResult(sandbox.ShellJobMeta{ID: id}, "job unreadable")
+ continue
+ }
+ if snap.Meta.ID == "" {
+ snap.Meta = j.meta
+ }
+ result := applyHostOutputFiles(ShellJobResult(snap, true), snap.Meta)
+ if result.ID == "" {
+ result.ID = id
+ }
+ s.noteObservedCursor(ref, opts.RunID, j.originRun, id, snap)
+ out[i] = result
+ }
+ return out, nil
+}
+
+func (s *LocalShellJobService) memoryResult(j *localJob) interfaces.WorkResult {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ state := j.status.State
+ if state == "" {
+ state = interfaces.WorkStateRunning
+ }
+ return interfaces.WorkResult{
+ ID: j.meta.ID, State: state, Title: j.meta.Title,
+ Terminal: interfaces.IsTerminalWorkState(state),
+ }
+}
+
+func (s *LocalShellJobService) ownedJob(ref interfaces.SessionRef, id string) (*localJob, error) {
+ s.mu.Lock()
+ j := s.jobs[id]
+ s.mu.Unlock()
+ if j == nil || j.meta.TenantID != ref.TenantID || j.meta.SessionID != ref.SessionID {
+ return nil, interfaces.ErrWorkNotFound
+ }
+ return j, nil
+}
+
+// Cancel asks the watcher to kill each id. The returned state is whatever was
+// recorded at this moment; the terminal state lands in the notice.
+func (s *LocalShellJobService) Cancel(
+ _ context.Context, ref interfaces.SessionRef, ids []string,
+) ([]interfaces.WorkResult, error) {
+ if err := validateShellJobIDs(ids); err != nil {
+ return nil, err
+ }
+ out := make([]interfaces.WorkResult, len(ids))
+ for i, id := range ids {
+ j, err := s.ownedJob(ref, id)
+ if err != nil {
+ out[i] = lostResult(sandbox.ShellJobMeta{ID: id}, "job not found")
+ continue
+ }
+ j.requestStop("cancelled")
+ out[i] = s.memoryResult(j)
+ }
+ return out, nil
+}
+
+// OnDeleteAborted reopens a session whose repository deletion failed.
+func (s *LocalShellJobService) OnDeleteAborted(_ context.Context, ref interfaces.SessionRef) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ delete(s.blocked, ref)
+ return nil
+}
+
+// OnDelete refuses later submits for the session, suppresses a late notice,
+// and cancels every live job. The shared listener clears the inbox afterwards.
+func (s *LocalShellJobService) OnDelete(ctx context.Context, ref interfaces.SessionRef) error {
+ s.submitMu.Lock()
+ defer s.submitMu.Unlock()
+ s.mu.Lock()
+ s.blocked[ref] = true
+ s.skillCapture.dropSession(ref)
+ jobs := s.sessionJobsLocked(ref)
+ s.dropSessionCursorsLocked(ref)
+ s.mu.Unlock()
+ for _, j := range jobs {
+ j.reportMu.Lock()
+ j.suppress = true
+ j.reportMu.Unlock()
+ j.requestStop("cancelled")
+ }
+ return s.waitReported(ctx, jobs)
+}
+
+// Shutdown kills every live child and records lost. A second call is a no-op.
+// The wait is bounded by ctx, or by 5s when ctx has no deadline.
+func (s *LocalShellJobService) Shutdown(ctx context.Context) error {
+ s.submitMu.Lock()
+ defer s.submitMu.Unlock()
+ s.mu.Lock()
+ if s.closing {
+ s.mu.Unlock()
+ return nil
+ }
+ s.closing = true
+ jobs := make([]*localJob, 0, len(s.jobs))
+ for _, j := range s.jobs {
+ jobs = append(jobs, j)
+ }
+ s.mu.Unlock()
+ s.cancel()
+ for _, j := range jobs {
+ j.reportMu.Lock()
+ j.suppress = true
+ j.reportMu.Unlock()
+ j.requestStop("lost")
+ }
+ return s.waitReported(ctx, jobs)
+}
+
+// EndShellJobRun keeps the run's output cursors for one hour after the run
+// ends, rather than for one hour after the last read.
+func (s *LocalShellJobService) EndShellJobRun(ref interfaces.SessionRef, runID string) {
+ if s == nil || runID == "" {
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ prefix := fmt.Sprintf("%d\x00%s\x00%s\x00", ref.TenantID, ref.SessionID, runID)
+ exp := time.Now().Add(time.Hour)
+ for key, cursor := range s.cursors {
+ if strings.HasPrefix(key, prefix) {
+ cursor.expires = exp
+ s.cursors[key] = cursor
+ }
+ }
+}
+
+func (s *LocalShellJobService) sessionJobsLocked(ref interfaces.SessionRef) []*localJob {
+ out := make([]*localJob, 0)
+ for _, j := range s.jobs {
+ if j.meta.TenantID == ref.TenantID && j.meta.SessionID == ref.SessionID {
+ out = append(out, j)
+ }
+ }
+ return out
+}
+
+func (s *LocalShellJobService) dropSessionCursorsLocked(ref interfaces.SessionRef) {
+ prefix := fmt.Sprintf("%d\x00%s\x00", ref.TenantID, ref.SessionID)
+ for key := range s.cursors {
+ if strings.HasPrefix(key, prefix) {
+ delete(s.cursors, key)
+ }
+ }
+}
+
+func (s *LocalShellJobService) waitReported(ctx context.Context, jobs []*localJob) error {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ if _, ok := ctx.Deadline(); !ok {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, localShellJobStopWait)
+ defer cancel()
+ }
+ for _, j := range jobs {
+ if j == nil || j.reported == nil {
+ continue
+ }
+ select {
+ case <-j.reported:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ return nil
+}
+
+func localCursorKey(ref interfaces.SessionRef, runID, workID string) string {
+ return fmt.Sprintf("%d\x00%s\x00%s\x00%s", ref.TenantID, ref.SessionID, runID, workID)
+}
+
+func (s *LocalShellJobService) cursorFrom(
+ ref interfaces.SessionRef, runID, id string,
+) *sandbox.ShellJobOffsets {
+ if s == nil || runID == "" || id == "" {
+ return nil
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.pruneCursorsLocked(time.Now())
+ cursor, ok := s.cursors[localCursorKey(ref, runID, id)]
+ if !ok {
+ return nil
+ }
+ from := cursor.from
+ return &from
+}
+
+// noteObservedCursor moves a run cursor to the end of the bytes just observed.
+// An empty RunID is an out-of-run read (HTTP detail, chip "view output") and
+// must not move any run's cursor.
+func (s *LocalShellJobService) noteObservedCursor(
+ ref interfaces.SessionRef, runID, originRun, id string, snap sandbox.ShellJobSnapshot,
+) {
+ if runID == "" || id == "" || snap.Missing {
+ return
+ }
+ _ = originRun
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.pruneCursorsLocked(time.Now())
+ s.cursors[localCursorKey(ref, runID, id)] = localCursor{
+ from: sandbox.ShellJobOffsets{
+ Stdout: nextStreamOffset(snap.Stdout),
+ Stderr: nextStreamOffset(snap.Stderr),
+ },
+ expires: time.Now().Add(time.Hour),
+ }
+}
+
+func (s *LocalShellJobService) pruneCursorsLocked(now time.Time) {
+ for key, cursor := range s.cursors {
+ if !cursor.expires.After(now) {
+ delete(s.cursors, key)
+ }
+ }
+}
+
+func applyHostOutputFiles(result interfaces.WorkResult, meta sandbox.ShellJobMeta) interfaces.WorkResult {
+ if !result.Terminal || !meta.OutputSeen || meta.OutputDir == "" {
+ return result
+ }
+ after, err := listHostOutputFiles(meta.OutputDir)
+ if err != nil {
+ return result
+ }
+ links := tools.ChangedOutputFileLinks(meta.OutputDir, sandbox.EntriesFromStamps(meta.OutputBefore), after)
+ result.OutputFiles = links
+ if payload, ok := result.Payload.(ShellJobPayload); ok {
+ payload.OutputFiles = links
+ result.Payload = payload
+ }
+ return result
+}
+
+func listHostOutputFiles(dir string) ([]sandbox.RemoteDirEntry, error) {
+ dir = strings.TrimSpace(dir)
+ if dir == "" {
+ return nil, errors.New("output dir is empty")
+ }
+ var out []sandbox.RemoteDirEntry
+ err := filepath.WalkDir(dir, func(p string, d os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ if p == dir && os.IsNotExist(walkErr) {
+ return nil
+ }
+ return walkErr
+ }
+ if d.IsDir() {
+ return nil
+ }
+ if len(out) > 200 {
+ return errors.New("output dir listing is too large")
+ }
+ info, infoErr := d.Info()
+ if infoErr != nil {
+ return infoErr
+ }
+ out = append(out, sandbox.RemoteDirEntry{
+ Name: d.Name(), Path: p, Type: sandbox.RemoteEntryFile,
+ Size: info.Size(), ModTime: info.ModTime(),
+ })
+ return nil
+ })
+ if err != nil && !os.IsNotExist(err) {
+ return nil, err
+ }
+ return out, nil
+}
diff --git a/internal/application/service/local_shell_job_service_test.go b/internal/application/service/local_shell_job_service_test.go
new file mode 100644
index 000000000..35b1ac64b
--- /dev/null
+++ b/internal/application/service/local_shell_job_service_test.go
@@ -0,0 +1,362 @@
+//go:build desktop
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/localsandbox"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type localTestProcess struct {
+ outR, errR *io.PipeReader
+ outW, errW *io.PipeWriter
+ once sync.Once
+ exit localsandbox.ExitStatus
+ killCount int
+}
+
+func newLocalTestProcess() *localTestProcess {
+ outR, outW := io.Pipe()
+ errR, errW := io.Pipe()
+ return &localTestProcess{outR: outR, outW: outW, errR: errR, errW: errW}
+}
+func (p *localTestProcess) Stdout() io.Reader { return p.outR }
+func (p *localTestProcess) Stderr() io.Reader { return p.errR }
+func (p *localTestProcess) PID() int { return 4321 }
+func (p *localTestProcess) Kill() error {
+ p.killCount++
+ p.finish()
+ return nil
+}
+func (p *localTestProcess) finish() { p.once.Do(func() { _ = p.outW.Close(); _ = p.errW.Close() }) }
+func (p *localTestProcess) Wait(context.Context) (localsandbox.ExitStatus, error) {
+ return p.exit, nil
+}
+
+type localTestStarter struct {
+ proc *localTestProcess
+ root string
+ calls int
+}
+
+func (f *localTestStarter) StartForSession(context.Context, localsandbox.RunRequest) (localsandbox.Process, error) {
+ f.calls++
+ return f.proc, nil
+}
+
+func (f *localTestStarter) GuardForSession(
+ context.Context, string,
+) (*localsandbox.PathGuard, localsandbox.Workspace, error) {
+ return nil, localsandbox.Workspace{Root: f.root}, nil
+}
+
+func localFixture(t *testing.T) (*LocalShellJobService, *localTestProcess, *BackgroundWork) {
+ t.Helper()
+ base := t.TempDir()
+ p := newLocalTestProcess()
+ starter := &localTestStarter{proc: p, root: filepath.Join(base, "project")}
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc, err := newLocalShellJobService(starter, bg, nil, filepath.Join(base, "jobs"),
+ func(context.Context, int) (int64, error) { return 100, nil })
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ _ = svc.Shutdown(ctx)
+ })
+ return svc, p, bg
+}
+
+func TestYieldFinishedCommandIsNotBackgroundedWhenTimerAlsoFires(t *testing.T) {
+ done := make(chan struct{})
+ close(done)
+ timer := make(chan time.Time, 1)
+ timer <- time.Now()
+ stop := make(chan struct{})
+ for range 40 {
+ require.False(t, backgroundedAfterYield(done, stop, timer),
+ "命令已经结束时,计时器同时就绪也不能把它登记成后台任务")
+ }
+}
+
+func TestLocalShortResultDiscardedNotFinished(t *testing.T) {
+ svc, p, bg := localFixture(t)
+ p.finish()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", Description: "quick", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.False(t, sub.Background)
+ require.Equal(t, interfaces.WorkStateSucceeded, sub.Result.State)
+ require.Eventually(t, func() bool {
+ items, _ := bg.Registry.List(context.Background(), ref)
+ return len(items) == 0
+ }, time.Second, 10*time.Millisecond, "Discard 移除登记")
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, notices, "短结果不进信箱、不发事件")
+}
+
+func TestLocalRunningHandleFinishesOnce(t *testing.T) {
+ svc, p, bg := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", Description: "build", YieldTime: time.Millisecond, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background)
+ require.Equal(t, interfaces.WorkStateRunning, sub.Result.State)
+ p.finish()
+ require.Eventually(t, func() bool {
+ notices, _ := bg.Inbox.Pending(context.Background(), ref)
+ return len(notices) == 1
+ }, 2*time.Second, 10*time.Millisecond)
+ notices, _ := bg.Inbox.Pending(context.Background(), ref)
+ require.Equal(t, sub.Result.ID, notices[0].ID)
+ require.Contains(t, notices[0].Text, "succeeded")
+}
+
+func TestLocalCrossSessionIDRejected(t *testing.T) {
+ svc, _, _ := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ got, err := svc.Status(context.Background(), interfaces.SessionRef{TenantID: 8, SessionID: "s"},
+ []string{sub.Result.ID, "sj_abcd"}, interfaces.StatusOptions{Detailed: true})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateLost, got[0].State)
+ require.Equal(t, sub.Result.ID, got[0].ID)
+ require.Equal(t, interfaces.WorkStateLost, got[1].State)
+ require.Equal(t, "sj_abcd", got[1].ID)
+}
+
+func TestLocalStatusAndCancelArePerID(t *testing.T) {
+ svc, p, _ := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background)
+ statusPath := filepath.Join(svc.root, sub.Result.ID, "status.json")
+ require.NoError(t, os.WriteFile(statusPath, []byte("{"), 0o600))
+
+ got, err := svc.Status(context.Background(), ref, []string{sub.Result.ID, "sj_abcd"},
+ interfaces.StatusOptions{Detailed: true})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateLost, got[0].State, "one unreadable job does not fail the batch")
+ require.Equal(t, sub.Result.ID, got[0].ID)
+ require.Equal(t, interfaces.WorkStateLost, got[1].State)
+ require.Equal(t, "sj_abcd", got[1].ID)
+
+ cancelled, err := svc.Cancel(context.Background(), ref, []string{"sj_abcd", sub.Result.ID})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateLost, cancelled[0].State)
+ require.Equal(t, "sj_abcd", cancelled[0].ID)
+ require.NotEqual(t, interfaces.WorkStateLost, cancelled[1].State)
+ require.Eventually(t, func() bool { return p.killCount >= 1 }, time.Second, 10*time.Millisecond)
+}
+
+func TestLocalSubmitIgnoresUnsupportedProcessIdentity(t *testing.T) {
+ base := t.TempDir()
+ p := newLocalTestProcess()
+ starter := &localTestStarter{proc: p, root: filepath.Join(base, "project")}
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ root := filepath.Join(base, "jobs")
+ svc, err := newLocalShellJobService(starter, bg, nil, root,
+ func(context.Context, int) (int64, error) { return 0, localsandbox.ErrUnsupportedPlatform })
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ _ = svc.Shutdown(ctx)
+ })
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ require.Zero(t, p.killCount)
+ raw, err := os.ReadFile(filepath.Join(root, sub.Result.ID, "status.json"))
+ require.NoError(t, err)
+ var status sandbox.ShellJobStatus
+ require.NoError(t, json.Unmarshal(raw, &status))
+ require.Zero(t, status.ProcStartedAt)
+}
+
+func TestLocalSubmitStillFailsWhenProcessIdentityCannotBeRecorded(t *testing.T) {
+ base := t.TempDir()
+ p := newLocalTestProcess()
+ starter := &localTestStarter{proc: p, root: filepath.Join(base, "project")}
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc, err := newLocalShellJobService(starter, bg, nil, filepath.Join(base, "jobs"),
+ func(context.Context, int) (int64, error) { return 0, errors.New("boom") })
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ _ = svc.Shutdown(ctx)
+ })
+ _, err = svc.SubmitShellJob(context.Background(), interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ tools.ShellJobSubmitRequest{Command: "true", YieldTime: time.Millisecond})
+ require.Error(t, err)
+ require.GreaterOrEqual(t, p.killCount, 1)
+}
+
+func TestLocalDeleteBlocksNewSubmitsAndLateNotices(t *testing.T) {
+ svc, _, bg := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background)
+ require.NoError(t, svc.OnDelete(context.Background(), ref))
+ require.NoError(t, bg.Inbox.Clear(context.Background(), ref), "共享 listener 清空信箱")
+ time.Sleep(50 * time.Millisecond) // 给看守终段机会:不允许 Clear 后再 Push
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, notices, "suppress 生效:迟到的 Finish 不重建信箱")
+ _, err = svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{Command: "true"})
+ require.Error(t, err, "删除后该会话提交永久拒绝")
+ status, err := svc.Status(context.Background(), ref, []string{sub.Result.ID},
+ interfaces.StatusOptions{Detailed: true})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateCancelled, status[0].State)
+}
+
+func TestLocalCursorsIncrementalAndHTTPDoesNotMove(t *testing.T) {
+ svc, p, _ := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond, RunID: "m1",
+ })
+ require.NoError(t, err)
+ _, err = p.outW.Write([]byte("first"))
+ require.NoError(t, err)
+ require.Eventually(t, func() bool {
+ r, err := svc.Status(context.Background(), ref, []string{sub.Result.ID},
+ interfaces.StatusOptions{Detailed: true, RunID: "m1"})
+ return err == nil && r[0].Payload.(ShellJobPayload).Stdout == "first"
+ }, time.Second, 10*time.Millisecond)
+ // HTTP 读(RunID 空):开头+结尾,不移动游标。
+ _, err = p.outW.Write([]byte("second"))
+ require.NoError(t, err)
+ require.Eventually(t, func() bool {
+ r, err := svc.Status(context.Background(), ref, []string{sub.Result.ID},
+ interfaces.StatusOptions{Detailed: true})
+ return err == nil && r[0].Payload.(ShellJobPayload).Stdout == "firstsecond"
+ }, time.Second, 10*time.Millisecond)
+ // 同 run 第二次读:只剩 HTTP 读之后才出现的新输出。
+ require.Eventually(t, func() bool {
+ r, err := svc.Status(context.Background(), ref, []string{sub.Result.ID},
+ interfaces.StatusOptions{Detailed: true, RunID: "m1"})
+ return err == nil && r[0].Payload.(ShellJobPayload).Stdout == "second"
+ }, time.Second, 10*time.Millisecond)
+ svc.EndShellJobRun(ref, "m1")
+ svc.mu.Lock()
+ cursor := svc.cursors[localCursorKey(ref, "m1", sub.Result.ID)]
+ svc.mu.Unlock()
+ require.WithinDuration(t, time.Now().Add(time.Hour), cursor.expires, time.Second,
+ "run 结束后一小时过期,而非最后读取后")
+}
+
+func TestLocalStopContextDoesNotKillChild(t *testing.T) {
+ svc, p, _ := localFixture(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ _, err := svc.SubmitShellJob(ctx, ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ cancel() // /stop 语义:只结束提交等待
+ r, err := svc.Status(context.Background(), ref, []string{r0(t, svc, ref)},
+ interfaces.StatusOptions{})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateRunning, r[0].State, "任务不受 run ctx 取消影响")
+ require.Zero(t, p.killCount)
+ p.finish()
+}
+
+func TestLocalShutdownKillsAndRecordsLost(t *testing.T) {
+ svc, p, _ := localFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ require.NoError(t, svc.Shutdown(context.Background()))
+ require.GreaterOrEqual(t, p.killCount, 1, "应用级 ctx 取消触发进程树 Kill")
+ r, err := svc.Status(context.Background(), ref, []string{r0(t, svc, ref)},
+ interfaces.StatusOptions{Detailed: true})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateLost, r[0].State)
+}
+
+func TestLocalSkillEnvCaptureRunsOnlyAfterSuccess(t *testing.T) {
+ for _, mode := range []string{"success", "failure", "delete"} {
+ t.Run(mode, func(t *testing.T) {
+ svc, proc, _ := localFixture(t)
+ if mode == "failure" {
+ proc.exit.Code = 1
+ }
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ sub, err := svc.SubmitShellJob(t.Context(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Millisecond,
+ })
+ require.NoError(t, err)
+ job, err := svc.ownedJob(ref, sub.Result.ID)
+ require.NoError(t, err)
+ calls := 0
+ svc.HoldSkillEnvCapture(ref, sub.Result.ID, func(context.Context) { calls++ })
+ if mode == "delete" {
+ require.NoError(t, svc.OnDelete(t.Context(), ref))
+ svc.HoldSkillEnvCapture(ref, sub.Result.ID, func(context.Context) { calls++ })
+ } else {
+ proc.finish()
+ }
+ select {
+ case <-job.reported:
+ case <-time.After(time.Second):
+ t.Fatal("watcher did not finish reporting")
+ }
+ want := 0
+ if mode == "success" {
+ want = 1
+ }
+ require.Equal(t, want, calls)
+ require.Empty(t, svc.skillCapture.pending())
+ })
+ }
+}
+
+// r0 取该会话唯一的 job id(测试辅助)。
+func r0(t *testing.T, svc *LocalShellJobService, ref interfaces.SessionRef) string {
+ t.Helper()
+ svc.mu.Lock()
+ defer svc.mu.Unlock()
+ for id, j := range svc.jobs {
+ if j.meta.TenantID == ref.TenantID && j.meta.SessionID == ref.SessionID {
+ return id
+ }
+ }
+ t.Fatal("no job")
+ return ""
+}
diff --git a/internal/application/service/session.go b/internal/application/service/session.go
index 7e892c5f7..be06c94c2 100644
--- a/internal/application/service/session.go
+++ b/internal/application/service/session.go
@@ -536,7 +536,7 @@ func (s *sessionService) UpdateSessionLastRequestState(
}
// DeleteSession removes a session by its ID
-func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
+func (s *sessionService) DeleteSession(ctx context.Context, id string) (deleteErr error) {
// Validate session ID
if id == "" {
logger.Error(ctx, "Failed to delete session: session ID cannot be empty")
@@ -557,6 +557,12 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
return fmt.Errorf("read sandbox workspace: %w", err)
}
+ finishBackgroundDelete, err := s.beginBackgroundDelete(ctx, []*types.Session{session})
+ if err != nil {
+ return fmt.Errorf("stop background work before deleting session: %w", err)
+ }
+ defer func() { finishBackgroundDelete(deleteErr == nil) }()
+
// Cleanup chat history knowledge entries for this session (async, best-effort).
// Use WithoutCancel so the goroutine survives after the HTTP request context is done.
bgCtx := context.WithoutCancel(ctx)
@@ -593,7 +599,6 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
}
}
- s.notifyBackgroundDelete(ctx, session)
if !handled[id] {
s.destroyBoundSandbox(ctx, id)
}
@@ -637,7 +642,7 @@ func (s *sessionService) notifyDeletedSessions(ctx context.Context, tenantID uin
}
// BatchDeleteSessions deletes multiple sessions by IDs
-func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string) error {
+func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string) (deleteErr error) {
if len(ids) == 0 {
logger.Error(ctx, "Failed to batch delete sessions: IDs list is empty")
return stderrors.New("session ids are required")
@@ -669,6 +674,12 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string)
return fmt.Errorf("read sandbox workspace: %w", err)
}
+ finishBackgroundDelete, err := s.beginBackgroundDelete(ctx, visible)
+ if err != nil {
+ return fmt.Errorf("stop background work before deleting session: %w", err)
+ }
+ defer func() { finishBackgroundDelete(deleteErr == nil) }()
+
// Cleanup associated resources for each session
bgCtx := context.WithoutCancel(ctx)
for _, id := range visibleIDs {
@@ -696,9 +707,6 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string)
// Tear down sandboxes while session rows (and pins) are still readable.
// A session on a shared machine keeps that machine; only its directory
// is removed after the rows are gone.
- for _, session := range visible {
- s.notifyBackgroundDelete(ctx, session)
- }
for _, id := range visibleIDs {
if handled[id] {
continue
@@ -728,7 +736,7 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string)
}
// DeleteAllSessions deletes all sessions for the current tenant
-func (s *sessionService) DeleteAllSessions(ctx context.Context) error {
+func (s *sessionService) DeleteAllSessions(ctx context.Context) (deleteErr error) {
tenantID := types.MustTenantIDFromContext(ctx)
userID := sessionUserIDFromContext(ctx)
logger.Infof(ctx, "Deleting all sessions for tenant %d", tenantID)
@@ -752,6 +760,11 @@ func (s *sessionService) DeleteAllSessions(ctx context.Context) error {
if err != nil {
return fmt.Errorf("read sandbox workspace: %w", err)
}
+ finishBackgroundDelete, err := s.beginBackgroundDelete(ctx, sessions)
+ if err != nil {
+ return fmt.Errorf("stop background work before deleting session: %w", err)
+ }
+ defer func() { finishBackgroundDelete(deleteErr == nil) }()
bgCtx := context.WithoutCancel(ctx)
for _, session := range sessions {
if session == nil || session.ID == "" {
@@ -783,7 +796,6 @@ func (s *sessionService) DeleteAllSessions(ctx context.Context) error {
if session == nil || session.ID == "" {
continue
}
- s.notifyBackgroundDelete(ctx, session)
if handled[session.ID] {
continue
}
@@ -887,15 +899,42 @@ func (s *sessionService) runUserWorkspaceCleanups(ctx context.Context, handled m
go s.workspaceCleaner.RunCleanups(bg, ids)
}
+// beginBackgroundDelete closes feature submission gates while rows and pins
+// remain readable. Any failed stop or repository deletion must reopen all
+// attempted sessions, including earlier members of a failed batch.
+func (s *sessionService) beginBackgroundDelete(ctx context.Context, sessions []*types.Session) (func(bool), error) {
+ var attempted []*types.Session
+ finish := func(deleted bool) {
+ if deleted || s.background == nil || s.background.Listeners == nil {
+ return
+ }
+ for _, session := range attempted {
+ s.background.Listeners.NotifyDeleteAborted(context.WithoutCancel(ctx),
+ interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})
+ }
+ }
+ for _, session := range sessions {
+ if session == nil {
+ continue
+ }
+ attempted = append(attempted, session)
+ if err := s.notifyBackgroundDelete(ctx, session); err != nil {
+ finish(false)
+ return nil, err
+ }
+ }
+ return finish, nil
+}
+
// notifyBackgroundDelete lets background features stop their work for a
// session about to be deleted, then clears its activity items and notices.
// Call it before destroyBoundSandbox, while features can still reach the
// sandbox, and while the session row is still readable.
-func (s *sessionService) notifyBackgroundDelete(ctx context.Context, session *types.Session) {
+func (s *sessionService) notifyBackgroundDelete(ctx context.Context, session *types.Session) error {
if s.background == nil || s.background.Listeners == nil || session == nil {
- return
+ return nil
}
- s.background.Listeners.NotifyDelete(context.WithoutCancel(ctx),
+ return s.background.Listeners.NotifyDelete(context.WithoutCancel(ctx),
interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})
}
diff --git a/internal/application/service/session_delete_user_mode_test.go b/internal/application/service/session_delete_user_mode_test.go
index 142142595..e5c154e26 100644
--- a/internal/application/service/session_delete_user_mode_test.go
+++ b/internal/application/service/session_delete_user_mode_test.go
@@ -216,6 +216,8 @@ func TestSessionDeleteFailureRollsBackRetirement(t *testing.T) {
for _, operation := range []string{"single", "batch", "all"} {
t.Run(operation, func(t *testing.T) {
f := newUserDeleteFixture(t, "user", nil)
+ _, _, jobIndex, bg := newRemoteServiceFixture(t)
+ f.svc.background = bg
f.place(t, userDeleteSessionID)
ids := []string{userDeleteSessionID}
if operation != "single" {
@@ -240,6 +242,9 @@ BEGIN SELECT RAISE(ABORT, 'injected session delete failure'); END`).Error)
for _, id := range ids {
_, err := f.svc.sessionRepo.GetByID(ctx, 7, id)
require.NoError(t, err, "the failed deletion must leave the session usable")
+ closed, err := jobIndex.SessionClosed(ctx, interfaces.SessionRef{TenantID: 7, SessionID: id})
+ require.NoError(t, err)
+ require.False(t, closed, "failed repository deletion must reopen background submissions")
row := f.assignment(t, id)
require.Nil(t, row.ReleasedAt)
require.Nil(t, row.CleanupDueAt, "GC must not remove files of the surviving session")
diff --git a/internal/application/service/shell_job_followup_test.go b/internal/application/service/shell_job_followup_test.go
new file mode 100644
index 000000000..fea869b8a
--- /dev/null
+++ b/internal/application/service/shell_job_followup_test.go
@@ -0,0 +1,175 @@
+package service
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func TestShellJobHeartbeatRefreshesProvidersWhileReadsAndAnotherProviderBlock(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ svc.heartbeatInterval = 5 * time.Millisecond
+ refs := []interfaces.SessionRef{{TenantID: 7, SessionID: "blocked"}, {TenantID: 7, SessionID: "queued"}}
+ for i, ref := range refs {
+ id := []string{"sj_a001", "sj_a002"}[i]
+ require.NoError(t, index.Add(t.Context(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: id, SandboxID: "sbx1", YieldUntil: time.Now().Add(-time.Minute),
+ }))
+ require.NoError(t, bg.Registry.Add(t.Context(), ref,
+ interfaces.BackgroundActivity{ID: id, Kind: "shell"}, time.Minute, 0))
+ }
+ refreshed := make(chan string, 100)
+ svc.remoteRead = func(ctx context.Context, _ interfaces.SessionRef, _ []string, _ map[string]string) (
+ []sandbox.ShellJobSnapshot, map[string]error,
+ ) {
+ <-ctx.Done()
+ return nil, nil
+ }
+ providerBlocked := make(chan struct{})
+ var blockedOnce sync.Once
+ svc.keeper = keeperFunc(func(ctx context.Context, ref interfaces.SessionRef) error {
+ if ref == refs[0] {
+ blockedOnce.Do(func() { close(providerBlocked) })
+ <-ctx.Done()
+ return ctx.Err()
+ }
+ select {
+ case refreshed <- ref.SessionID:
+ default:
+ }
+ return nil
+ })
+ // A sweep cannot reach renewRunning while waiting for this session lock.
+ locked, release := make(chan struct{}), make(chan struct{})
+ lockDone := make(chan error, 1)
+ go func() {
+ lockDone <- index.WithSessionLock(context.Background(), refs[0], func(context.Context) error {
+ close(locked)
+ <-release
+ return nil
+ })
+ }()
+ <-locked
+ ctx, cancel := context.WithCancel(t.Context())
+ done := make(chan error, 1)
+ t.Cleanup(func() {
+ cancel()
+ close(release)
+ require.NoError(t, <-lockDone)
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Error("heartbeat did not stop with the sweep")
+ }
+ })
+ go func() { done <- svc.SweepOnce(ctx) }()
+ select {
+ case <-providerBlocked:
+ case <-time.After(time.Second):
+ t.Fatal("a blocked sweep never refreshed its sandbox provider")
+ }
+ for range 2 {
+ select {
+ case sid := <-refreshed:
+ require.Equal(t, refs[1].SessionID, sid)
+ case <-time.After(time.Second):
+ t.Fatal("one slow provider starved another session's TTL refresh")
+ }
+ }
+}
+
+func TestShellJobSubmitHonorsCancellationWhileWaitingForSessionLock(t *testing.T) {
+ for _, deadline := range []bool{false, true} {
+ t.Run(map[bool]string{false: "stop", true: "deadline"}[deadline], func(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ locked, release := make(chan struct{}), make(chan struct{})
+ lockDone := make(chan error, 1)
+ go func() {
+ lockDone <- index.WithSessionLock(context.Background(), ref, func(context.Context) error {
+ close(locked)
+ <-release
+ return nil
+ })
+ }()
+ <-locked
+ ctx, cancel := context.WithCancel(t.Context())
+ want := context.Canceled
+ if deadline {
+ cancel()
+ ctx, cancel = context.WithTimeout(t.Context(), 20*time.Millisecond)
+ want = context.DeadlineExceeded
+ }
+ done := make(chan error, 1)
+ t.Cleanup(func() {
+ cancel()
+ close(release)
+ require.NoError(t, <-lockDone)
+ })
+ go func() {
+ _, err := svc.SubmitShellJob(ctx, ref, tools.ShellJobSubmitRequest{Command: "true"})
+ done <- err
+ }()
+ if !deadline {
+ cancel()
+ }
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, want)
+ case <-time.After(time.Second):
+ t.Fatal("cancelled submission is still waiting for the session lock")
+ }
+ live, err := bg.Registry.List(t.Context(), ref)
+ require.NoError(t, err)
+ require.Empty(t, live, "a cancelled waiter must not launch a job")
+ })
+ }
+}
+
+func TestShellJobSubmitCancellationAfterLaunchPreservesBackgroundWork(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ cancel()
+ return &sandbox.ExecuteResult{Error: "context canceled after launch"}, nil
+ }
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ _, err := svc.SubmitShellJob(ctx, ref, tools.ShellJobSubmitRequest{Command: "sleep 60"})
+ require.True(t, err == nil || errors.Is(err, context.Canceled))
+ live, err := bg.Registry.List(t.Context(), ref)
+ require.NoError(t, err)
+ require.Len(t, live, 1)
+ entries, err := index.Entries(t.Context())
+ require.NoError(t, err)
+ require.Len(t, entries, 1)
+}
+
+func TestShellJobRemoteKeepsStdinOutOfMetadataAndMasksDisplayCommand(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ encoded := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("x", 65536)))
+ svc.remoteExec = func(_ context.Context, _ string, script string, _ sandbox.ShellExecOptions) (
+ *sandbox.ExecuteResult, error,
+ ) {
+ require.Equal(t, 1, strings.Count(script, encoded), "stdin belongs only in the executable wrapper")
+ require.Equal(t, 1, strings.Count(script, "private-token"), "metadata must mask the credential")
+ marker := shellJobMarkerInScript.FindString(script)
+ return &sandbox.ExecuteResult{Stdout: marker + "\n" + `{"state":"succeeded","exit_code":0}` + "\n0 0"}, nil
+ }
+ sub, err := svc.SubmitShellJob(t.Context(), interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ tools.ShellJobSubmitRequest{
+ Command: "printf %s " + encoded + " | base64 -d | TOKEN=private-token cat",
+ DisplayCommand: "TOKEN=private-token cat", YieldTime: time.Second,
+ })
+ require.NoError(t, err)
+ require.Equal(t, "TOKEN=*** cat", sub.Result.Payload.(ShellJobPayload).Command)
+}
diff --git a/internal/application/service/shell_job_index.go b/internal/application/service/shell_job_index.go
new file mode 100644
index 000000000..32e882a3c
--- /dev/null
+++ b/internal/application/service/shell_job_index.go
@@ -0,0 +1,424 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/redis/go-redis/v9"
+
+ "github.com/Tencent/WeKnora/internal/common/redislock"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+const (
+ shellJobIndexKey = "weknora:background:shelljob:index"
+ shellJobSweepKeyFmt = "weknora:background:shelljob:sweep:{%s}"
+ shellJobSweepTTL = 15 * time.Second
+ shellJobHandoffSec = 5 // index score = yield_until + 5s (D1)
+)
+
+// ShellJobIndex is the Redis schedule of shell jobs waiting to be swept.
+type ShellJobIndex struct {
+ client redis.UniversalClient
+ namespace string
+}
+
+// ShellJobIndexEntry is one scheduled job: which session and sandbox it
+// belongs to, and when its yield window ends.
+type ShellJobIndexEntry struct {
+ SessionRef interfaces.SessionRef
+ WorkID string
+ SandboxID string
+ YieldUntil time.Time
+}
+
+// NewShellJobIndex schedules shell jobs in client. An empty namespace is "weknora".
+func NewShellJobIndex(client redis.UniversalClient, namespace string) *ShellJobIndex {
+ if strings.TrimSpace(namespace) == "" {
+ namespace = "weknora"
+ }
+ return &ShellJobIndex{client: client, namespace: namespace}
+}
+
+func (x *ShellJobIndex) member(e ShellJobIndexEntry) string {
+ return fmt.Sprintf("%s:%d:%s:%s:%s",
+ x.namespace, e.SessionRef.TenantID, e.SessionRef.SessionID, e.SandboxID, e.WorkID)
+}
+
+func (x *ShellJobIndex) parseMember(member string) (ShellJobIndexEntry, bool) {
+ parts := strings.SplitN(member, ":", 5)
+ if len(parts) != 5 || parts[0] != x.namespace {
+ return ShellJobIndexEntry{}, false
+ }
+ var tenant uint64
+ if _, err := fmt.Sscanf(parts[1], "%d", &tenant); err != nil {
+ return ShellJobIndexEntry{}, false
+ }
+ if !strings.Contains(parts[4], "sj_") {
+ return ShellJobIndexEntry{}, false
+ }
+ return ShellJobIndexEntry{
+ SessionRef: interfaces.SessionRef{TenantID: tenant, SessionID: parts[2]},
+ SandboxID: parts[3], WorkID: parts[4],
+ }, true
+}
+
+// Add schedules e for a sweep five seconds after its yield window.
+func (x *ShellJobIndex) Add(ctx context.Context, e ShellJobIndexEntry) error {
+ score := float64(e.YieldUntil.Unix()) + shellJobHandoffSec
+ return x.client.ZAdd(ctx, shellJobIndexKey, redis.Z{
+ Score: score, Member: x.member(e),
+ }).Err()
+}
+
+func (x *ShellJobIndex) reservationKey(s interfaces.SessionRef, id string) string {
+ return fmt.Sprintf("weknora:background:shelljob:reserved:%s:%d:%s:%s", x.namespace, s.TenantID, s.SessionID, id)
+}
+
+// ReserveID arbitrates submissions across replicas before Lifecycle.Begin can
+// touch an existing activity. Keep the handle reserved through notice retention
+// and refresh it for live jobs; even a directory collision must not reuse it.
+func (x *ShellJobIndex) ReserveID(ctx context.Context, s interfaces.SessionRef, id string) (bool, error) {
+ ok, err := x.client.SetNX(ctx, x.reservationKey(s, id), "1", interfaces.NoticeRetention).Result()
+ if err != nil || !ok {
+ return false, err
+ }
+ // Also recognize jobs submitted before reservation keys were introduced.
+ entries, err := x.Entries(ctx)
+ if err != nil {
+ return false, err
+ }
+ for _, e := range entries {
+ if e.SessionRef == s && e.WorkID == id {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+// Entries includes jobs still in their yield window, which need protection
+// while another session's maintenance exec is blocked.
+func (x *ShellJobIndex) Entries(ctx context.Context) ([]ShellJobIndexEntry, error) {
+ return x.entries(ctx, "+inf")
+}
+
+// WithSessionLock prevents deletion racing a launch or a late sweep result.
+// A renewable lock also covers maintenance execs longer than the sweep claim.
+func (x *ShellJobIndex) WithSessionLock(
+ ctx context.Context, s interfaces.SessionRef, fn func(context.Context) error,
+) error {
+ key := fmt.Sprintf("weknora:background:shelljob:lifecycle:{%s}", x.hashTag(s))
+ return redislock.WithRenewableLock(ctx, x.client, key, time.Minute, 10*time.Second, fn)
+}
+
+func (x *ShellJobIndex) closedKey(s interfaces.SessionRef) string {
+ return fmt.Sprintf("weknora:background:shelljob:closed:{%s}", x.hashTag(s))
+}
+
+// SessionClosed reports whether deletion has closed the session's submission gate.
+func (x *ShellJobIndex) SessionClosed(ctx context.Context, s interfaces.SessionRef) (bool, error) {
+ n, err := x.client.Exists(ctx, x.closedKey(s)).Result()
+ return n != 0, err
+}
+
+// Requeue puts e back on the schedule with its current yield time.
+func (x *ShellJobIndex) Requeue(ctx context.Context, e ShellJobIndexEntry) error {
+ return x.Add(ctx, e)
+}
+
+// Due lists jobs whose sweep time has arrived.
+func (x *ShellJobIndex) Due(ctx context.Context, now time.Time) ([]ShellJobIndexEntry, error) {
+ return x.entries(ctx, fmt.Sprintf("%d", now.Unix()))
+}
+
+func (x *ShellJobIndex) entries(ctx context.Context, maxScore string) ([]ShellJobIndexEntry, error) {
+ scored, err := x.client.ZRangeByScoreWithScores(ctx, shellJobIndexKey, &redis.ZRangeBy{
+ Min: "-inf", Max: maxScore,
+ }).Result()
+ if err != nil {
+ return nil, err
+ }
+ out := make([]ShellJobIndexEntry, 0, len(scored))
+ for _, z := range scored {
+ member, ok := z.Member.(string)
+ if !ok {
+ continue
+ }
+ e, ok := x.parseMember(member)
+ if !ok {
+ continue
+ }
+ // Score is yield_until plus the 5s handoff. Callers use YieldUntil to
+ // tell a just-due job from one whose window ended long ago.
+ e.YieldUntil = time.Unix(int64(z.Score)-shellJobHandoffSec, 0).UTC()
+ out = append(out, e)
+ }
+ return out, nil
+}
+
+// Claim takes the due jobs of every session this replica can lock.
+func (x *ShellJobIndex) Claim(ctx context.Context, now time.Time, replica string) (
+ map[interfaces.SessionRef][]ShellJobIndexEntry, error,
+) {
+ due, err := x.Due(ctx, now)
+ if err != nil {
+ return nil, err
+ }
+ // 先按会话分组,再每会话一次 SET NX(shell 稿 §5.4 的顺序)。绝不能逐条
+ // SetNX:同一会话的锁 key 相同,第二笔会被自己刚设的锁挡掉、从本轮丢出。
+ grouped := make(map[interfaces.SessionRef][]ShellJobIndexEntry)
+ for _, e := range due {
+ grouped[e.SessionRef] = append(grouped[e.SessionRef], e)
+ }
+ out := make(map[interfaces.SessionRef][]ShellJobIndexEntry)
+ for ref, entries := range grouped {
+ key := fmt.Sprintf(shellJobSweepKeyFmt, x.hashTag(ref))
+ ok, err := x.client.SetNX(ctx, key, replica, shellJobSweepTTL).Result()
+ if err != nil {
+ return nil, err
+ }
+ if ok {
+ out[ref] = entries
+ }
+ }
+ return out, nil
+}
+
+// Release drops the session sweep lock when this replica still holds it.
+func (x *ShellJobIndex) Release(ctx context.Context, s interfaces.SessionRef, replica string) error {
+ key := fmt.Sprintf(shellJobSweepKeyFmt, x.hashTag(s))
+ current, err := x.client.Get(ctx, key).Result()
+ if err == redis.Nil {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if current != replica {
+ return nil
+ }
+ return x.client.Del(ctx, key).Err()
+}
+
+// ClaimWork removes one job from the schedule so another replica will not sweep it.
+func (x *ShellJobIndex) ClaimWork(ctx context.Context, s interfaces.SessionRef, workID string) (bool, error) {
+ // Member carries the sandbox id; ZREM by the session+work pair alone is not
+ // possible, so remove by range: find the member first.
+ members, err := x.client.ZRangeByScore(ctx, shellJobIndexKey, &redis.ZRangeBy{
+ Min: "-inf", Max: "+inf",
+ }).Result()
+ if err != nil {
+ return false, err
+ }
+ prefix := x.sessionPrefix(s)
+ for _, m := range members {
+ if strings.HasPrefix(m, prefix) && strings.HasSuffix(m, ":"+workID) {
+ n, err := x.client.ZRem(ctx, shellJobIndexKey, m).Result()
+ if err != nil {
+ return false, err
+ }
+ return n == 1, nil
+ }
+ }
+ // Not in the index (already claimed by the submitter's ZREM): not ours.
+ return false, nil
+}
+
+// SandboxIDs resolves the expected sandbox of each id from this session's index
+// members (D1: the member carries sandbox_id). Ids absent from the index map to
+// "" — the caller falls back to the current binding.
+func (x *ShellJobIndex) SandboxIDs(ctx context.Context, s interfaces.SessionRef,
+ ids []string,
+) map[string]string {
+ out := make(map[string]string, len(ids))
+ if len(ids) == 0 {
+ return out
+ }
+ members, err := x.client.ZRange(ctx, shellJobIndexKey, 0, -1).Result()
+ if err != nil {
+ return out
+ }
+ prefix := x.sessionPrefix(s)
+ byWork := make(map[string]string, len(members))
+ for _, m := range members {
+ if strings.HasPrefix(m, prefix) {
+ rest := strings.TrimPrefix(m, prefix) // <sandbox_id>:<work_id>
+ if i := strings.LastIndex(rest, ":"); i > 0 {
+ byWork[rest[i+1:]] = rest[:i]
+ }
+ }
+ }
+ for _, id := range ids {
+ out[id] = byWork[id]
+ }
+ return out
+}
+
+func (x *ShellJobIndex) sessionPrefix(s interfaces.SessionRef) string {
+ return fmt.Sprintf("%s:%d:%s:", x.namespace, s.TenantID, s.SessionID)
+}
+
+// RemoveSession drops every scheduled job for s.
+func (x *ShellJobIndex) RemoveSession(ctx context.Context, s interfaces.SessionRef) error {
+ members, err := x.client.ZRange(ctx, shellJobIndexKey, 0, -1).Result()
+ if err != nil {
+ return err
+ }
+ prefix := fmt.Sprintf("%s:%d:%s:", x.namespace, s.TenantID, s.SessionID)
+ victims := make([]any, 0, len(members))
+ for _, m := range members {
+ if strings.HasPrefix(m, prefix) {
+ victims = append(victims, m)
+ }
+ }
+ if len(victims) == 0 {
+ return nil
+ }
+ return x.client.ZRem(ctx, shellJobIndexKey, victims...).Err()
+}
+
+func (x *ShellJobIndex) hashTag(s interfaces.SessionRef) string {
+ return fmt.Sprintf("%s:%d:%s", x.namespace, s.TenantID, s.SessionID)
+}
+
+func (x *ShellJobIndex) outcomeKey(s interfaces.SessionRef, workID string) string {
+ return fmt.Sprintf("weknora:background:shelljob:outcome:%s:%d:%s:%s",
+ x.namespace, s.TenantID, s.SessionID, workID)
+}
+
+// RecordOutcome keeps a terminal state where every replica can see it. The
+// submitting process is the only one that can run a credential write-back, and
+// an inbox notice can be acknowledged before that process sweeps.
+func (x *ShellJobIndex) RecordOutcome(ctx context.Context, s interfaces.SessionRef, workID, state string) error {
+ if x == nil || workID == "" || state == "" {
+ return nil
+ }
+ return x.client.Set(ctx, x.outcomeKey(s, workID), state, interfaces.NoticeRetention).Err()
+}
+
+// Outcome returns the terminal state recorded for one job.
+func (x *ShellJobIndex) Outcome(ctx context.Context, s interfaces.SessionRef, workID string) (string, bool, error) {
+ if x == nil || workID == "" {
+ return "", false, nil
+ }
+ val, err := x.client.Get(ctx, x.outcomeKey(s, workID)).Result()
+ if err == redis.Nil {
+ return "", false, nil
+ }
+ if err != nil {
+ return "", false, err
+ }
+ if val == "" {
+ return "", false, nil
+ }
+ return val, true, nil
+}
+
+func (x *ShellJobIndex) cursorKey(s interfaces.SessionRef, runID, workID string) string {
+ return fmt.Sprintf("weknora:background:shelljob:cursor:%s:%d:%s:%s:%s",
+ x.namespace, s.TenantID, s.SessionID, runID, workID)
+}
+
+func (x *ShellJobIndex) cursorRunKey(s interfaces.SessionRef, runID string) string {
+ return fmt.Sprintf("weknora:background:shelljob:cursors:%s:%d:%s:%s",
+ x.namespace, s.TenantID, s.SessionID, runID)
+}
+
+func (x *ShellJobIndex) cursorSessionKey(s interfaces.SessionRef) string {
+ return fmt.Sprintf("weknora:background:shelljob:cursors:%s:%d:%s",
+ x.namespace, s.TenantID, s.SessionID)
+}
+
+// PutCursor stores one run's read offset so another replica can continue it.
+func (x *ShellJobIndex) PutCursor(ctx context.Context, s interfaces.SessionRef, runID, workID string,
+ from sandbox.ShellJobOffsets, ttl time.Duration,
+) error {
+ if x == nil || runID == "" || workID == "" {
+ return nil
+ }
+ if ttl <= 0 {
+ ttl = time.Hour
+ }
+ raw, err := json.Marshal(from)
+ if err != nil {
+ return err
+ }
+ key := x.cursorKey(s, runID, workID)
+ runSet := x.cursorRunKey(s, runID)
+ sessionSet := x.cursorSessionKey(s)
+ pipe := x.client.TxPipeline()
+ pipe.Set(ctx, key, raw, ttl)
+ pipe.SAdd(ctx, runSet, workID)
+ pipe.Expire(ctx, runSet, ttl)
+ pipe.SAdd(ctx, sessionSet, key, runSet)
+ pipe.Expire(ctx, sessionSet, ttl)
+ _, err = pipe.Exec(ctx)
+ return err
+}
+
+// GetCursor returns a run's stored read offset for one job.
+func (x *ShellJobIndex) GetCursor(ctx context.Context, s interfaces.SessionRef, runID, workID string) (
+ sandbox.ShellJobOffsets, bool, error,
+) {
+ if x == nil || runID == "" || workID == "" {
+ return sandbox.ShellJobOffsets{}, false, nil
+ }
+ raw, err := x.client.Get(ctx, x.cursorKey(s, runID, workID)).Result()
+ if err == redis.Nil {
+ return sandbox.ShellJobOffsets{}, false, nil
+ }
+ if err != nil {
+ return sandbox.ShellJobOffsets{}, false, err
+ }
+ var from sandbox.ShellJobOffsets
+ if err := json.Unmarshal([]byte(raw), &from); err != nil {
+ return sandbox.ShellJobOffsets{}, false, err
+ }
+ return from, true, nil
+}
+
+// TouchRunCursors extends every cursor for one run. EndShellJobRun calls it
+// so the offsets live for an hour after the run, not after the last read.
+func (x *ShellJobIndex) TouchRunCursors(
+ ctx context.Context, s interfaces.SessionRef, runID string, ttl time.Duration,
+) error {
+ if x == nil || runID == "" {
+ return nil
+ }
+ if ttl <= 0 {
+ ttl = time.Hour
+ }
+ runSet := x.cursorRunKey(s, runID)
+ ids, err := x.client.SMembers(ctx, runSet).Result()
+ if err != nil {
+ return err
+ }
+ pipe := x.client.TxPipeline()
+ for _, id := range ids {
+ pipe.Expire(ctx, x.cursorKey(s, runID, id), ttl)
+ }
+ pipe.Expire(ctx, runSet, ttl)
+ pipe.Expire(ctx, x.cursorSessionKey(s), ttl)
+ _, err = pipe.Exec(ctx)
+ return err
+}
+
+// DeleteSessionCursors removes every cursor key recorded for the session.
+func (x *ShellJobIndex) DeleteSessionCursors(ctx context.Context, s interfaces.SessionRef) error {
+ if x == nil {
+ return nil
+ }
+ sessionSet := x.cursorSessionKey(s)
+ keys, err := x.client.SMembers(ctx, sessionSet).Result()
+ if err != nil {
+ return err
+ }
+ victims := make([]string, 0, len(keys)+1)
+ victims = append(victims, keys...)
+ victims = append(victims, sessionSet)
+ return x.client.Del(ctx, victims...).Err()
+}
diff --git a/internal/application/service/shell_job_index_test.go b/internal/application/service/shell_job_index_test.go
new file mode 100644
index 000000000..89eb1efba
--- /dev/null
+++ b/internal/application/service/shell_job_index_test.go
@@ -0,0 +1,148 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/require"
+)
+
+func newIndexFixture(t *testing.T) (*ShellJobIndex, *miniredis.Miniredis) {
+ t.Helper()
+ mr := miniredis.RunT(t)
+ client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ return NewShellJobIndex(client, "weknora"), mr
+}
+
+func TestShellJobIndexLifecycle(t *testing.T) {
+ x, _ := newIndexFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ entry := ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123",
+ SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }
+
+ require.NoError(t, x.Add(ctx, entry))
+ // 同一会话第二笔:两笔必须在同一次 Claim 里一起回来——抢占是按会话一次
+ // SET NX,不是逐条。逐条 SetNX 会让第二笔被自己刚设的锁挡掉(回归钉子)。
+ entry2 := ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_ab01",
+ SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }
+ require.NoError(t, x.Add(ctx, entry2))
+ due, err := x.Due(ctx, time.Unix(1000, 0))
+ require.NoError(t, err)
+ require.Empty(t, due, "score 含 5 秒交接余量,窗口结束时刻尚不 due")
+ due, err = x.Due(ctx, time.Unix(1006, 0))
+ require.NoError(t, err)
+ require.Len(t, due, 2)
+ require.Equal(t, "sbx1", due[0].SandboxID)
+ require.True(t, due[0].YieldUntil.Equal(time.Unix(1000, 0).UTC()),
+ "到期条目要带回 yield_until,score 里的 5 秒交接余量不算进窗口")
+
+ claimed, err := x.Claim(ctx, time.Unix(1006, 0), "replica-a")
+ require.NoError(t, err)
+ require.Len(t, claimed[ref], 2, "同一会话的全部到期任务随一次抢占一起返回")
+ claimed, err = x.Claim(ctx, time.Unix(1006, 0), "replica-b")
+ require.NoError(t, err)
+ require.Empty(t, claimed, "另一副本 15 秒内抢不到同一会话")
+ require.NoError(t, x.Release(ctx, ref, "replica-a"))
+ claimed, err = x.Claim(ctx, time.Unix(1006, 0), "replica-b")
+ require.NoError(t, err)
+ require.Len(t, claimed[ref], 2)
+}
+
+func TestShellJobIndexClaimWorkExactlyOneWinner(t *testing.T) {
+ x, _ := newIndexFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, x.Add(ctx, ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }))
+
+ first, err := x.ClaimWork(ctx, ref, "sj_0123")
+ require.NoError(t, err)
+ require.True(t, first, "ZREM 返回 1 的一方获得报告权")
+ second, err := x.ClaimWork(ctx, ref, "sj_0123")
+ require.NoError(t, err)
+ require.False(t, second, "第二个 ZREM 返回 0,不报告(D2)")
+
+ require.NoError(t, x.Requeue(ctx, ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }))
+ first, err = x.ClaimWork(ctx, ref, "sj_0123")
+ require.NoError(t, err)
+ require.True(t, first, "Requeue 后下一轮可重试")
+}
+
+func TestShellJobIndexSandboxIDs(t *testing.T) {
+ x, _ := newIndexFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, x.Add(ctx, ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123", SandboxID: "sbx-a", YieldUntil: time.Unix(1000, 0),
+ }))
+ // 另一会话的同名任务不得串扰
+ other := interfaces.SessionRef{TenantID: 7, SessionID: "other"}
+ require.NoError(t, x.Add(ctx, ShellJobIndexEntry{
+ SessionRef: other,
+ WorkID: "sj_ab01", SandboxID: "sbx-x", YieldUntil: time.Unix(1000, 0),
+ }))
+
+ got := x.SandboxIDs(ctx, ref, []string{"sj_0123", "sj_ab01"})
+ require.Equal(t, "sbx-a", got["sj_0123"], "expected 沙盒来自索引成员(D1)")
+ require.Empty(t, got["sj_ab01"], "索引外的 ID 映射为空,由调用方退回当前绑定")
+}
+
+func TestShellJobIndexRemoveSession(t *testing.T) {
+ x, _ := newIndexFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, x.Add(ctx, ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }))
+ require.NoError(t, x.Add(ctx, ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_ab01", SandboxID: "sbx1", YieldUntil: time.Unix(1000, 0),
+ }))
+ require.NoError(t, x.RemoveSession(ctx, ref))
+ due, err := x.Due(ctx, time.Unix(2000, 0))
+ require.NoError(t, err)
+ require.Empty(t, due)
+}
+
+func TestShellJobIndexOutcomeAndCursor(t *testing.T) {
+ x, mr := newIndexFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(t, x.RecordOutcome(ctx, ref, "sj_0123", interfaces.WorkStateSucceeded))
+ state, ok, err := x.Outcome(ctx, ref, "sj_0123")
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.Equal(t, interfaces.WorkStateSucceeded, state)
+ ttl := mr.TTL("weknora:background:shelljob:outcome:weknora:7:s:sj_0123")
+ require.Greater(t, ttl, 23*time.Hour)
+ _, ok, err = x.Outcome(ctx, ref, "sj_ab12")
+ require.NoError(t, err)
+ require.False(t, ok)
+
+ require.NoError(t, x.PutCursor(ctx, ref, "m1", "sj_0123", sandbox.ShellJobOffsets{Stdout: 5}, time.Hour))
+ got, ok, err := x.GetCursor(ctx, ref, "m1", "sj_0123")
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.Equal(t, int64(5), got.Stdout)
+ require.NoError(t, x.DeleteSessionCursors(ctx, ref))
+ _, ok, err = x.GetCursor(ctx, ref, "m1", "sj_0123")
+ require.NoError(t, err)
+ require.False(t, ok)
+}
diff --git a/internal/application/service/shell_job_regression_test.go b/internal/application/service/shell_job_regression_test.go
new file mode 100644
index 000000000..4c23ea1eb
--- /dev/null
+++ b/internal/application/service/shell_job_regression_test.go
@@ -0,0 +1,738 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func TestShellJobReadFailureMustNotHideCompletion(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ id := "sj_0123"
+ reader.errs = map[string]error{id: errors.New("sandbox exec timed out")}
+ marks := background.NewRunNotices(ref, bg.Registry, bg.Inbox)
+ wait := tools.NewWorkWaitTool(svc, ref, "m1", marks, nil, []string{"shell_exec_async"})
+ result, err := wait.Execute(ctx, json.RawMessage("{\"work_ids\":[\"sj_0123\"],\"wait_sec\":0}"))
+ require.ErrorContains(t, err, "sandbox exec timed out")
+ require.Nil(t, result)
+ require.Empty(t, marks.TakeSeen())
+ end := time.Now().UTC()
+ reader.errs = nil
+ reader.snapshots[id] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: id, TenantID: 7, SessionID: "s", SandboxID: "sbx1", Title: "build"},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateSucceeded, EndedAt: &end},
+ }
+ require.NoError(
+ t,
+ index.Add(
+ ctx,
+ ShellJobIndexEntry{SessionRef: ref, WorkID: id, SandboxID: "sbx1", YieldUntil: end.Add(-time.Minute)},
+ ),
+ )
+ require.NoError(t, svc.SweepOnce(ctx))
+ delivery, err := marks.RoundStart(ctx)
+ require.NoError(t, err)
+ require.Contains(t, delivery.Text, id, "a transient read must not suppress the real completion")
+}
+
+func TestShellJobForeignStatusMustNotReturnMetadata(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: "sj_0123",
+ TenantID: 7,
+ SessionID: "other-session",
+ SandboxID: "sbx1",
+ Command: "private-command",
+ WorkDir: "/workspace/private",
+ Title: "private-title",
+ },
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ }
+ got, err := svc.Status(
+ context.Background(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ []string{"sj_0123"},
+ interfaces.StatusOptions{Detailed: true},
+ )
+ require.NoError(t, err)
+ p := got[0].Payload.(ShellJobPayload)
+ t.Logf("foreign response: title=%s payload=%+v", got[0].Title, p)
+ require.Empty(t, p.Command)
+ require.Empty(t, p.SessionID)
+ require.Empty(t, p.WorkDir)
+ require.NotContains(t, got[0].Title, "private")
+}
+
+func TestShellJobForeignCancelMustCheckOwnership(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "other-session", SandboxID: "sbx1"},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ }
+ signalled := false
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ cmd string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ if strings.Contains(cmd, "kill -TERM") {
+ signalled = true
+ }
+ return &sandbox.ExecuteResult{Stdout: "{\"state\":\"cancelled\"}"}, nil
+ }
+ _, err := svc.Cancel(context.Background(), interfaces.SessionRef{TenantID: 7, SessionID: "s"}, []string{"sj_0123"})
+ require.ErrorIs(t, err, interfaces.ErrWorkNotFound)
+ require.False(t, signalled, "another session's id reached the signal command")
+}
+
+func TestShellJobRemoteOutputBudget(t *testing.T) {
+ svc, _, _, bg := newRemoteServiceFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ root := t.TempDir()
+ ids := make([]string, 16)
+ for i := range ids {
+ ids[i] = fmt.Sprintf("sj_%04x", i)
+ dir := filepath.Join(root, ids[i])
+ require.NoError(t, os.MkdirAll(dir, 0o700))
+ require.NoError(
+ t,
+ sandbox.WriteShellJobJSON(
+ filepath.Join(dir, "meta.json"),
+ sandbox.ShellJobMeta{ID: ids[i], TenantID: 7, SessionID: "s", SandboxID: "sbx1"},
+ ),
+ )
+ require.NoError(
+ t,
+ sandbox.WriteShellJobJSON(
+ filepath.Join(dir, "status.json"),
+ sandbox.ShellJobStatus{State: interfaces.WorkStateSucceeded},
+ ),
+ )
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout"), []byte(strings.Repeat("\x01", 16*1024)), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stderr"), []byte(strings.Repeat("\x02", 8*1024)), 0o600))
+ }
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.remoteExec = func(
+ ctx context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ cmd := exec.CommandContext(ctx, "sh")
+ cmd.Stdin = strings.NewReader(strings.ReplaceAll(script, sandbox.ShellJobsRoot, root))
+ out, err := cmd.Output()
+ return &sandbox.ExecuteResult{Stdout: string(out)}, err
+ }
+ results, err := svc.Status(ctx, ref, ids, interfaces.StatusOptions{Detailed: true, OutputBudget: 32 * 1024 / 16})
+ require.NoError(t, err)
+ total := 0
+ for _, r := range results {
+ p := r.Payload.(ShellJobPayload)
+ total += len(p.Stdout) + len(p.Stderr)
+ }
+ t.Logf("requested total budget 32768; actual body bytes=%d", total)
+ if total > 32*1024 {
+ t.Errorf("remote results ignore the caller's output budget: %d", total)
+ }
+ registry := tools.NewToolRegistry()
+ marks := background.NewRunNotices(ref, bg.Registry, bg.Inbox)
+ registry.RegisterTool(tools.NewWorkWaitTool(svc, ref, "m2", marks, nil, []string{"shell_exec_async"}))
+ args, _ := json.Marshal(map[string]any{"work_ids": ids, "wait_sec": 0})
+ rendered, err := registry.ExecuteTool(ctx, "work_wait", args)
+ require.NoError(t, err)
+ t.Logf(
+ "rendered bytes=%d validJSON=%v acknowledged=%d",
+ len(rendered.Output),
+ json.Valid([]byte(rendered.Output)),
+ len(marks.TakeSeen()),
+ )
+ require.True(t, json.Valid([]byte(rendered.Output)), "the actual tool registry cuts the batched JSON")
+}
+
+func TestShellJobInlineResultMustPreserveTail(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ marker := shellJobMarkerInScript.FindString(script)
+ text := strings.Repeat("x", 20000) + "FINAL-DIAGNOSTIC\n"
+ out := text + marker + "\n" + "{\"state\":\"failed\",\"exit_code\":1}" + fmt.Sprintf("\n%d 0\n", len(text))
+ return &sandbox.ExecuteResult{Stdout: out}, nil
+ }
+ result, err := svc.SubmitShellJob(
+ context.Background(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ tools.ShellJobSubmitRequest{Command: "build", YieldTime: time.Second},
+ )
+ require.NoError(t, err)
+ p := result.Result.Payload.(ShellJobPayload)
+ t.Logf(
+ "stdout=%d total=%d truncated=%v omitted=%d",
+ len(p.Stdout),
+ p.TotalBytes.Stdout,
+ p.Truncated,
+ p.OmittedBytes.Stdout,
+ )
+ require.Contains(t, p.Stdout, "FINAL-DIAGNOSTIC")
+ require.True(t, p.Truncated)
+ require.Equal(t, p.TotalBytes.Stdout-int64(len(p.Stdout)), p.OmittedBytes.Stdout)
+}
+
+func TestShellJobDeleteMustStopRemoteWork(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", "s", "sbx1")
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(
+ t,
+ index.Add(
+ ctx,
+ ShellJobIndexEntry{SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: time.Now()},
+ ),
+ )
+ require.NoError(
+ t,
+ bg.Registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell"}, time.Minute, 0),
+ )
+ cancelled := false
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ if strings.Contains(script, "kill") {
+ cancelled = true
+ }
+ return &sandbox.ExecuteResult{Stdout: `{"state":"cancelled"}`}, nil
+ }
+ require.NoError(t, svc.OnDelete(ctx, ref))
+ due, err := index.Due(ctx, time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Empty(t, due)
+ require.True(t, cancelled, "removing an index entry does not kill a detached command on a retained shared machine")
+}
+
+func TestShellJobReplacedSandboxMustStopRenewing(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ctx := context.Background()
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ require.NoError(
+ t,
+ index.Add(
+ ctx,
+ ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123",
+ SandboxID: "old",
+ YieldUntil: time.Now().Add(-time.Minute),
+ },
+ ),
+ )
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ _ string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ require.Equal(t, "old", opts.ExpectedSandboxID)
+ return nil, fmt.Errorf("maintenance: %w", sandbox.ErrBoundSandboxChanged)
+ }
+ require.NoError(t, svc.SweepOnce(ctx))
+ require.NoError(t, svc.SweepOnce(ctx))
+ pending, err := bg.Inbox.Pending(ctx, ref)
+ require.NoError(t, err)
+ live, err := bg.Registry.List(ctx, ref)
+ require.NoError(t, err)
+ t.Logf("permanently replaced sandbox: live=%d notices=%d", len(live), len(pending))
+ require.Len(t, pending, 1, "a confirmed binding change cannot recover on a subsequent retry")
+}
+
+func TestShellJobRotatedIncrementalReadMustNotSkipUnreadTail(t *testing.T) {
+ svc, _, index, _ := newRemoteServiceFixture(t)
+ ctx := context.WithValue(context.Background(), shellReadRunKey{}, "m1")
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ root := t.TempDir()
+ dir := filepath.Join(root, "sj_0123")
+ require.NoError(t, os.MkdirAll(dir, 0o700))
+ require.NoError(
+ t,
+ sandbox.WriteShellJobJSON(
+ filepath.Join(dir, "meta.json"),
+ sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "sbx1"},
+ ),
+ )
+ require.NoError(
+ t,
+ sandbox.WriteShellJobJSON(
+ filepath.Join(dir, "status.json"),
+ sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ ),
+ )
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.1"), []byte(strings.Repeat("x", 20000)), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout"), []byte("FINAL-DIAGNOSTIC"), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.rotated"), []byte("65536"), 0o600))
+ require.NoError(t, index.PutCursor(ctx, ref, "m1", "sj_0123", sandbox.ShellJobOffsets{Stdout: 1}, time.Hour))
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.remoteExec = func(
+ ctx context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ cmd := exec.CommandContext(ctx, "sh")
+ cmd.Stdin = strings.NewReader(strings.ReplaceAll(script, sandbox.ShellJobsRoot, root))
+ out, err := cmd.Output()
+ return &sandbox.ExecuteResult{Stdout: string(out)}, err
+ }
+ first, err := svc.Status(ctx, ref, []string{"sj_0123"}, interfaces.StatusOptions{Detailed: true, RunID: "m1"})
+ require.NoError(t, err)
+ p := first[0].Payload.(ShellJobPayload)
+ cursor, _, err := index.GetCursor(ctx, ref, "m1", "sj_0123")
+ require.NoError(t, err)
+ t.Logf(
+ "window from=%d len=%d omitted=%d total=%d nextCursor=%d",
+ p.FromBytes.Stdout,
+ len(p.Stdout),
+ p.OmittedBytes.Stdout,
+ p.TotalBytes.Stdout,
+ cursor.Stdout,
+ )
+ require.Equal(
+ t,
+ p.FromBytes.Stdout+int64(len(p.Stdout)),
+ cursor.Stdout,
+ "the cursor skips remaining bytes still present on disk",
+ )
+ second, err := svc.Status(ctx, ref, []string{"sj_0123"}, interfaces.StatusOptions{Detailed: true, RunID: "m1"})
+ require.NoError(t, err)
+ require.Contains(t, second[0].Payload.(ShellJobPayload).Stdout, "FINAL-DIAGNOSTIC")
+}
+
+func ownedRunningShellSnapshot(id, session, box string) sandbox.ShellJobSnapshot {
+ return sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: id, TenantID: 7, SessionID: session, SandboxID: box, Title: "original job"},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning, StartedAt: time.Now()}, WatcherAlive: true,
+ }
+}
+
+type shellLeaseClock struct{ nanos atomic.Int64 }
+
+func (c *shellLeaseClock) Now() time.Time { return time.Unix(0, c.nanos.Load()) }
+
+type observedShellRegistry struct {
+ interfaces.ActivityRegistry
+ renewed chan string
+}
+
+func (r observedShellRegistry) Renew(
+ ctx context.Context,
+ ref interfaces.SessionRef,
+ id string,
+ lease time.Duration,
+) (bool, error) {
+ found, err := r.ActivityRegistry.Renew(ctx, ref, id, lease)
+ select {
+ case r.renewed <- id:
+ default:
+ }
+ return found, err
+}
+
+func TestShellJobLeaseSurvivesBlockedReadAndQueuedSessions(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ clock := &shellLeaseClock{}
+ clock.nanos.Store(time.Now().UnixNano())
+ observed := observedShellRegistry{
+ ActivityRegistry: background.NewMemoryActivityRegistry(clock),
+ renewed: make(chan string, 100),
+ }
+ bg.Registry = observed
+ svc.lifecycle.Registry = observed
+ svc.heartbeatInterval = 5 * time.Millisecond
+ refs := []interfaces.SessionRef{{TenantID: 7, SessionID: "first"}, {TenantID: 7, SessionID: "queued"}}
+ for i, ref := range refs {
+ id := fmt.Sprintf("sj_%04x", i)
+ require.NoError(
+ t,
+ observed.Add(t.Context(), ref, interfaces.BackgroundActivity{ID: id, Kind: "shell"}, shellJobRenewLease, 0),
+ )
+ require.NoError(
+ t,
+ index.Add(
+ t.Context(),
+ ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: id,
+ SandboxID: "sbx1",
+ YieldUntil: time.Now().Add(-time.Minute),
+ },
+ ),
+ )
+ }
+ entered := make(chan struct{}, 2)
+ release := make(chan struct{})
+ var releaseOnce sync.Once
+ t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
+ svc.remoteRead = func(
+ _ context.Context,
+ ref interfaces.SessionRef,
+ ids []string,
+ _ map[string]string,
+ ) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ entered <- struct{}{}
+ <-release
+ return []sandbox.ShellJobSnapshot{ownedRunningShellSnapshot(ids[0], ref.SessionID, "sbx1")}, nil
+ }
+ done := make(chan error, 1)
+ go func() { done <- svc.SweepOnce(t.Context()) }()
+ select {
+ case <-entered:
+ case <-time.After(2 * time.Second):
+ t.Fatal("sweep never entered remote read")
+ }
+ for round := 0; round < 5; round++ {
+ for len(observed.renewed) > 0 {
+ <-observed.renewed
+ }
+ clock.nanos.Add(int64(10 * time.Second))
+ seen := map[string]bool{}
+ deadline := time.NewTimer(time.Second)
+ for len(seen) < 2 {
+ select {
+ case id := <-observed.renewed:
+ seen[id] = true
+ case <-deadline.C:
+ t.Fatal("blocked read starved activity renewal")
+ }
+ }
+ deadline.Stop()
+ for _, ref := range refs {
+ live, err := observed.List(t.Context(), ref)
+ require.NoError(t, err)
+ require.Len(t, live, 1, "a queued session must remain protected too")
+ }
+ }
+ releaseOnce.Do(func() { close(release) })
+ require.NoError(t, <-done)
+}
+
+func TestShellJobIDCollisionPreservesExistingRegistrationAndIndex(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ original := ShellJobIndexEntry{
+ SessionRef: ref,
+ WorkID: "sj_0123",
+ SandboxID: "old-box",
+ YieldUntil: time.Now().UTC().Add(-time.Minute).Truncate(time.Second),
+ }
+ require.NoError(t, index.Add(t.Context(), original))
+ require.NoError(
+ t,
+ bg.Registry.Add(
+ t.Context(),
+ ref,
+ interfaces.BackgroundActivity{ID: original.WorkID, Kind: "shell", Title: "original"},
+ time.Hour,
+ 0,
+ ),
+ )
+ ids := []string{original.WorkID, "sj_f00d"}
+ svc.newJobID = func() (string, error) { id := ids[0]; ids = ids[1:]; return id, nil }
+ var executed []string
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ require.NotContains(t, script, original.WorkID)
+ executed = append(executed, script)
+ return &sandbox.ExecuteResult{Error: "connection interrupted after launch"}, nil
+ }
+ got, err := svc.SubmitShellJob(
+ t.Context(),
+ ref,
+ tools.ShellJobSubmitRequest{Command: "sleep 60", YieldTime: time.Second},
+ )
+ require.NoError(t, err)
+ require.Equal(t, "sj_f00d", got.Result.ID)
+ require.Len(t, executed, 1)
+ entries, err := index.Entries(t.Context())
+ require.NoError(t, err)
+ require.Contains(t, entries, original)
+ live, err := bg.Registry.List(t.Context(), ref)
+ require.NoError(t, err)
+ require.Len(t, live, 2)
+ for _, a := range live {
+ if a.ID == original.WorkID {
+ require.Equal(t, "original", a.Title)
+ }
+ }
+ // A second replica must also fail to reserve the newly launched handle.
+ other := NewShellJobIndex(index.client, index.namespace)
+ reserved, err := other.ReserveID(t.Context(), ref, "sj_f00d")
+ require.NoError(t, err)
+ require.False(t, reserved)
+}
+
+func TestShellJobMalformedReadDoesNotConsumeCursors(t *testing.T) {
+ svc, _, index, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{Stdout: `{"meta":{"work_id":"sj_0123"},"status":`}, nil
+ }
+ _, err := svc.Status(t.Context(), ref, []string{"sj_0123"}, interfaces.StatusOptions{Detailed: true, RunID: "run"})
+ require.ErrorContains(t, err, "unreadable")
+ _, found, err := index.GetCursor(t.Context(), ref, "run", "sj_0123")
+ require.NoError(t, err)
+ require.False(t, found)
+}
+
+func TestShellJobInlineResultReadsFinalTailAfterForwardingCap(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ svc.newJobID = func() (string, error) { return "sj_f004", nil }
+ snap := ownedRunningShellSnapshot("sj_f004", "s", "sbx1")
+ snap.Status.State = interfaces.WorkStateFailed
+ snap.Stdout = sandbox.ShellJobStream{
+ Text: "HEAD\nFINAL-DIAGNOSTIC",
+ TotalBytes: 2 * 1024 * 1024,
+ Truncated: true,
+ OmittedBytes: 2*1024*1024 - 21,
+ }
+ reader.snapshots[snap.Meta.ID] = snap
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ script string,
+ _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ marker := shellJobMarkerInScript.FindString(script)
+ return &sandbox.ExecuteResult{
+ Stdout: strings.Repeat(
+ "x",
+ 1024*1024,
+ ) + marker + "\n" + `{"state":"failed","exit_code":1}` + "\n2097152 0\n",
+ }, nil
+ }
+ result, err := svc.SubmitShellJob(
+ t.Context(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ tools.ShellJobSubmitRequest{Command: "build", YieldTime: time.Second},
+ )
+ require.NoError(t, err)
+ require.False(t, result.Background)
+ require.Contains(t, result.Result.Payload.(ShellJobPayload).Stdout, "FINAL-DIAGNOSTIC")
+ require.Equal(t, 1, reader.execCalls)
+}
+
+func TestShellJobSharedMachineDeleteStopsBeforeRemovingSession(t *testing.T) {
+ for _, operation := range []string{"single", "batch", "all"} {
+ t.Run(operation, func(t *testing.T) {
+ f := newUserDeleteFixture(t, "user", nil)
+ f.place(t, userDeleteSessionID)
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ f.svc.background = bg
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: userDeleteSessionID}
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", userDeleteSessionID, "sbx1")
+ require.NoError(
+ t,
+ index.Add(
+ t.Context(),
+ ShellJobIndexEntry{SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: time.Now()},
+ ),
+ )
+ require.NoError(
+ t,
+ bg.Registry.Add(
+ t.Context(),
+ ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell"},
+ time.Minute,
+ 0,
+ ),
+ )
+ unavailable := true
+ calls := 0
+ svc.remoteExec = func(
+ ctx context.Context,
+ _ string,
+ script string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ calls++
+ _, err := f.svc.sessionRepo.GetByID(ctx, 7, userDeleteSessionID)
+ require.NoError(t, err, "the session pin must still be readable when stopping")
+ require.Equal(t, "sbx1", opts.ExpectedSandboxID)
+ require.Contains(t, script, "kill -TERM")
+ if unavailable {
+ return nil, errors.New("maintenance unavailable")
+ }
+ return &sandbox.ExecuteResult{Stdout: `{"state":"cancelled"}`}, nil
+ }
+ ctx := testSessionScopeContext(7, "u-1")
+ remove := func() error {
+ switch operation {
+ case "batch":
+ return f.svc.BatchDeleteSessions(ctx, []string{userDeleteSessionID})
+ case "all":
+ return f.svc.DeleteAllSessions(ctx)
+ default:
+ return f.svc.DeleteSession(ctx, userDeleteSessionID)
+ }
+ }
+ require.ErrorContains(t, remove(), "maintenance unavailable")
+ _, err := f.svc.sessionRepo.GetByID(ctx, 7, userDeleteSessionID)
+ require.NoError(t, err)
+ live, err := bg.Registry.List(ctx, ref)
+ require.NoError(t, err)
+ require.Len(t, live, 1)
+ unavailable = false
+ require.NoError(t, remove())
+ require.Equal(t, 2, calls)
+ require.Empty(t, f.ops.destroyedIDs(), "the shared machine must remain")
+ entries, err := index.Entries(ctx)
+ require.NoError(t, err)
+ require.Empty(t, entries)
+ _, err = svc.SubmitShellJob(ctx, ref, tools.ShellJobSubmitRequest{Command: "sleep 60"})
+ require.ErrorIs(t, err, interfaces.ErrWorkNotFound)
+ })
+ }
+}
+
+func TestShellJobCancelUsesTheJobsIndexedSandbox(t *testing.T) {
+ svc, reader, index, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", "s", "original-box")
+ require.NoError(
+ t,
+ index.Add(
+ t.Context(),
+ ShellJobIndexEntry{SessionRef: ref, WorkID: "sj_0123", SandboxID: "original-box", YieldUntil: time.Now()},
+ ),
+ )
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "replacement-box", true }
+ calls := 0
+ svc.remoteExec = func(
+ _ context.Context,
+ _ string,
+ _ string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ calls++
+ require.Equal(t, "original-box", opts.ExpectedSandboxID)
+ return nil, sandbox.ErrBoundSandboxChanged
+ }
+ _, err := svc.Cancel(t.Context(), ref, []string{"sj_0123"})
+ require.ErrorIs(t, err, sandbox.ErrBoundSandboxChanged)
+ require.Equal(t, 1, calls, "must never retry the signal on the replacement sandbox")
+ require.Equal(t, "original-box", reader.calls[0].sandboxIDs["sj_0123"])
+}
+
+func TestShellJobForeignMetadataIsNotExposedOnReadFailure(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ snap := ownedRunningShellSnapshot("sj_0123", "other-session", "sbx1")
+ snap.Meta.Command = "private-command"
+ snap.Meta.Title = "private-title"
+ reader.snapshots[snap.Meta.ID] = snap
+ for _, readErr := range []error{sandbox.ErrBoundSandboxChanged, errors.New("transport unavailable")} {
+ reader.errs = map[string]error{snap.Meta.ID: readErr}
+ results, err := svc.Status(
+ t.Context(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ []string{snap.Meta.ID},
+ interfaces.StatusOptions{Detailed: true},
+ )
+ if shellSandboxGone(readErr) {
+ require.NoError(t, err)
+ } else {
+ require.Error(t, err)
+ }
+ encoded, err := json.Marshal(results)
+ require.NoError(t, err)
+ require.NotContains(t, string(encoded), "private")
+ require.NotContains(t, string(encoded), "other-session")
+ }
+}
+
+func TestShellJobBatchDeleteFailureReopensEarlierSessions(t *testing.T) {
+ f := newUserDeleteFixture(t, "user", nil)
+ f.place(t, userDeleteSessionID)
+ otherID := "1f8fad5b-d9cb-469f-a165-70867728950e"
+ f.place(t, otherID)
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ f.svc.background = bg
+ for i, sid := range []string{userDeleteSessionID, otherID} {
+ id := fmt.Sprintf("sj_%04x", i)
+ reader.snapshots[id] = ownedRunningShellSnapshot(id, sid, "sbx1")
+ require.NoError(
+ t,
+ index.Add(
+ t.Context(),
+ ShellJobIndexEntry{
+ SessionRef: interfaces.SessionRef{TenantID: 7, SessionID: sid},
+ WorkID: id,
+ SandboxID: "sbx1",
+ YieldUntil: time.Now(),
+ },
+ ),
+ )
+ }
+ calls := 0
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ calls++
+ if calls == 2 {
+ return nil, errors.New("stop failed")
+ }
+ return &sandbox.ExecuteResult{Stdout: `{"state":"cancelled"}`}, nil
+ }
+ ctx := testSessionScopeContext(7, "u-1")
+ require.ErrorContains(t, f.svc.BatchDeleteSessions(ctx, []string{userDeleteSessionID, otherID}), "stop failed")
+ require.Equal(t, 2, calls)
+ for _, sid := range []string{userDeleteSessionID, otherID} {
+ closed, err := index.SessionClosed(ctx, interfaces.SessionRef{TenantID: 7, SessionID: sid})
+ require.NoError(t, err)
+ require.False(t, closed)
+ _, err = f.svc.sessionRepo.GetByID(ctx, 7, sid)
+ require.NoError(t, err)
+ }
+}
+
+func TestShellJobUnavailableBindingDoesNotConfirmLoss(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "", false }
+ results, err := svc.Status(
+ t.Context(),
+ interfaces.SessionRef{TenantID: 7, SessionID: "s"},
+ []string{"sj_0123"},
+ interfaces.StatusOptions{Detailed: true},
+ )
+ require.ErrorContains(t, err, "no bound sandbox")
+ require.Empty(t, results)
+}
diff --git a/internal/application/service/shell_job_service.go b/internal/application/service/shell_job_service.go
new file mode 100644
index 000000000..87ae2cc6e
--- /dev/null
+++ b/internal/application/service/shell_job_service.go
@@ -0,0 +1,1896 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+const (
+ shellJobYieldDefault = 10 * time.Second
+ shellJobYieldFloor = 250 * time.Millisecond
+ shellJobYieldMax = 30 * time.Second
+ shellJobKindCap = 64
+ // shellJobRenewLease outlives several 5s sweeper ticks so one missed round
+ // does not drop rewind / rebuild protection.
+ shellJobRenewLease = 30 * time.Second
+ shellJobHandoff = 5 * time.Second
+ shellJobIDAttempts = 8
+)
+
+const (
+ shellJobNoteRunning = "Still running. Keep working on other parts of the task; " +
+ "you will be told when it finishes. " +
+ "Call work_wait only when you cannot continue without its output. work_cancel stops it. " +
+ "It shares the workspace with your other tools: do not edit files it is writing. " +
+ "Cancel it once you no longer need it. " +
+ "Do not pipe the command through tail or head: the job log stays empty until that pipe closes."
+ shellJobNoteRunningLite = " It runs in the user's project directory and stops if WeKnora Lite is closed. " +
+ "Other chats on the same project can still write to this directory."
+ shellJobNoteLost = "The sandbox was replaced or the job's watcher died, so the result is unknown. " +
+ "Re-run it, or use shell_exec if it is short."
+ shellJobNoteLostLite = "WeKnora Lite was closed while it was running."
+)
+
+// LocalShellJobs is the Lite host implementation contract. The remote service
+// delegates to it for host sessions.
+type LocalShellJobs interface {
+ SubmitShellJob(
+ context.Context, interfaces.SessionRef, tools.ShellJobSubmitRequest,
+ ) (tools.ShellJobSubmission, error)
+ Status(context.Context, interfaces.SessionRef, []string, interfaces.StatusOptions) ([]interfaces.WorkResult, error)
+ Cancel(context.Context, interfaces.SessionRef, []string) ([]interfaces.WorkResult, error)
+ OnDelete(context.Context, interfaces.SessionRef) error
+ Shutdown(context.Context) error
+ EndShellJobRun(interfaces.SessionRef, string)
+}
+
+// ShellJobPayload is the model- and UI-facing body of one shell job result.
+type ShellJobPayload struct {
+ DisplayType string `json:"display_type"`
+ WorkID string `json:"work_id"`
+ State string `json:"state"`
+ SessionID string `json:"session_id"`
+ Command string `json:"command"`
+ WorkDir string `json:"work_dir"`
+ ExitCode *int `json:"exit_code,omitempty"`
+ DurationMS int64 `json:"duration_ms"`
+ StartedAt time.Time `json:"started_at"`
+ Stdout string `json:"stdout"`
+ Stderr string `json:"stderr"`
+ FromBytes sandbox.ShellJobOffsets `json:"from_bytes"`
+ TotalBytes sandbox.ShellJobOffsets `json:"total_bytes"`
+ OmittedBytes sandbox.ShellJobOffsets `json:"omitted_bytes"`
+ Truncated bool `json:"truncated"`
+ Note string `json:"note,omitempty"`
+ OutputFiles []string `json:"output_files,omitempty"`
+}
+
+// ShellJobService runs remote shell jobs: submit, status, cancel, and the
+// per-replica sweeper.
+type ShellJobService struct {
+ pinned *PinnedSessionSandbox
+ bg *BackgroundWork
+ index *ShellJobIndex
+ events *BackgroundEventEmitter
+ local LocalShellJobs
+ // keeper is the same value stored on lifecycle.KeepAlive. E2B exec does
+ // not extend the provider timer, so submit and the sweeper both need it.
+ // A nil keeper fails SubmitShellJob before anything is registered.
+ keeper interfaces.SessionKeepAliver
+ lifecycle background.Lifecycle
+ // remoteRead, remoteExec and boundSandbox are the seams tests replace.
+ // Production installs them from pinned. The command is a separate argument
+ // of remoteExec; ShellExecOptions has no Command field.
+ newJobID func() (string, error)
+ heartbeatInterval time.Duration
+ remoteRead remoteReadFunc
+ remoteExec remoteExecFunc
+ boundSandbox boundSandboxFunc
+ // listFiles overrides output-directory listing in tests.
+ listFiles func(context.Context, string, string) ([]sandbox.RemoteDirEntry, error)
+ skillCapture skillEnvCaptures
+ replica string
+ swept sync.Mutex
+ mu sync.Mutex
+ cursors map[string]shellJobCursor // key: tenant\0session\0run\0work
+}
+
+type shellJobCursor struct {
+ from sandbox.ShellJobOffsets
+ expires time.Time
+}
+
+// remoteRead returns one snapshot per id, in order. errs carries per-id
+// failures. Preserve typed errors so callers can distinguish confirmed loss
+// from a transport failure without acknowledging an unread completion.
+type remoteReadFunc func(ctx context.Context, s interfaces.SessionRef,
+ ids []string, sandboxIDs map[string]string) ([]sandbox.ShellJobSnapshot, map[string]error)
+
+type remoteExecFunc func(ctx context.Context, sessionID string, command string,
+ opts sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error)
+
+type boundSandboxFunc func(ctx context.Context, sessionID string) (string, bool)
+
+// shellJobLease covers the yield window plus the 10s exec slack and a few
+// seconds for the sweeper to take over. Begin's own keep-alive runs only
+// after start returns, so the lease has to outlive that wait.
+func shellJobLease(yield time.Duration) time.Duration {
+ return yield + 15*time.Second
+}
+
+var (
+ _ interfaces.WorkHandler = (*ShellJobService)(nil)
+ _ interfaces.SessionLifecycleListener = (*ShellJobService)(nil)
+)
+
+// NewShellJobService wires the remote shell-job service. keeper is the only
+// way a keep-aliver enters: BackgroundWork has no KeepAliver field. events
+// is stored on the lifecycle only when the pointer is non-nil, so a typed
+// nil never sits in the interface.
+func NewShellJobService(pinned *PinnedSessionSandbox, bg *BackgroundWork,
+ index *ShellJobIndex, events *BackgroundEventEmitter,
+ keeper interfaces.SessionKeepAliver,
+) *ShellJobService {
+ svc := &ShellJobService{
+ pinned: pinned,
+ bg: bg,
+ index: index,
+ events: events,
+ keeper: keeper,
+ replica: fmt.Sprintf("%s-%d", hostname(), osGetpid()),
+ cursors: make(map[string]shellJobCursor),
+ newJobID: sandbox.NewShellJobID,
+ heartbeatInterval: 5 * time.Second,
+ }
+ if bg != nil {
+ svc.lifecycle = background.Lifecycle{
+ Registry: bg.Registry,
+ Inbox: bg.Inbox,
+ KeepAlive: keeper,
+ Logf: logger.Warnf,
+ }
+ if events != nil {
+ svc.lifecycle.Events = events
+ }
+ if bg.Listeners != nil {
+ bg.Listeners.Register(svc)
+ }
+ }
+ if pinned != nil {
+ svc.remoteRead = svc.batchReadViaPinned
+ svc.remoteExec = func(ctx context.Context, sessionID, command string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ return pinned.ExecShellCommandWithOptions(ctx, sessionID, command, opts)
+ }
+ svc.boundSandbox = pinned.BoundSandboxID
+ }
+ return svc
+}
+
+// WithLocal installs the host implementation used for Lite sessions.
+func (s *ShellJobService) WithLocal(local LocalShellJobs) *ShellJobService {
+ if s == nil {
+ return nil
+ }
+ s.local = local
+ return s
+}
+
+// RemoteAvailable reports whether a remote submit can run. An injected
+// remoteExec counts: tests construct the service with a nil pin and then
+// install the seam.
+func (s *ShellJobService) RemoteAvailable() bool {
+ return s != nil && s.index != nil && (s.pinned != nil || s.remoteExec != nil)
+}
+
+// LocalAvailable reports whether a host implementation is installed.
+func (s *ShellJobService) LocalAvailable() bool { return s != nil && s.local != nil }
+
+// SubmitShellJob starts a remote shell job, or delegates to the host
+// implementation when this session lives on the user's machine.
+func (s *ShellJobService) SubmitShellJob(ctx context.Context, ref interfaces.SessionRef,
+ req tools.ShellJobSubmitRequest,
+) (tools.ShellJobSubmission, error) {
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ return s.local.SubmitShellJob(ctx, ref, req)
+ }
+ if !s.RemoteAvailable() {
+ return tools.ShellJobSubmission{}, fmt.Errorf("remote shell jobs are unavailable")
+ }
+ var result tools.ShellJobSubmission
+ err := s.index.WithSessionLock(ctx, ref, func(lockCtx context.Context) error {
+ closed, err := s.index.SessionClosed(lockCtx, ref)
+ if err != nil {
+ return err
+ }
+ if closed {
+ return interfaces.ErrWorkNotFound
+ }
+ submitCtx, cancel := context.WithCancel(ctx)
+ stop := context.AfterFunc(lockCtx, cancel)
+ defer stop()
+ defer cancel()
+ result, err = s.submitRemoteShellJob(submitCtx, ref, req)
+ return err
+ })
+ return result, err
+}
+
+func (s *ShellJobService) submitRemoteShellJob(ctx context.Context, ref interfaces.SessionRef,
+ req tools.ShellJobSubmitRequest,
+) (tools.ShellJobSubmission, error) {
+ // E2B exec does not extend the provider timer. Missing wiring has to fail
+ // here, before Begin, or the sandbox pauses and the job shows up later as lost.
+ if s.keeper == nil {
+ return tools.ShellJobSubmission{}, errors.New("shell job service has no keep-aliver wired")
+ }
+ yield := clampYieldTime(req.YieldTime)
+ now := time.Now().UTC()
+ bindCtx, bindSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.ensure_bound_sandbox",
+ Input: map[string]interface{}{"session_id": ref.SessionID},
+ })
+ sandboxID, err := s.ensureBoundSandbox(bindCtx, ref)
+ bindSpan.Finish(map[string]interface{}{"sandbox_id": sandboxID}, nil, err)
+ if err != nil {
+ return tools.ShellJobSubmission{}, err
+ }
+ docker := false
+ if s.pinned != nil {
+ dockerCtx, dockerSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.backend_check",
+ Input: map[string]interface{}{"session_id": ref.SessionID},
+ })
+ docker = s.pinned.BackendIsDocker(maintenanceCtx(dockerCtx, ref), ref.SessionID)
+ dockerSpan.Finish(map[string]interface{}{"docker": docker}, nil, nil)
+ }
+ var (
+ jobID string
+ meta sandbox.ShellJobMeta
+ activity interfaces.BackgroundActivity
+ preview shellJobPreview
+ forwarded sandbox.ShellJobOffsets
+ finished *tools.ShellJobSubmission
+ degraded bool
+ // unparsed records that the submit exec ended without its epilogue: the
+ // marker line never reached the stream, so the job's state at the end of
+ // the yield window is unknown.
+ unparsed bool
+ )
+ for range shellJobIDAttempts {
+ preview = shellJobPreview{}
+ forwarded = sandbox.ShellJobOffsets{}
+ finished = nil
+ degraded = false
+ unparsed = false
+ jobID, err = s.newJobID()
+ if err != nil {
+ return tools.ShellJobSubmission{}, err
+ }
+ reserved, reserveErr := s.index.ReserveID(ctx, ref, jobID)
+ if reserveErr != nil {
+ return tools.ShellJobSubmission{}, reserveErr
+ }
+ if !reserved {
+ err = errShellJobIDTaken
+ continue
+ }
+ meta = sandbox.ShellJobMeta{
+ ID: jobID, TenantID: ref.TenantID, SandboxTenantID: ref.TenantID,
+ SessionID: ref.SessionID, Command: req.MaskedCommand(),
+ WorkDir: req.WorkDir, SandboxID: sandboxID, CreatedAt: now,
+ YieldUntil: now.Add(yield), ParentMessageID: req.ParentMessageID,
+ ToolCallID: req.ToolCallID, Title: req.Description,
+ OutputDir: req.OutputDir, OutputSeen: req.OutputSeen, OutputBefore: req.OutputBefore,
+ }
+ script, marker := sandbox.ShellJobSubmitScriptWithBackend(meta, req.Command, int(yield.Milliseconds()), docker)
+ activity = interfaces.BackgroundActivity{
+ ID: jobID, Kind: "shell", Title: req.Description, StartedAt: now,
+ }
+ stream := sandbox.NewShellJobMarkerStream(marker)
+ beginCtx, beginSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.lifecycle_begin",
+ Input: map[string]interface{}{
+ "work_id": jobID, "yield_ms": yield.Milliseconds(),
+ },
+ })
+ err = s.lifecycle.Begin(beginCtx, ref, activity, shellJobLease(yield), shellJobKindCap,
+ func(ctx context.Context) error {
+ idxCtx, idxSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.index_add",
+ Input: map[string]interface{}{"work_id": jobID, "sandbox_id": sandboxID},
+ })
+ err := s.index.Add(idxCtx, ShellJobIndexEntry{
+ SessionRef: ref, WorkID: jobID, SandboxID: sandboxID, YieldUntil: meta.YieldUntil,
+ })
+ idxSpan.Finish(nil, nil, err)
+ if err != nil {
+ return err
+ }
+ // Begin's own keep-alive runs only after start returns. start blocks
+ // for the whole yield window, so the sandbox has to be refreshed now.
+ keepCtx, keepSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.keep_alive",
+ Input: map[string]interface{}{"session_id": ref.SessionID},
+ })
+ err = s.keeper.KeepAlive(keepCtx, ref)
+ keepSpan.Finish(nil, nil, err)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] keep-alive at submit id=%s: %v", jobID, err)
+ }
+ execCtx, execSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.submit_exec",
+ Input: map[string]interface{}{
+ "work_id": jobID, "yield_ms": yield.Milliseconds(),
+ "command": langfuse.TruncateRunes(script, 256),
+ },
+ })
+ var sawStdout, sawStderr atomic.Bool
+ execRes, err := s.remoteExec(execCtx, ref.SessionID, script, sandbox.ShellExecOptions{
+ // No WorkDir: shared machines admit only the session layout
+ // root as cwd, and the manager's empty default is exactly
+ // that root. The script uses absolute paths, so it does not
+ // care where it starts.
+ ExpectedSandboxID: sandboxID,
+ Timeout: yield + 10*time.Second,
+ Env: req.Env,
+ OnOutput: func(streamName string, chunk []byte) {
+ // stderr is never part of the epilogue. A line that
+ // looks like "1 2" would otherwise replace the offsets.
+ if streamName == "stderr" {
+ sawStderr.Store(true)
+ if len(chunk) == 0 {
+ return
+ }
+ preview.add("stderr", chunk)
+ if req.OnOutput != nil {
+ req.OnOutput("stderr", chunk)
+ }
+ return
+ }
+ sawStdout.Store(true)
+ body, _ := stream.Feed(chunk)
+ if len(body) == 0 {
+ return
+ }
+ preview.add("stdout", body)
+ if req.OnOutput != nil {
+ req.OnOutput("stdout", body)
+ }
+ },
+ })
+ execOut := map[string]interface{}{}
+ if execRes != nil {
+ execOut["exit_code"] = execRes.ExitCode
+ execOut["killed"] = execRes.Killed
+ }
+ execSpan.Finish(execOut, nil, err)
+ // A Go error means the wrapper never started (lookup, prepare, or
+ // an early cancel). Roll the index member back so Begin can drop
+ // the registration without leaving a sweeper ghost.
+ if err != nil {
+ if _, rmErr := s.index.ClaimWork(context.WithoutCancel(ctx), ref, jobID); rmErr != nil {
+ logger.Warnf(ctx, "[ShellJob] rollback index after pre-launch failure id=%s: %v", jobID, rmErr)
+ }
+ return err
+ }
+ if execRes != nil && execRes.ExitCode == sandbox.ShellJobDirExistsExit {
+ if _, rmErr := s.index.ClaimWork(context.WithoutCancel(ctx), ref, jobID); rmErr != nil {
+ logger.Warnf(ctx, "[ShellJob] rollback index after id collision id=%s: %v", jobID, rmErr)
+ }
+ return errShellJobIDTaken
+ }
+ // A nil Go error with Error or Killed means the exec was cut after
+ // the wrapper may already be running. Keep the registration and the
+ // index entry. Do not read status: the caller context may already
+ // be canceled, and a miss would look like lost.
+ if execRes != nil && (execRes.Error != "" || execRes.Killed) {
+ logger.Warnf(ctx, "[ShellJob] submit exec degraded id=%s: %s", jobID, execRes.Error)
+ degraded = true
+ return nil
+ }
+ // Cube buffers the whole command and does not call OnOutput.
+ // Feed that buffer once. Docker and E2B already called it, so
+ // feeding the same bytes again would duplicate the epilogue.
+ if !sawStdout.Load() && execRes != nil && execRes.Stdout != "" {
+ body, _ := stream.Feed([]byte(execRes.Stdout))
+ if len(body) > 0 {
+ preview.add("stdout", body)
+ if req.OnOutput != nil {
+ req.OnOutput("stdout", body)
+ }
+ }
+ }
+ if !sawStderr.Load() && execRes != nil && execRes.Stderr != "" {
+ chunk := []byte(execRes.Stderr)
+ preview.add("stderr", chunk)
+ if req.OnOutput != nil {
+ req.OnOutput("stderr", chunk)
+ }
+ }
+ _, epilogue := stream.Final()
+ status, offsets, parsed := sandbox.ParseShellJobEpilogue(epilogue)
+ if !parsed {
+ // The marker or the final status was lost. Re-check the state
+ // before registering: a job that already finished must be
+ // answered here, or the sweeper reports it a second time as a
+ // completion notice after the model already has its output.
+ unparsed = true
+ logger.Warnf(ctx, "[ShellJob] submit epilogue missing id=%s", jobID)
+ return nil
+ }
+ forwarded = offsets
+ if status.State != interfaces.WorkStateRunning {
+ sub := tools.ShellJobSubmission{
+ Result: shellTerminalResult(meta, status, offsets, &preview, false),
+ Background: false,
+ }
+ finished = &sub
+ }
+ return nil
+ })
+ beginSpan.Finish(map[string]interface{}{
+ "finished": finished != nil, "degraded": degraded, "unparsed": unparsed,
+ }, nil, err)
+ if errors.Is(err, errShellJobIDTaken) {
+ continue
+ }
+ break
+ }
+ if err != nil {
+ return tools.ShellJobSubmission{}, err
+ }
+ if finished != nil {
+ if forwarded.Stdout > preview.stdout.total || forwarded.Stderr > preview.stderr.total {
+ results, readErr := s.Status(ctx, ref, []string{jobID}, interfaces.StatusOptions{Detailed: true})
+ if readErr == nil && len(results) == 1 && results[0].Terminal &&
+ results[0].State != interfaces.WorkStateLost {
+ finished.Result = results[0]
+ }
+ }
+ finCtx, finSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.finish_cleanup",
+ Input: map[string]interface{}{"work_id": jobID},
+ })
+ removed, rmErr := s.index.ClaimWork(context.WithoutCancel(finCtx), ref, jobID)
+ if rmErr != nil {
+ logger.Warnf(ctx, "[ShellJob] drop finished index id=%s: %v", jobID, rmErr)
+ } else if !removed {
+ // The sweeper won the claim, so it may already have queued a notice
+ // for a job whose result this call is also returning inline.
+ logger.Warnf(ctx, "[ShellJob] finished job %s was already claimed", jobID)
+ }
+ if dErr := s.lifecycle.Discard(context.WithoutCancel(finCtx), ref, jobID); dErr != nil {
+ logger.Warnf(ctx, "[ShellJob] discard id=%s: %v", jobID, dErr)
+ }
+ finished.Result = s.applyOutputFiles(finCtx, ref, finished.Result, meta)
+ finSpan.Finish(map[string]interface{}{"removed": removed}, nil, rmErr)
+ return *finished, nil
+ }
+ // The epilogue never arrived, so we do not know whether the job finished
+ // inside the yield window. Registering a finished job as background makes
+ // the sweeper deliver a completion notice for a result the model already
+ // has; check the state once before committing to that path.
+ if unparsed && !degraded {
+ if submission, ok := s.finishedBeforeRegistering(ctx, ref, jobID, meta, &preview); ok {
+ return submission, nil
+ }
+ }
+ s.noteRunning(ctx, ref, req, activity, jobID, forwarded)
+ if degraded {
+ return runningSubmission(meta, &preview), nil
+ }
+ statusCtx, statusSpan := langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
+ Name: "shelljob.status",
+ Input: map[string]interface{}{"work_id": jobID, "detailed": true},
+ })
+ first, err := s.Status(statusCtx, ref, []string{jobID}, interfaces.StatusOptions{
+ Detailed: true, RunID: req.RunID,
+ })
+ statusSpan.Finish(map[string]interface{}{"results": len(first)}, nil, err)
+ if err != nil {
+ return runningSubmission(meta, &preview), nil
+ }
+ if len(first) == 0 {
+ return tools.ShellJobSubmission{}, fmt.Errorf("shell job %s has no status", jobID)
+ }
+ first[0] = fillShellJobPreview(first[0], &preview)
+ return tools.ShellJobSubmission{Result: first[0], Background: true}, nil
+}
+
+func (s *ShellJobService) noteRunning(ctx context.Context, ref interfaces.SessionRef,
+ req tools.ShellJobSubmitRequest, activity interfaces.BackgroundActivity,
+ jobID string, from sandbox.ShellJobOffsets,
+) {
+ if s.events != nil {
+ if err := s.events.EmitWorkStarted(ctx, ref, req.ParentMessageID, activity, req.ToolCallID); err != nil {
+ logger.Warnf(ctx, "[ShellJob] work_started id=%s: %v", jobID, err)
+ }
+ }
+ s.mu.Lock()
+ s.setCursorsLocked(ref, req.RunID, jobID, from)
+ s.mu.Unlock()
+}
+
+func runningSubmission(meta sandbox.ShellJobMeta, preview *shellJobPreview) tools.ShellJobSubmission {
+ snap := sandbox.ShellJobSnapshot{
+ Meta: meta,
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, StartedAt: meta.CreatedAt,
+ },
+ }
+ if preview != nil {
+ snap.Stdout = preview.stdout.stream(preview.stdout.total)
+ snap.Stderr = preview.stderr.stream(preview.stderr.total)
+ }
+ return tools.ShellJobSubmission{Result: ShellJobResult(snap, false), Background: true}
+}
+
+// finishedBeforeRegistering answers a job whose submit epilogue was lost but
+// which has already reached a terminal state. It drops the index member and the
+// activity registration so the sweeper cannot report the same job again as a
+// completion notice for a result this call is returning inline.
+//
+// The read deliberately carries no RunID: this is not a run, and it must not
+// move the incremental cursor a later work_wait relies on.
+func (s *ShellJobService) finishedBeforeRegistering(
+ ctx context.Context, ref interfaces.SessionRef, jobID string,
+ meta sandbox.ShellJobMeta, preview *shellJobPreview,
+) (tools.ShellJobSubmission, bool) {
+ results, err := s.Status(ctx, ref, []string{jobID}, interfaces.StatusOptions{Detailed: true})
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] check state before registering id=%s: %v", jobID, err)
+ return tools.ShellJobSubmission{}, false
+ }
+ if len(results) != 1 || !interfaces.IsTerminalWorkState(results[0].State) {
+ return tools.ShellJobSubmission{}, false
+ }
+ // Only a readable terminal record can produce an inline result. Leave an
+ // absent record to the sweeper, which also owns process cleanup.
+ if results[0].State == interfaces.WorkStateLost {
+ return tools.ShellJobSubmission{}, false
+ }
+ result := fillShellJobPreview(results[0], preview)
+ if removed, rmErr := s.index.ClaimWork(context.WithoutCancel(ctx), ref, jobID); rmErr != nil {
+ logger.Warnf(ctx, "[ShellJob] drop finished index id=%s: %v", jobID, rmErr)
+ } else if !removed {
+ logger.Warnf(ctx, "[ShellJob] finished job %s was already claimed", jobID)
+ }
+ if dErr := s.lifecycle.Discard(context.WithoutCancel(ctx), ref, jobID); dErr != nil {
+ logger.Warnf(ctx, "[ShellJob] discard id=%s: %v", jobID, dErr)
+ }
+ result = s.applyOutputFiles(ctx, ref, result, meta)
+ return tools.ShellJobSubmission{Result: result, Background: false}, true
+}
+
+// shellJobPreview retains a bounded head and tail of the bytes forwarded
+// during the yield window, keeping short jobs to a single exec.
+type shellJobPreview struct {
+ stdout shellPreviewStream
+ stderr shellPreviewStream
+}
+
+type shellPreviewStream struct {
+ head []byte
+ tail []byte
+ total int64
+}
+
+func (p *shellPreviewStream) add(body []byte, budget int) {
+ p.total += int64(len(body))
+ headN := min(len(body), budget/4-len(p.head))
+ p.head = append(p.head, body[:headN]...)
+ p.tail = append(p.tail, body[headN:]...)
+ if len(p.tail) > budget-budget/4 {
+ p.tail = append([]byte(nil), p.tail[len(p.tail)-(budget-budget/4):]...)
+ }
+}
+
+func (p *shellPreviewStream) String() string {
+ return strings.ToValidUTF8(string(p.head)+string(p.tail), "")
+}
+
+func (p *shellPreviewStream) stream(total int64) sandbox.ShellJobStream {
+ omitted := max(0, total-int64(len(p.head)+len(p.tail)))
+ return sandbox.ShellJobStream{
+ Text: p.String(), TotalBytes: total,
+ OmittedBytes: omitted, Truncated: omitted > 0,
+ }
+}
+
+func (p *shellJobPreview) add(stream string, body []byte) {
+ if stream == "stderr" {
+ p.stderr.add(body, 8*1024)
+ return
+ }
+ p.stdout.add(body, 16*1024)
+}
+
+func fillShellJobPreview(result interfaces.WorkResult, preview *shellJobPreview) interfaces.WorkResult {
+ if preview == nil {
+ return result
+ }
+ payload, ok := result.Payload.(ShellJobPayload)
+ if !ok {
+ return result
+ }
+ if payload.Stdout == "" {
+ payload.Stdout = preview.stdout.String()
+ }
+ if payload.Stderr == "" {
+ payload.Stderr = preview.stderr.String()
+ }
+ result.Payload = payload
+ return result
+}
+
+// Status returns one result per id. Detailed=false never execs.
+func (s *ShellJobService) Status(ctx context.Context, ref interfaces.SessionRef,
+ ids []string, opts interfaces.StatusOptions,
+) ([]interfaces.WorkResult, error) {
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ return s.local.Status(ctx, ref, ids, opts)
+ }
+ if !s.RemoteAvailable() {
+ return nil, interfaces.ErrWorkNotFound
+ }
+ if err := validateShellJobIDs(ids); err != nil {
+ return nil, err
+ }
+ if !opts.Detailed {
+ return s.cheapStatus(ctx, ref, ids)
+ }
+ if s.remoteRead == nil {
+ return nil, fmt.Errorf("remote shell jobs are unavailable")
+ }
+ readCtx := context.WithValue(maintenanceCtx(ctx, ref), shellReadRunKey{}, opts.RunID)
+ readCtx = context.WithValue(readCtx, shellReadBudgetKey{}, opts.OutputBudget)
+ snapshots, errs := s.remoteRead(readCtx, ref, ids, s.sandboxFor(ctx, ref, ids))
+ if len(snapshots) != len(ids) {
+ return nil, fmt.Errorf("shell job batch read returned %d of %d", len(snapshots), len(ids))
+ }
+ for i, snap := range snapshots {
+ if err := errs[ids[i]]; err != nil && !shellSandboxGone(err) {
+ return nil, fmt.Errorf("read shell job %s: %w", ids[i], err)
+ }
+ if errs[ids[i]] == nil && !snap.Missing && !belongsToOtherSession(snap.Meta, ref) &&
+ snap.Status.State != interfaces.WorkStateRunning && !interfaces.IsTerminalWorkState(snap.Status.State) {
+ return nil, fmt.Errorf("read shell job %s: unreadable status", ids[i])
+ }
+ }
+ out := make([]interfaces.WorkResult, len(ids))
+ for i, snap := range snapshots {
+ if errs != nil && errs[ids[i]] != nil {
+ out[i] = lostResult(sandbox.ShellJobMeta{ID: ids[i]}, errs[ids[i]].Error())
+ continue
+ }
+ if snap.Missing || belongsToOtherSession(snap.Meta, ref) {
+ out[i] = lostResult(sandbox.ShellJobMeta{ID: ids[i]}, "job directory missing or belongs to another session")
+ continue
+ }
+ out[i] = s.applyOutputFiles(ctx, ref, ShellJobResult(snap, false), snap.Meta)
+ if out[i].ID == "" {
+ out[i].ID = ids[i]
+ }
+ s.noteReadCursor(ref, opts.RunID, ids[i], snap)
+ }
+ return out, nil
+}
+
+// cheapStatus is the work_wait poll source: one result per id, unknown when
+// neither the registry nor the inbox can decide. A bad id is the only whole-call error.
+func (s *ShellJobService) cheapStatus(ctx context.Context, ref interfaces.SessionRef,
+ ids []string,
+) ([]interfaces.WorkResult, error) {
+ if err := validateShellJobIDs(ids); err != nil {
+ return nil, err
+ }
+ if s.bg == nil || s.bg.Registry == nil || s.bg.Inbox == nil {
+ return nil, interfaces.ErrWorkNotFound
+ }
+ live, err := s.bg.Registry.List(ctx, ref)
+ if err != nil {
+ return nil, err
+ }
+ liveSet := make(map[string]interfaces.BackgroundActivity, len(live))
+ for _, a := range live {
+ liveSet[a.ID] = a
+ }
+ notices, err := s.bg.Inbox.Pending(ctx, ref)
+ if err != nil {
+ return nil, err
+ }
+ noticeState := make(map[string]string, len(notices))
+ for _, n := range notices {
+ if state := ParseShellJobNoticeState(n.Text); state != "" {
+ noticeState[n.ID] = state
+ }
+ }
+ out := make([]interfaces.WorkResult, len(ids))
+ for i, id := range ids {
+ switch {
+ case liveSet[id].ID == id:
+ a := liveSet[id]
+ out[i] = interfaces.WorkResult{
+ ID: id, State: interfaces.WorkStateRunning, Title: a.Title, Terminal: false,
+ }
+ case noticeState[id] != "":
+ out[i] = interfaces.WorkResult{
+ ID: id, State: noticeState[id], Terminal: interfaces.IsTerminalWorkState(noticeState[id]),
+ }
+ default:
+ out[i] = interfaces.WorkResult{ID: id, State: interfaces.WorkStateUnknown}
+ }
+ }
+ return out, nil
+}
+
+// ParseShellJobNoticeState reads the terminal state out of a shell-job notice.
+// The text is "Background shell <id> <state>: <title>. …".
+func ParseShellJobNoticeState(text string) string {
+ const prefix = "Background shell "
+ if !strings.HasPrefix(text, prefix) {
+ return ""
+ }
+ rest := strings.TrimPrefix(text, prefix)
+ space := strings.IndexByte(rest, ' ')
+ if space <= 0 || space+1 >= len(rest) {
+ return ""
+ }
+ stateAndMore := rest[space+1:]
+ colon := strings.IndexByte(stateAndMore, ':')
+ if colon <= 0 {
+ return ""
+ }
+ state := stateAndMore[:colon]
+ switch state {
+ case interfaces.WorkStateRunning, interfaces.WorkStateSucceeded, interfaces.WorkStateFailed,
+ interfaces.WorkStateCancelled, interfaces.WorkStateLost:
+ return state
+ default:
+ return ""
+ }
+}
+
+// Cancel asks the watcher of each id to stop. The returned state is whatever
+// status.json said at that moment; the durable terminal state arrives as a notice.
+func (s *ShellJobService) Cancel(ctx context.Context, ref interfaces.SessionRef,
+ ids []string,
+) ([]interfaces.WorkResult, error) {
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ return s.local.Cancel(ctx, ref, ids)
+ }
+ if !s.RemoteAvailable() {
+ return nil, interfaces.ErrWorkNotFound
+ }
+ if err := validateShellJobIDs(ids); err != nil {
+ return nil, err
+ }
+ sandboxIDs := s.sandboxFor(ctx, ref, ids)
+ out := make([]interfaces.WorkResult, len(ids))
+ for i, id := range ids {
+ result, err := s.cancelRemoteJob(ctx, ref, id, sandboxIDs[id], false)
+ if err != nil {
+ return nil, err
+ }
+ out[i] = result
+ }
+ return out, nil
+}
+
+func (s *ShellJobService) cancelRemoteJob(ctx context.Context, ref interfaces.SessionRef,
+ id, sandboxID string, force bool,
+) (interfaces.WorkResult, error) {
+ if sandboxID == "" {
+ return interfaces.WorkResult{}, errors.New("session has no readable bound sandbox")
+ }
+ if s.remoteRead == nil || s.remoteExec == nil {
+ return interfaces.WorkResult{}, fmt.Errorf("remote shell jobs are unavailable")
+ }
+ snapshots, errs := s.remoteRead(maintenanceCtx(ctx, ref), ref, []string{id}, map[string]string{id: sandboxID})
+ if err := errs[id]; err != nil {
+ return interfaces.WorkResult{}, err
+ }
+ if len(snapshots) != 1 {
+ return interfaces.WorkResult{}, fmt.Errorf("unreadable cancellation status for %s", id)
+ }
+ snap := snapshots[0]
+ if snap.Missing || snap.Meta.ID != id || snap.Meta.TenantID != ref.TenantID ||
+ snap.Meta.SessionID != ref.SessionID || snap.Meta.SandboxID != sandboxID {
+ return interfaces.WorkResult{}, interfaces.ErrWorkNotFound
+ }
+ if interfaces.IsTerminalWorkState(snap.Status.State) {
+ return interfaces.WorkResult{ID: id, State: snap.Status.State, Terminal: true}, nil
+ }
+ if snap.Status.State != interfaces.WorkStateRunning {
+ return interfaces.WorkResult{}, fmt.Errorf("unreadable cancellation status for %s", id)
+ }
+ res, err := s.remoteExec(maintenanceCtx(ctx, ref), ref.SessionID,
+ sandbox.ShellJobCancelScript(snap.Meta, force), sandbox.ShellExecOptions{
+ ExpectedSandboxID: sandboxID, Timeout: 10 * time.Second, SkipWorkspacePrep: true,
+ })
+ if err != nil {
+ return interfaces.WorkResult{}, err
+ }
+ if res != nil && res.ExitCode == 3 {
+ return interfaces.WorkResult{}, interfaces.ErrWorkNotFound
+ }
+ if res == nil || res.ExitCode != 0 || res.Error != "" || res.Killed {
+ return interfaces.WorkResult{}, fmt.Errorf("cancel exec failed for %s; retry cancellation", id)
+ }
+ var status sandbox.ShellJobStatus
+ if json.Unmarshal([]byte(strings.TrimSpace(res.Stdout)), &status) != nil ||
+ (status.State != interfaces.WorkStateRunning && !interfaces.IsTerminalWorkState(status.State)) {
+ return interfaces.WorkResult{}, fmt.Errorf("cancel status unreadable for %s; retry cancellation", id)
+ }
+ return interfaces.WorkResult{
+ ID: id, State: status.State, Title: snap.Meta.Title,
+ Terminal: interfaces.IsTerminalWorkState(status.State),
+ }, nil
+}
+
+// sandboxFor is the expected sandbox of each id. The index member wins.
+// Anything the index does not know falls back to the current binding, which
+// is lookup-only. The map is never nil; an empty string means "do not exec".
+func (s *ShellJobService) sandboxFor(ctx context.Context, ref interfaces.SessionRef,
+ ids []string,
+) map[string]string {
+ out := make(map[string]string, len(ids))
+ indexed := map[string]string{}
+ if s.index != nil {
+ indexed = s.index.SandboxIDs(ctx, ref, ids)
+ }
+ fallback := ""
+ looked := false
+ for _, id := range ids {
+ if sbx := indexed[id]; sbx != "" {
+ out[id] = sbx
+ continue
+ }
+ if !looked {
+ looked = true
+ if s.boundSandbox != nil {
+ if current, ok := s.boundSandbox(maintenanceCtx(ctx, ref), ref.SessionID); ok {
+ fallback = current
+ }
+ }
+ }
+ out[id] = fallback
+ }
+ return out
+}
+
+// batchReadViaPinned groups ids by sandbox and runs one maintenance exec per
+// group. Every exec sets ExpectedSandboxID and SkipWorkspacePrep. An empty
+// sandbox id is a per-id loss and does not exec. A Go error marks only that
+// group's ids.
+func (s *ShellJobService) batchReadViaPinned(ctx context.Context, ref interfaces.SessionRef,
+ ids []string, sandboxIDs map[string]string,
+) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ groups := make(map[string][]string)
+ for _, id := range ids {
+ sbx := ""
+ if sandboxIDs != nil {
+ sbx = sandboxIDs[id]
+ }
+ groups[sbx] = append(groups[sbx], id)
+ }
+ snapshots := make([]sandbox.ShellJobSnapshot, len(ids))
+ byID := make(map[string]int, len(ids))
+ for i, id := range ids {
+ byID[id] = i
+ snapshots[i] = sandbox.ShellJobSnapshot{Missing: true, Meta: sandbox.ShellJobMeta{ID: id}}
+ }
+ errs := make(map[string]error)
+ for sandboxID, groupIDs := range groups {
+ if sandboxID == "" {
+ for _, id := range groupIDs {
+ // The lookup-only binding API can also return false on a storage
+ // outage. Only a pinned maintenance exec can confirm replacement.
+ errs[id] = errors.New("session has no bound sandbox")
+ }
+ continue
+ }
+ if s.remoteExec == nil {
+ for _, id := range groupIDs {
+ errs[id] = errors.New("remote shell jobs are unavailable")
+ }
+ continue
+ }
+ budget, _ := ctx.Value(shellReadBudgetKey{}).(int)
+ script := sandbox.ShellJobBatchReadScriptWithBudget(
+ sandboxID, groupIDs, s.cursorsFor(ctx, ref, groupIDs), budget,
+ )
+ res, err := s.remoteExec(ctx, ref.SessionID, script, sandbox.ShellExecOptions{
+ ExpectedSandboxID: sandboxID,
+ Timeout: 30 * time.Second,
+ SkipWorkspacePrep: true,
+ })
+ if err != nil || res == nil || res.ExitCode != 0 || res.Error != "" || res.Killed {
+ reason := fmt.Sprintf("batch read on sandbox %s failed", sandboxID)
+ if err != nil {
+ reason = err.Error()
+ } else if res != nil && res.Error != "" {
+ reason = res.Error
+ }
+ for _, id := range groupIDs {
+ if err != nil {
+ errs[id] = fmt.Errorf("%s: %w", reason, err)
+ } else {
+ errs[id] = errors.New(reason)
+ }
+ }
+ continue
+ }
+ seen := make(map[string]bool, len(groupIDs))
+ for _, snap := range sandbox.ParseShellJobBatch(res.Stdout) {
+ i, ok := byID[snap.Meta.ID]
+ if !ok {
+ continue
+ }
+ snapshots[i] = snap
+ seen[snap.Meta.ID] = true
+ }
+ for _, id := range groupIDs {
+ if !seen[id] {
+ errs[id] = fmt.Errorf("unreadable batch response for shell job %s", id)
+ }
+ }
+ }
+ return snapshots, errs
+}
+
+// StartSweeper polls due index members until ctx is canceled. A service that
+// cannot reach a remote sandbox does not start a goroutine.
+func (s *ShellJobService) StartSweeper(ctx context.Context, interval time.Duration) {
+ if s == nil || !s.RemoteAvailable() || interval <= 0 || ctx == nil {
+ return
+ }
+ go func() {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if err := s.SweepOnce(ctx); err != nil {
+ logger.Warnf(ctx, "[ShellJob] sweep: %v", err)
+ }
+ }
+ }
+ }()
+}
+
+// protectScheduledJobs renews leases independently of remote execs and session
+// locks. A blocked read must not leave any queued session unprotected.
+func (s *ShellJobService) protectScheduledJobs(ctx context.Context) func() {
+ ctx, cancel := context.WithCancel(ctx)
+ var providers sync.WaitGroup
+ var mu sync.Mutex
+ inFlight := make(map[interfaces.SessionRef]bool)
+ refreshProvider := func(ref interfaces.SessionRef) {
+ if s.keeper == nil || ctx.Err() != nil {
+ return
+ }
+ mu.Lock()
+ if inFlight[ref] {
+ mu.Unlock()
+ return
+ }
+ inFlight[ref] = true
+ mu.Unlock()
+ providers.Add(1)
+ go func() {
+ defer providers.Done()
+ defer func() {
+ mu.Lock()
+ delete(inFlight, ref)
+ mu.Unlock()
+ }()
+ refreshCtx, stop := context.WithTimeout(ctx, backgroundKeepAliveTimeout)
+ defer stop()
+ if err := s.keeper.KeepAlive(refreshCtx, ref); err != nil && refreshCtx.Err() == nil {
+ logger.Warnf(ctx, "[ShellJob] provider heartbeat session=%s: %v", ref.SessionID, err)
+ }
+ }()
+ }
+ renew := func() {
+ entries, err := s.index.Entries(ctx)
+ if err != nil || s.bg == nil || s.bg.Registry == nil {
+ return
+ }
+ sessions := make(map[interfaces.SessionRef]bool)
+ for _, e := range entries {
+ // Renew only: a snapshot taken before a terminal claim must not
+ // resurrect a discarded activity or a deleted session.
+ found, err := s.bg.Registry.Renew(ctx, e.SessionRef, e.WorkID, shellJobRenewLease)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] heartbeat id=%s: %v", e.WorkID, err)
+ }
+ if found {
+ sessions[e.SessionRef] = true
+ }
+ _ = s.index.client.Expire(
+ ctx, s.index.reservationKey(e.SessionRef, e.WorkID), interfaces.NoticeRetention,
+ ).Err()
+ }
+ // Provider TTLs are independent of Redis leases. A slow provider must
+ // not block lease renewal or keep another session from being refreshed.
+ for ref := range sessions {
+ refreshProvider(ref)
+ }
+ }
+ renew()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ ticker := time.NewTicker(s.heartbeatInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ renew()
+ }
+ }
+ }()
+ return func() {
+ cancel()
+ <-done
+ providers.Wait()
+ }
+}
+
+// SweepOnce claims due jobs, grouped by session, and reports or renews each one.
+func (s *ShellJobService) SweepOnce(ctx context.Context) error {
+ if s == nil || s.index == nil {
+ return nil
+ }
+ s.swept.Lock()
+ defer s.swept.Unlock()
+ stopHeartbeat := s.protectScheduledJobs(ctx)
+ defer stopHeartbeat()
+ s.settleCapturedOutcomes(ctx)
+ now := time.Now().UTC()
+ claimed, err := s.index.Claim(ctx, now, s.replica)
+ if err != nil {
+ return err
+ }
+ for ref, entries := range claimed {
+ if err := s.index.WithSessionLock(ctx, ref, func(lockCtx context.Context) error {
+ closed, err := s.index.SessionClosed(lockCtx, ref)
+ if err != nil || closed {
+ return err
+ }
+ return s.sweepSession(lockCtx, ref, entries)
+ }); err != nil {
+ logger.Warnf(ctx, "[ShellJob] sweep session=%s: %v", ref.SessionID, err)
+ }
+ if err := s.index.Release(ctx, ref, s.replica); err != nil {
+ logger.Warnf(ctx, "[ShellJob] release sweep lock session=%s: %v", ref.SessionID, err)
+ }
+ }
+ return nil
+}
+
+func (s *ShellJobService) sweepSession(ctx context.Context, ref interfaces.SessionRef,
+ entries []ShellJobIndexEntry,
+) error {
+ ids := make([]string, 0, len(entries))
+ sandboxIDs := make(map[string]string, len(entries))
+ entryByID := make(map[string]ShellJobIndexEntry, len(entries))
+ now := time.Now().UTC()
+ for _, e := range entries {
+ if e.YieldUntil.Add(shellJobHandoff).After(now) {
+ continue
+ }
+ ids = append(ids, e.WorkID)
+ sandboxIDs[e.WorkID] = e.SandboxID
+ entryByID[e.WorkID] = e
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+ if s.remoteRead == nil {
+ return fmt.Errorf("remote shell jobs are unavailable")
+ }
+ snapshots, errs := s.remoteRead(maintenanceCtx(ctx, ref), ref, ids, sandboxIDs)
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if len(snapshots) != len(ids) {
+ return fmt.Errorf("sweep batch read returned %d of %d", len(snapshots), len(ids))
+ }
+ byID := make(map[string]sandbox.ShellJobSnapshot, len(snapshots))
+ for i, snap := range snapshots {
+ key := snap.Meta.ID
+ if key == "" && i < len(ids) {
+ key = ids[i]
+ }
+ if key != "" {
+ byID[key] = snap
+ }
+ }
+ for _, id := range ids {
+ snap := byID[id]
+ entry := entryByID[id]
+ parentMessageID := snap.Meta.ParentMessageID
+ if belongsToOtherSession(snap.Meta, ref) {
+ parentMessageID = ""
+ }
+ switch {
+ case errs != nil && errs[id] != nil && (entry.SandboxID == "" || shellSandboxGone(errs[id])):
+ s.finishLost(ctx, ref, entry, errs[id].Error(), parentMessageID)
+ case errs != nil && errs[id] != nil:
+ // A timed-out or paused sandbox read is not evidence the watcher
+ // died. Dropping the index member and the activity lease here lets
+ // the next resolve rebuild the sandbox under a command that is
+ // still running. Leave both in place and renew the lease so the
+ // next sweep can read again. An index member with no sandbox id
+ // can never be read, so that case is still lost.
+ logger.Warnf(ctx, "[ShellJob] sweep read id=%s: %v", id, errs[id])
+ s.renewRunning(ctx, ref, id, snap)
+ case snap.Missing || belongsToOtherSession(snap.Meta, ref) || snap.Meta.SandboxID == "" ||
+ (entry.SandboxID != "" && snap.Meta.SandboxID != entry.SandboxID):
+ s.finishLost(ctx, ref, entry, "job directory missing or sandbox replaced", parentMessageID)
+ case interfaces.IsTerminalWorkState(snap.Status.State):
+ s.finishTerminal(ctx, ref, entry, snap)
+ case snap.Status.State == interfaces.WorkStateRunning && !snap.WatcherAlive:
+ s.stopOrphanedJob(ctx, ref, entry, snap.Status)
+ s.finishLost(ctx, ref, entry, "watcher process is gone", parentMessageID)
+ default:
+ s.renewRunning(ctx, ref, id, snap)
+ }
+ }
+ return nil
+}
+
+func (s *ShellJobService) settleCapturedOutcomes(ctx context.Context) {
+ if s.index == nil {
+ return
+ }
+ for _, item := range s.skillCapture.pending() {
+ state, ok, err := s.index.Outcome(ctx, item.ref, item.id)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] outcome id=%s: %v", item.id, err)
+ continue
+ }
+ if !ok {
+ continue
+ }
+ s.skillCapture.settle(ctx, item.ref, item.id, state)
+ }
+}
+
+func (s *ShellJobService) recordShellJobOutcome(ctx context.Context, ref interfaces.SessionRef, workID, state string) {
+ if s.index == nil {
+ return
+ }
+ if err := s.index.RecordOutcome(ctx, ref, workID, state); err != nil {
+ logger.Warnf(ctx, "[ShellJob] record outcome id=%s: %v", workID, err)
+ }
+}
+
+// stopOrphanedJob kills the command process group and the docker activity
+// loop after the watcher itself is gone. A missing process is logged and
+// still reported lost. A replaced sandbox is not passed here: its processes
+// are already gone, and exec would land in the new sandbox.
+func (s *ShellJobService) stopOrphanedJob(ctx context.Context, ref interfaces.SessionRef,
+ entry ShellJobIndexEntry, status sandbox.ShellJobStatus,
+) {
+ if s.remoteExec == nil || entry.SandboxID == "" {
+ return
+ }
+ if status.PGID <= 1 && status.TouchPID <= 1 {
+ return
+ }
+ script := sandbox.ShellJobKillOrphansScript(status.PGID, status.TouchPID)
+ execCtx, cancel := context.WithTimeout(maintenanceCtx(ctx, ref), 10*time.Second)
+ defer cancel()
+ res, err := s.remoteExec(execCtx, ref.SessionID, script, sandbox.ShellExecOptions{
+ ExpectedSandboxID: entry.SandboxID,
+ Timeout: 10 * time.Second,
+ SkipWorkspacePrep: true,
+ })
+ if err != nil || (res != nil && res.Error != "") {
+ reason := ""
+ if err != nil {
+ reason = err.Error()
+ } else if res != nil {
+ reason = res.Error
+ }
+ logger.Warnf(ctx, "[ShellJob] kill orphans id=%s: %s", entry.WorkID, reason)
+ }
+}
+
+func (s *ShellJobService) finishTerminal(ctx context.Context, ref interfaces.SessionRef,
+ entry ShellJobIndexEntry, snap sandbox.ShellJobSnapshot,
+) {
+ s.skillCapture.settle(ctx, ref, entry.WorkID, snap.Status.State)
+ claimed, err := s.index.ClaimWork(ctx, ref, entry.WorkID)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] claim terminal id=%s: %v", entry.WorkID, err)
+ return
+ }
+ if !claimed {
+ return
+ }
+ s.recordShellJobOutcome(ctx, ref, entry.WorkID, snap.Status.State)
+ result := s.applyOutputFiles(ctx, ref, ShellJobResult(snap, false), snap.Meta)
+ if result.ID == "" {
+ result.ID = entry.WorkID
+ }
+ if err := s.lifecycle.Finish(ctx, ref, shellJobNotice(entry.WorkID, snap, result.State, result.OutputFiles), result,
+ snap.Meta.ParentMessageID); err != nil {
+ logger.Warnf(ctx, "[ShellJob] finish id=%s: %v", entry.WorkID, err)
+ if rqErr := s.index.Requeue(ctx, entry); rqErr != nil {
+ logger.Warnf(ctx, "[ShellJob] requeue id=%s: %v", entry.WorkID, rqErr)
+ }
+ }
+}
+
+func (s *ShellJobService) finishLost(ctx context.Context, ref interfaces.SessionRef,
+ entry ShellJobIndexEntry, reason, parentMessageID string,
+) {
+ s.skillCapture.drop(ref, entry.WorkID)
+ claimed, err := s.index.ClaimWork(ctx, ref, entry.WorkID)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] claim lost id=%s: %v", entry.WorkID, err)
+ return
+ }
+ if !claimed {
+ return
+ }
+ s.recordShellJobOutcome(ctx, ref, entry.WorkID, interfaces.WorkStateLost)
+ meta := sandbox.ShellJobMeta{
+ ID: entry.WorkID, TenantID: ref.TenantID, SessionID: ref.SessionID,
+ SandboxID: entry.SandboxID, Title: reason,
+ }
+ result := lostResult(meta, reason)
+ snap := sandbox.ShellJobSnapshot{
+ Meta: meta,
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateLost, EndedAt: ptrTime(time.Now().UTC())},
+ }
+ if err := s.lifecycle.Finish(ctx, ref, shellJobNotice(entry.WorkID, snap, interfaces.WorkStateLost, nil), result,
+ parentMessageID); err != nil {
+ logger.Warnf(ctx, "[ShellJob] finish lost id=%s: %v", entry.WorkID, err)
+ if rqErr := s.index.Requeue(ctx, entry); rqErr != nil {
+ logger.Warnf(ctx, "[ShellJob] requeue id=%s: %v", entry.WorkID, rqErr)
+ }
+ }
+}
+
+func (s *ShellJobService) renewRunning(ctx context.Context, ref interfaces.SessionRef,
+ id string, snap sandbox.ShellJobSnapshot,
+) {
+ found, err := s.lifecycle.Alive(ctx, ref, id, shellJobRenewLease)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] renew id=%s: %v", id, err)
+ return
+ }
+ if found {
+ return
+ }
+ if s.bg == nil || s.bg.Registry == nil {
+ return
+ }
+ started := snap.Status.StartedAt
+ if started.IsZero() {
+ started = time.Now().UTC()
+ }
+ activity := interfaces.BackgroundActivity{
+ ID: id, Kind: "shell", Title: snap.Meta.Title, StartedAt: started,
+ }
+ if err := s.bg.Registry.Add(ctx, ref, activity, shellJobRenewLease, 0); err != nil {
+ logger.Warnf(ctx, "[ShellJob] re-add id=%s: %v", id, err)
+ return
+ }
+ if s.keeper != nil {
+ if err := s.keeper.KeepAlive(ctx, ref); err != nil {
+ logger.Warnf(ctx, "[ShellJob] keep-alive after re-add id=%s: %v", id, err)
+ }
+ }
+}
+
+// OnStop does not cancel shell jobs. Stopping the turn only returns the
+// submit tool early; the watcher keeps running.
+func (s *ShellJobService) OnStop(context.Context, interfaces.SessionRef) error { return nil }
+
+// OnDelete stops detached jobs before the session's shared-machine workspace
+// or pin disappears. Failed cancellation leaves tracking intact and aborts deletion.
+func (s *ShellJobService) OnDelete(ctx context.Context, ref interfaces.SessionRef) error {
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ return s.local.OnDelete(ctx, ref)
+ }
+ if s.index == nil {
+ return nil
+ }
+ return s.index.WithSessionLock(ctx, ref, func(ctx context.Context) error {
+ entries, err := s.index.Entries(ctx)
+ if err != nil {
+ return err
+ }
+ for _, e := range entries {
+ if e.SessionRef != ref {
+ continue
+ }
+ result, err := s.cancelRemoteJob(ctx, ref, e.WorkID, e.SandboxID, true)
+ if err != nil && !errors.Is(err, interfaces.ErrWorkNotFound) && !shellSandboxGone(err) {
+ return err
+ }
+ if err == nil && !result.Terminal {
+ return fmt.Errorf("shell job %s is still stopping; retry session deletion", e.WorkID)
+ }
+ }
+ if err := s.index.client.Set(ctx, s.index.closedKey(ref), "1", interfaces.NoticeRetention).Err(); err != nil {
+ return err
+ }
+ if err := s.index.RemoveSession(ctx, ref); err != nil {
+ return err
+ }
+ if err := s.index.DeleteSessionCursors(ctx, ref); err != nil {
+ return err
+ }
+ s.skillCapture.dropSession(ref)
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ prefix := fmt.Sprintf("%d\x00%s\x00", ref.TenantID, ref.SessionID)
+ for key := range s.cursors {
+ if strings.HasPrefix(key, prefix) {
+ delete(s.cursors, key)
+ }
+ }
+ return nil
+ })
+}
+
+// OnDeleteAborted allows new jobs after a failed session deletion.
+func (s *ShellJobService) OnDeleteAborted(ctx context.Context, ref interfaces.SessionRef) error {
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ if rollback, ok := s.local.(interface {
+ OnDeleteAborted(context.Context, interfaces.SessionRef) error
+ }); ok {
+ return rollback.OnDeleteAborted(ctx, ref)
+ }
+ return nil
+ }
+ if s.index == nil {
+ return nil
+ }
+ return s.index.WithSessionLock(ctx, ref, func(ctx context.Context) error {
+ return s.index.client.Del(ctx, s.index.closedKey(ref)).Err()
+ })
+}
+
+// EndShellJobRun keeps the run's output cursors for another hour so a late
+// read can still be incremental.
+func (s *ShellJobService) EndShellJobRun(ref interfaces.SessionRef, runID string) {
+ if s == nil || runID == "" {
+ return
+ }
+ if s.local != nil && s.onHostSession(context.Background(), ref) {
+ s.local.EndShellJobRun(ref, runID)
+ return
+ }
+ if s.index != nil {
+ if err := s.index.TouchRunCursors(context.Background(), ref, runID, time.Hour); err != nil {
+ logger.Warnf(context.Background(), "[ShellJob] touch cursors run=%s: %v", runID, err)
+ }
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ prefix := fmt.Sprintf("%d\x00%s\x00%s\x00", ref.TenantID, ref.SessionID, runID)
+ exp := time.Now().Add(time.Hour)
+ for key, cursor := range s.cursors {
+ if strings.HasPrefix(key, prefix) {
+ cursor.expires = exp
+ s.cursors[key] = cursor
+ }
+ }
+}
+
+// ShellJobResult maps a directory snapshot into the shared work result.
+// lite selects the host-session wording of the running and lost notes.
+func ShellJobResult(snap sandbox.ShellJobSnapshot, lite bool) interfaces.WorkResult {
+ state := snap.Status.State
+ if state == "" {
+ state = interfaces.WorkStateUnknown
+ }
+ payload := ShellJobPayload{
+ DisplayType: "shell_exec",
+ WorkID: snap.Meta.ID,
+ State: state,
+ SessionID: snap.Meta.SessionID,
+ Command: snap.Meta.Command,
+ WorkDir: snap.Meta.WorkDir,
+ ExitCode: snap.Status.ExitCode,
+ DurationMS: shellJobDurationMS(snap.Status),
+ StartedAt: snap.Status.StartedAt,
+ Stdout: snap.Stdout.Text,
+ Stderr: snap.Stderr.Text,
+ FromBytes: sandbox.ShellJobOffsets{
+ Stdout: snap.Stdout.FromBytes, Stderr: snap.Stderr.FromBytes,
+ },
+ TotalBytes: sandbox.ShellJobOffsets{
+ Stdout: snap.Stdout.TotalBytes, Stderr: snap.Stderr.TotalBytes,
+ },
+ OmittedBytes: sandbox.ShellJobOffsets{
+ Stdout: snap.Stdout.OmittedBytes, Stderr: snap.Stderr.OmittedBytes,
+ },
+ Truncated: snap.Stdout.Truncated || snap.Stderr.Truncated ||
+ snap.Stdout.OmittedBytes > 0 || snap.Stderr.OmittedBytes > 0,
+ Note: shellJobNote(snap, lite),
+ }
+ return interfaces.WorkResult{
+ ID: snap.Meta.ID, State: state, Title: snap.Meta.Title,
+ Terminal: interfaces.IsTerminalWorkState(state), Payload: payload,
+ }
+}
+
+func shellJobNote(snap sandbox.ShellJobSnapshot, lite bool) string {
+ var b strings.Builder
+ switch snap.Status.State {
+ case interfaces.WorkStateRunning:
+ b.WriteString(shellJobNoteRunning)
+ if lite {
+ b.WriteString(shellJobNoteRunningLite)
+ }
+ case interfaces.WorkStateLost:
+ if lite {
+ b.WriteString(shellJobNoteLostLite)
+ } else {
+ b.WriteString(shellJobNoteLost)
+ }
+ }
+ if extra := shellJobTruncationNote(snap); extra != "" {
+ if b.Len() > 0 {
+ b.WriteByte(' ')
+ }
+ b.WriteString(extra)
+ }
+ return b.String()
+}
+
+func shellJobTruncationNote(snap sandbox.ShellJobSnapshot) string {
+ omitted := snap.Stdout.OmittedBytes + snap.Stderr.OmittedBytes
+ truncated := omitted > 0 || snap.Stdout.Truncated || snap.Stderr.Truncated
+ if !truncated {
+ return ""
+ }
+ if snap.Stdout.FromBytes > 0 || snap.Stderr.FromBytes > 0 {
+ return fmt.Sprintf("%d bytes of older output were rotated away.", omitted)
+ }
+ return fmt.Sprintf("Showed the first and last part; %d bytes in between were omitted.", omitted)
+}
+
+func shellTerminalResult(meta sandbox.ShellJobMeta, status sandbox.ShellJobStatus,
+ offsets sandbox.ShellJobOffsets, preview *shellJobPreview, lite bool,
+) interfaces.WorkResult {
+ snap := sandbox.ShellJobSnapshot{
+ Meta: meta,
+ Status: status,
+ Stdout: sandbox.ShellJobStream{TotalBytes: offsets.Stdout},
+ Stderr: sandbox.ShellJobStream{TotalBytes: offsets.Stderr},
+ }
+ if preview != nil {
+ snap.Stdout = preview.stdout.stream(offsets.Stdout)
+ snap.Stderr = preview.stderr.stream(offsets.Stderr)
+ }
+ return ShellJobResult(snap, lite)
+}
+
+func lostResult(meta sandbox.ShellJobMeta, _ string) interfaces.WorkResult {
+ snap := sandbox.ShellJobSnapshot{
+ Meta: meta,
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateLost},
+ Missing: true,
+ }
+ return ShellJobResult(snap, false)
+}
+
+func shellJobNotice(
+ id string, snap sandbox.ShellJobSnapshot, state string, links []string,
+) interfaces.BackgroundNotice {
+ title := snap.Meta.Title
+ if strings.TrimSpace(title) == "" {
+ title = id
+ }
+ finished := time.Now().UTC()
+ if snap.Status.EndedAt != nil && !snap.Status.EndedAt.IsZero() {
+ finished = snap.Status.EndedAt.UTC()
+ }
+ text := fmt.Sprintf("Background shell %s %s: %s. Use work_wait(wait_sec=0) to read its output.",
+ id, state, title)
+ if len(links) > 0 {
+ text += " Output files: `" + strings.Join(links, "`, `") + "`."
+ }
+ return interfaces.BackgroundNotice{
+ ID: id, Kind: "shell", FinishedAt: finished, Text: text,
+ }
+}
+
+// HoldSkillEnvCapture keeps a credential write-back until this process sees
+// the job succeed. The pairs never go into the job files or Redis.
+func (s *ShellJobService) HoldSkillEnvCapture(
+ ctx context.Context, ref interfaces.SessionRef, jobID string, run func(context.Context),
+) {
+ if s == nil || run == nil || jobID == "" {
+ return
+ }
+ if s.local != nil && s.onHostSession(ctx, ref) {
+ if holder, ok := s.local.(interface {
+ HoldSkillEnvCapture(interfaces.SessionRef, string, func(context.Context))
+ }); ok {
+ holder.HoldSkillEnvCapture(ref, jobID, run)
+ return
+ }
+ }
+ s.skillCapture.put(ref, jobID, run)
+}
+
+func (s *ShellJobService) applyOutputFiles(
+ ctx context.Context, ref interfaces.SessionRef, result interfaces.WorkResult, meta sandbox.ShellJobMeta,
+) interfaces.WorkResult {
+ if !result.Terminal || !meta.OutputSeen || meta.OutputDir == "" {
+ return result
+ }
+ after, err := s.listOutputFiles(ctx, ref.SessionID, meta.OutputDir)
+ if err != nil {
+ return result
+ }
+ links := tools.ChangedOutputFileLinks(meta.OutputDir, sandbox.EntriesFromStamps(meta.OutputBefore), after)
+ result.OutputFiles = links
+ if payload, ok := result.Payload.(ShellJobPayload); ok {
+ payload.OutputFiles = links
+ result.Payload = payload
+ }
+ return result
+}
+
+func (s *ShellJobService) listOutputFiles(
+ ctx context.Context, sessionID, dir string,
+) ([]sandbox.RemoteDirEntry, error) {
+ if s.listFiles != nil {
+ return s.listFiles(ctx, sessionID, dir)
+ }
+ if s.pinned == nil {
+ return nil, errors.New("shell job service has no sandbox file listing")
+ }
+ return s.pinned.ListSessionFiles(ctx, sessionID, dir)
+}
+
+// skillEnvCaptures holds in-memory credential write-backs keyed by session and job.
+type skillEnvCaptures struct {
+ mu sync.Mutex
+ fns map[string]func(context.Context)
+}
+
+func skillCaptureKey(ref interfaces.SessionRef, jobID string) string {
+ return fmt.Sprintf("%d\x00%s\x00%s", ref.TenantID, ref.SessionID, jobID)
+}
+
+func (c *skillEnvCaptures) put(ref interfaces.SessionRef, jobID string, run func(context.Context)) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.fns == nil {
+ c.fns = map[string]func(context.Context){}
+ }
+ c.fns[skillCaptureKey(ref, jobID)] = run
+}
+
+func (c *skillEnvCaptures) settle(ctx context.Context, ref interfaces.SessionRef, jobID, state string) {
+ run := c.take(ref, jobID)
+ if run == nil || state != interfaces.WorkStateSucceeded {
+ return
+ }
+ run(ctx)
+}
+
+func (c *skillEnvCaptures) drop(ref interfaces.SessionRef, jobID string) {
+ _ = c.take(ref, jobID)
+}
+
+func (c *skillEnvCaptures) dropSession(ref interfaces.SessionRef) {
+ prefix := fmt.Sprintf("%d\x00%s\x00", ref.TenantID, ref.SessionID)
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ for key := range c.fns {
+ if strings.HasPrefix(key, prefix) {
+ delete(c.fns, key)
+ }
+ }
+}
+
+type pendingSkillCapture struct {
+ ref interfaces.SessionRef
+ id string
+}
+
+func (c *skillEnvCaptures) pending() []pendingSkillCapture {
+ if c == nil {
+ return nil
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ out := make([]pendingSkillCapture, 0, len(c.fns))
+ for key := range c.fns {
+ parts := strings.Split(key, "\x00")
+ if len(parts) != 3 {
+ continue
+ }
+ var tenant uint64
+ if _, err := fmt.Sscanf(parts[0], "%d", &tenant); err != nil {
+ continue
+ }
+ out = append(out, pendingSkillCapture{
+ ref: interfaces.SessionRef{TenantID: tenant, SessionID: parts[1]},
+ id: parts[2],
+ })
+ }
+ return out
+}
+
+func (c *skillEnvCaptures) take(ref interfaces.SessionRef, jobID string) func(context.Context) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ key := skillCaptureKey(ref, jobID)
+ run := c.fns[key]
+ delete(c.fns, key)
+ return run
+}
+
+func (s *ShellJobService) setCursorsLocked(ref interfaces.SessionRef, runID, workID string,
+ from sandbox.ShellJobOffsets,
+) {
+ if runID == "" || workID == "" {
+ return
+ }
+ if s.index != nil {
+ if err := s.index.PutCursor(context.Background(), ref, runID, workID, from, time.Hour); err != nil {
+ logger.Warnf(context.Background(), "[ShellJob] cursor id=%s: %v", workID, err)
+ }
+ return
+ }
+ s.pruneCursorsLocked(time.Now())
+ s.cursors[cursorKey(ref, runID, workID)] = shellJobCursor{
+ from: from, expires: time.Now().Add(time.Hour),
+ }
+}
+
+func (s *ShellJobService) pruneCursorsLocked(now time.Time) {
+ for key, cursor := range s.cursors {
+ if !cursor.expires.After(now) {
+ delete(s.cursors, key)
+ }
+ }
+}
+
+func cursorKey(ref interfaces.SessionRef, runID, workID string) string {
+ return fmt.Sprintf("%d\x00%s\x00%s\x00%s", ref.TenantID, ref.SessionID, runID, workID)
+}
+
+type (
+ shellReadRunKey struct{}
+ shellReadBudgetKey struct{}
+)
+
+func shellSandboxGone(err error) bool {
+ return errors.Is(err, sandbox.ErrBoundSandboxChanged) || errors.Is(err, sandbox.ErrSandboxSessionDeleted)
+}
+
+func (s *ShellJobService) cursorsFor(
+ ctx context.Context, ref interfaces.SessionRef, ids []string,
+) map[string]sandbox.ShellJobOffsets {
+ runID, _ := ctx.Value(shellReadRunKey{}).(string)
+ if s == nil || runID == "" || len(ids) == 0 {
+ return nil
+ }
+ if s.index != nil {
+ out := make(map[string]sandbox.ShellJobOffsets, len(ids))
+ for _, id := range ids {
+ from, ok, err := s.index.GetCursor(ctx, ref, runID, id)
+ if err != nil {
+ logger.Warnf(ctx, "[ShellJob] cursor id=%s: %v", id, err)
+ continue
+ }
+ if !ok {
+ continue
+ }
+ out[id] = from
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+ }
+ now := time.Now()
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.pruneCursorsLocked(now)
+ out := make(map[string]sandbox.ShellJobOffsets, len(ids))
+ for _, id := range ids {
+ cursor, ok := s.cursors[cursorKey(ref, runID, id)]
+ if !ok {
+ continue
+ }
+ out[id] = cursor.from
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+func (s *ShellJobService) noteReadCursor(
+ ref interfaces.SessionRef, runID, workID string, snap sandbox.ShellJobSnapshot,
+) {
+ if s == nil || runID == "" || workID == "" || snap.Missing {
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.setCursorsLocked(ref, runID, workID, sandbox.ShellJobOffsets{
+ Stdout: nextStreamOffset(snap.Stdout),
+ Stderr: nextStreamOffset(snap.Stderr),
+ })
+}
+
+func nextStreamOffset(st sandbox.ShellJobStream) int64 {
+ if st.NextBytes != nil {
+ return *st.NextBytes
+ }
+ end := st.FromBytes + int64(len(st.Text))
+ if st.OmittedBytes > 0 && st.FromBytes == 0 {
+ return st.TotalBytes
+ }
+ if end > 0 && end < st.TotalBytes && st.Truncated && st.OmittedBytes == 0 {
+ return end
+ }
+ if end > st.TotalBytes {
+ return st.TotalBytes
+ }
+ if end == 0 {
+ return st.TotalBytes
+ }
+ return st.TotalBytes
+}
+
+// ensureBoundSandbox returns the session's sandbox id. A first command has
+// no binding yet: one ordinary exec (empty ExpectedSandboxID) takes the same
+// resolveSession path as shell_exec and creates it. The job script itself
+// still runs against the id this returns.
+func (s *ShellJobService) ensureBoundSandbox(ctx context.Context, ref interfaces.SessionRef) (string, error) {
+ if s.boundSandbox == nil {
+ return "", fmt.Errorf("session has no bound sandbox")
+ }
+ lookupCtx := maintenanceCtx(ctx, ref)
+ if id, ok := s.boundSandbox(lookupCtx, ref.SessionID); ok && id != "" {
+ return id, nil
+ }
+ if s.remoteExec == nil {
+ return "", fmt.Errorf("session has no bound sandbox")
+ }
+ if _, err := s.remoteExec(lookupCtx, ref.SessionID, "true", sandbox.ShellExecOptions{
+ // No WorkDir: the warm-up must take the same resolveSession path as
+ // shell_exec, whose cwd is the manager's layout-root default. A pinned
+ // legacy "/workspace" is outside a shared machine's WorkDirRoots.
+ }); err != nil {
+ return "", err
+ }
+ if id, ok := s.boundSandbox(lookupCtx, ref.SessionID); ok && id != "" {
+ return id, nil
+ }
+ return "", fmt.Errorf("session has no bound sandbox")
+}
+
+func (s *ShellJobService) onHostSession(ctx context.Context, ref interfaces.SessionRef) bool {
+ if s == nil || s.pinned == nil {
+ return false
+ }
+ return !s.pinned.VersionsWorkspace(ctx, ref.SessionID)
+}
+
+func maintenanceCtx(ctx context.Context, ref interfaces.SessionRef) context.Context {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ return types.WithSandboxTenantID(
+ types.WithExecutionTenant(context.WithoutCancel(ctx), ref.TenantID),
+ ref.TenantID,
+ )
+}
+
+func clampYieldTime(d time.Duration) time.Duration {
+ if d <= 0 {
+ return shellJobYieldDefault
+ }
+ if d < shellJobYieldFloor {
+ return shellJobYieldFloor
+ }
+ if d > shellJobYieldMax {
+ return shellJobYieldMax
+ }
+ return d
+}
+
+func validateShellJobIDs(ids []string) error {
+ for _, id := range ids {
+ if !sandbox.ValidShellJobID(id) {
+ return interfaces.ErrWorkNotFound
+ }
+ }
+ return nil
+}
+
+func belongsToOtherSession(meta sandbox.ShellJobMeta, ref interfaces.SessionRef) bool {
+ if meta.SessionID != "" && meta.SessionID != ref.SessionID {
+ return true
+ }
+ if meta.TenantID != 0 && meta.TenantID != ref.TenantID {
+ return true
+ }
+ return false
+}
+
+func shellJobDurationMS(status sandbox.ShellJobStatus) int64 {
+ if status.StartedAt.IsZero() {
+ return 0
+ }
+ end := time.Now()
+ if status.EndedAt != nil && !status.EndedAt.IsZero() {
+ end = *status.EndedAt
+ }
+ d := end.Sub(status.StartedAt)
+ if d < 0 {
+ return 0
+ }
+ return d.Milliseconds()
+}
+
+var errShellJobIDTaken = errors.New("shell job id already exists")
+
+// allocateShellJobID draws a handle, skipping ids taken reports as in use.
+func allocateShellJobID(taken func(string) bool) (string, error) {
+ for range shellJobIDAttempts {
+ id, err := sandbox.NewShellJobID()
+ if err != nil {
+ return "", err
+ }
+ if taken != nil && taken(id) {
+ continue
+ }
+ return id, nil
+ }
+ return "", errShellJobIDTaken
+}
+
+func hostname() string {
+ name, err := os.Hostname()
+ if err != nil || strings.TrimSpace(name) == "" {
+ return "unknown"
+ }
+ return name
+}
+
+func osGetpid() int { return os.Getpid() }
+
+func ptrTime(t time.Time) *time.Time { return &t }
diff --git a/internal/application/service/shell_job_service_test.go b/internal/application/service/shell_job_service_test.go
new file mode 100644
index 000000000..a60cb8dfe
--- /dev/null
+++ b/internal/application/service/shell_job_service_test.go
@@ -0,0 +1,1026 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "regexp"
+ "testing"
+ "time"
+
+ "github.com/alicebob/miniredis/v2"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+var shellJobMarkerInScript = regexp.MustCompile(`__WEKNORA_SJ___[0-9a-f]{12}__`)
+
+// fakeRemoteReader 模拟批量读取,签名与 remoteReadFunc 缝一致。它必须忠实
+// 履行契约:len(snapshots)==len(ids) 恒成立、缺席任务给 Missing 占位、
+// errs 按 workID 逐条注入——否则它测不出生产实现的 D1 合规性。
+type fakeRemoteReader struct {
+ snapshots map[string]sandbox.ShellJobSnapshot
+ errs map[string]error
+ execCalls int
+ // calls 记录每次读取收到的 (ids, sandboxIDs),测试据此断言 D1。
+ calls []fakeBatchReadCall
+}
+
+type fakeBatchReadCall struct {
+ ids []string
+ sandboxIDs map[string]string
+}
+
+func (f *fakeRemoteReader) batchRead(_ context.Context, _ interfaces.SessionRef,
+ ids []string, sandboxIDs map[string]string,
+) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ f.execCalls++
+ copied := make(map[string]string, len(sandboxIDs))
+ for id, sbx := range sandboxIDs {
+ copied[id] = sbx
+ }
+ f.calls = append(f.calls, fakeBatchReadCall{
+ ids: append([]string(nil), ids...),
+ sandboxIDs: copied,
+ })
+ out := make([]sandbox.ShellJobSnapshot, len(ids))
+ for i, id := range ids {
+ if snap, ok := f.snapshots[id]; ok {
+ out[i] = snap
+ } else {
+ out[i] = sandbox.ShellJobSnapshot{Missing: true, Meta: sandbox.ShellJobMeta{ID: id}}
+ }
+ }
+ return out, f.errs
+}
+
+func newRemoteServiceFixture(t *testing.T) (*ShellJobService, *fakeRemoteReader, *ShellJobIndex, *BackgroundWork) {
+ t.Helper()
+ mr := miniredisRunT(t)
+ client := redisNewClient(mr)
+ t.Cleanup(func() { _ = client.Close() })
+ index := NewShellJobIndex(client, "weknora")
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ reader := &fakeRemoteReader{snapshots: map[string]sandbox.ShellJobSnapshot{}}
+ // keeper 经构造器注入(与生产同一条路径);需要断言顺序的测试用
+ // newRemoteServiceFixtureWithKeeper 换成自己的 fake。
+ svc := newRemoteServiceFixtureWithKeeper(t, bg, index, keeperFunc(
+ func(context.Context, interfaces.SessionRef) error { return nil }))
+ svc.remoteRead = reader.batchRead
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "sbx1", true }
+ return svc, reader, index, bg
+}
+
+func newRemoteServiceFixtureWithKeeper(t *testing.T, bg *BackgroundWork,
+ index *ShellJobIndex, keeper interfaces.SessionKeepAliver,
+) *ShellJobService {
+ t.Helper()
+ svc := NewShellJobService(nil, bg, index, nil, keeper)
+ // pinned 为 nil。RemoteAvailable 在注入 remoteExec 后才为真,提交才能走到
+ // keeper 检查;否则会先报 unavailable,测不到「没接线必须当场失败」。
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ return svc
+}
+
+// keeperFunc 把保活断言挂进服务:keepalive 与 exec 的先后顺序必须可观测。
+type keeperFunc func(context.Context, interfaces.SessionRef) error
+
+func (f keeperFunc) KeepAlive(ctx context.Context, s interfaces.SessionRef) error { return f(ctx, s) }
+
+func TestSubmitKeepAlivesBeforeTheYieldWait(t *testing.T) {
+ var order []string
+ mr := miniredisRunT(t)
+ client := redisNewClient(mr)
+ t.Cleanup(func() { _ = client.Close() })
+ index := NewShellJobIndex(client, "weknora")
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ // keeper 经构造器注入——与生产 provider 的接线路径一致,不是字段直塞。
+ svc := newRemoteServiceFixtureWithKeeper(t, bg, index, keeperFunc(
+ func(context.Context, interfaces.SessionRef) error {
+ order = append(order, "keepalive")
+ return nil
+ }))
+ reader := &fakeRemoteReader{snapshots: map[string]sandbox.ShellJobSnapshot{}}
+ svc.remoteRead = reader.batchRead
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ order = append(order, "exec")
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "sbx1", true }
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: 30 * time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ // Begin 在 start 返回之后还会再保活一次。前两步必须是「保活、再 exec」。
+ require.Equal(t, []string{"keepalive", "exec", "keepalive"}, order,
+ "保活必须在 yield 等待之前:Lifecycle.Begin 自带的保活要等 start 返回(最长 40s 后)")
+}
+
+// E2B 的 exec 不续期,keeper 漏接线绝不能静默跳过:提交必须当场失败,
+// 而不是几分钟后以「莫名的 lost」出现。
+func TestSubmitFailsLoudlyWithoutKeepAliver(t *testing.T) {
+ mr := miniredisRunT(t)
+ client := redisNewClient(mr)
+ t.Cleanup(func() { _ = client.Close() })
+ index := NewShellJobIndex(client, "weknora")
+ bg := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := newRemoteServiceFixtureWithKeeper(t, bg, index, nil)
+ require.True(t, svc.lifecycle.Events == nil, "nil emitter must not become a typed-nil interface")
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "sbx1", true }
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.ErrorContains(t, err, "no keep-aliver wired")
+ items, listErr := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, listErr)
+ require.Empty(t, items, "Begin 之前就失败:不留登记")
+ due, dueErr := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, dueErr)
+ require.Empty(t, due, "Begin 之前就失败:不留索引成员")
+}
+
+func TestAllocateShellJobIDSkipsTakenHandles(t *testing.T) {
+ id, err := allocateShellJobID(func(string) bool { return false })
+ require.NoError(t, err)
+ require.True(t, sandbox.ValidShellJobID(id))
+
+ _, err = allocateShellJobID(func(string) bool { return true })
+ require.ErrorIs(t, err, errShellJobIDTaken)
+}
+
+func TestSubmitRetriesWhenTheJobDirectoryExists(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ calls := 0
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ calls++
+ if calls == 1 {
+ return &sandbox.ExecuteResult{ExitCode: sandbox.ShellJobDirExistsExit}, nil
+ }
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.NoError(t, err)
+ require.Equal(t, 2, calls)
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.True(t, sandbox.ValidShellJobID(items[0].ID))
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Len(t, due, 1)
+}
+
+func TestSubmitProvisionsSandboxWhenUnbound(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ bound := ""
+ var sawProvision bool
+ svc.boundSandbox = func(context.Context, string) (string, bool) {
+ if bound == "" {
+ return "", false
+ }
+ return bound, true
+ }
+ svc.remoteExec = func(
+ _ context.Context, _ string, command string, opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ if opts.ExpectedSandboxID == "" {
+ require.Equal(t, "true", command)
+ // No pinned cwd: shared machines admit only the session layout
+ // root, which is exactly the manager's empty-WorkDir default.
+ require.Empty(t, opts.WorkDir)
+ sawProvision = true
+ bound = "sbx-new"
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ require.Equal(t, "sbx-new", opts.ExpectedSandboxID)
+ require.NotEqual(t, "true", command)
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "sleep 30", YieldTime: time.Second,
+ })
+ require.NoError(t, err)
+ require.True(t, sawProvision)
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Len(t, due, 1)
+ require.Equal(t, "sbx-new", due[0].SandboxID)
+}
+
+func TestSubmitProvisionFailureLeavesNoRegistration(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "", false }
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return nil, errors.New("create sandbox: boom")
+ }
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.ErrorContains(t, err, "create sandbox: boom")
+ items, listErr := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, listErr)
+ require.Empty(t, items)
+ due, dueErr := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, dueErr)
+ require.Empty(t, due)
+}
+
+func TestSubmitLeaseCoversTheWholeYieldWindow(t *testing.T) {
+ require.Equal(t, 45*time.Second, shellJobLease(30*time.Second))
+ require.Equal(t, 25*time.Second, shellJobLease(10*time.Second),
+ "默认窗口下接近设计稿的 30 秒;长窗口必须按比例放宽,否则登记在窗口内过期")
+}
+
+func TestSubmitPreLaunchFailureLeavesNoGhostIndexEntry(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ // /stop 或 resolve 失败发生在 client.Exec 之前:返回真 Go 错误。
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return nil, errors.New("context canceled before exec")
+ }
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.Error(t, err)
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Empty(t, due, "wrapper 从未启动:索引成员必须回滚,否则 sweeper 会发假 lost 通知")
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, items, "Begin 已回滚登记")
+}
+
+func TestSubmitMidExecCancellationKeepsRegistration(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ // /stop 落在 exec 中段:transport 错误被 remoteExecuteResult 包装进
+ // ExecuteResult.Error,Go err 为 nil —— 必须按「已启动、等待被切断」处理。
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{ExitCode: -1, Error: "context canceled"}, nil
+ }
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background, "/stop 中段取消:返回运行中句柄,sweeper 窗口后裁决")
+ require.Equal(t, interfaces.WorkStateRunning, sub.Result.State)
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1, "登记保留:rewind/重建保护不因 /stop 失效")
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.NotEmpty(t, due, "索引成员保留:命令可能仍在运行")
+ require.Zero(t, reader.execCalls, "中段取消后不得再详细读取:context 可能已取消,读失败会看起来像 lost")
+}
+
+func TestSweepReportsTerminalAndClaimsOnce(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ ended := now.Add(-time.Minute)
+ exit := 0
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: "sj_0123", TenantID: 7, SessionID: "s",
+ SandboxID: "sbx1", Title: "build", ParentMessageID: "m1",
+ },
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateSucceeded,
+ ExitCode: &exit, StartedAt: now.Add(-2 * time.Minute), EndedAt: &ended,
+ },
+ WatcherAlive: true,
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1",
+ YieldUntil: now.Add(-time.Minute),
+ }))
+ err := bg.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell"}, 30*time.Second, 0)
+ require.NoError(t, err)
+
+ require.NoError(t, svc.SweepOnce(context.Background()))
+
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+ require.Equal(t, "sj_0123", notices[0].ID)
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, items, "Finish 移除登记")
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Empty(t, due, "终态后索引成员被 ZREM")
+
+ // 第二轮 sweep 不重复通知(D2:ZREM 已把成员拿走)
+ require.NoError(t, svc.SweepOnce(context.Background()))
+ notices, err = bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+}
+
+// 共享机器(user-mode 沙盒)只接受 layout.WorkDirRoots 作为 exec 的 cwd,
+// 管理器对空 WorkDir 自动取会话 layout 根(session_manager 的默认)。
+// 任一维护 exec 显式指定 "/" 或旧 "/workspace" 都会被 CleanWorkDirIn
+// 拒绝,把 shell_exec_async 的提交、取消与 work_wait 全部打断。
+func TestMaintenanceExecsLeaveWorkDirToTheSessionLayout(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ var execOpts []sandbox.ShellExecOptions
+ inner := svc.remoteExec
+ record := func(
+ ctx context.Context, sid, cmd string, opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ execOpts = append(execOpts, opts)
+ return inner(ctx, sid, cmd, opts)
+ }
+ svc.remoteExec = record
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ execOpts = nil
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "", false }
+ _, _ = svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.NotEmpty(t, execOpts, "无绑定时 ensureBoundSandbox 也要走一次预热 exec")
+ require.Empty(t, execOpts[0].WorkDir, "预热 exec 不得钉死 cwd:旧 /workspace 在共享机器上被拒")
+
+ svc.boundSandbox = func(context.Context, string) (string, bool) { return "sbx1", true }
+ execOpts = nil
+ _, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second,
+ })
+ require.NoError(t, err)
+ require.NotEmpty(t, execOpts)
+ for _, opts := range execOpts {
+ require.Empty(t, opts.WorkDir, "提交 exec 不得指定 cwd:\"/\" 在共享机器上被拒")
+ }
+
+ execOpts = nil
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", "s", "sbx1")
+ inner = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{Stdout: `{"state":"cancelled"}`}, nil
+ }
+ _, err = svc.Cancel(context.Background(), ref, []string{"sj_0123"})
+ require.NoError(t, err)
+ require.NotEmpty(t, execOpts)
+ require.Empty(t, execOpts[0].WorkDir, "取消 exec 不得指定 cwd")
+
+ recorded := 0
+ svc.remoteExec = func(_ context.Context, _ string, _ string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ recorded++
+ require.Empty(t, opts.WorkDir, "批量读 exec 不得指定 cwd")
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ _, _ = svc.batchReadViaPinned(context.Background(), ref,
+ []string{"sj_0123"}, map[string]string{"sj_0123": "sbx1"})
+ require.NotZero(t, recorded)
+}
+
+func TestSweepKeepsJobWhenReadFails(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.errs = map[string]error{"sj_0123": errors.New("sandbox exec timed out")}
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "sbx1", Title: "build",
+ },
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning, StartedAt: now},
+ WatcherAlive: true,
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: now.Add(-time.Minute),
+ }))
+ require.NoError(t, bg.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell", Title: "build"}, time.Minute, 0))
+
+ require.NoError(t, svc.SweepOnce(context.Background()))
+
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, notices, "读取失败不是终态,不能写完成通知")
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1, "登记还在,沙盒重建保护不能因为一次读取失败消失")
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.NotEmpty(t, due, "索引成员留下,下一轮再读")
+}
+
+func TestSweepMarksLostWhenTheIndexHasNoSandbox(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.errs = map[string]error{"sj_0123": errors.New("session has no bound sandbox")}
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "", YieldUntil: now.Add(-time.Minute),
+ }))
+ require.NoError(t, bg.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell", Title: "build"}, time.Minute, 0))
+
+ require.NoError(t, svc.SweepOnce(context.Background()))
+
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+ require.Contains(t, notices[0].Text, "lost")
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, items)
+}
+
+func TestCancelReportsTheStatusItRead(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", "s", "sbx1")
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{
+ ExitCode: 0,
+ Stdout: `{"state":"succeeded","watcher_pid":1,"pgid":2,"exit_code":0}`,
+ }, nil
+ }
+ results, err := svc.Cancel(context.Background(), ref, []string{"sj_0123"})
+ require.NoError(t, err)
+ require.Equal(t, interfaces.WorkStateSucceeded, results[0].State)
+ require.True(t, results[0].Terminal)
+}
+
+func TestCancelDoesNotInventCancelledWithoutAStatus(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ reader.snapshots["sj_0123"] = ownedRunningShellSnapshot("sj_0123", "s", "sbx1")
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ return &sandbox.ExecuteResult{ExitCode: 0, Stdout: ""}, nil
+ }
+ results, err := svc.Cancel(context.Background(), ref, []string{"sj_0123"})
+ require.ErrorContains(t, err, "cancel status unreadable")
+ require.Empty(t, results)
+}
+
+func TestCancelScriptPrintsStatusInsteadOfExitingEarly(t *testing.T) {
+ script := sandbox.ShellJobCancelScript(sandbox.ShellJobMeta{ID: "sj_0123"}, false)
+ require.NotContains(t, script, "exit 0")
+ require.Contains(t, script, `cat "$D/status.json"`)
+ require.Contains(t, script, "kill -TERM")
+}
+
+func TestSweepMarksLostWhenWatcherDeadOrSandboxChanged(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "old"},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning, StartedAt: now},
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "new", YieldUntil: now.Add(-time.Minute),
+ }))
+ require.NoError(t, svc.SweepOnce(context.Background()))
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+ require.Contains(t, notices[0].Text, "lost")
+}
+
+func TestSweepRenewsAndReaddsExpiredRegistration(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "sbx1"},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning, StartedAt: now},
+ WatcherAlive: true,
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: now.Add(-time.Minute),
+ }))
+ // 不预置登记:模拟租约已过期
+ require.NoError(t, svc.SweepOnce(context.Background()))
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1, "sweeper 对仍在运行的任务重新 Add(kindCap=0)")
+ require.Equal(t, "sj_0123", items[0].ID)
+ // D1 钉子:sweep 的批量读取必须以索引成员的 sandbox_id 作为 expected。
+ require.Len(t, reader.calls, 1)
+ require.Equal(t, "sbx1", reader.calls[0].sandboxIDs["sj_0123"],
+ "expected 来自索引成员(D1「任务索引成员带 sandbox_id」),绝不能空——空值会走 resolveSession")
+}
+
+// D1 钉子:生产批量读取按 (会话, 沙盒) 分组、每组一次 exec、每次都带
+// ExpectedSandboxID + SkipWorkspacePrep。这是防止「无 turn resolve 立即重建」
+// (session_lifecycle.go:186-196)的核心执行点,fake 读取器测不到,必须
+// 用 fake remoteExec 直测 batchReadViaPinned。
+func TestBatchReadLocksExpectedSandbox(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ var calls []sandbox.ShellExecOptions
+ svc.remoteExec = func(_ context.Context, _ string, _ string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ calls = append(calls, opts)
+ return &sandbox.ExecuteResult{
+ ExitCode: 0,
+ Stdout: `{"meta":{"work_id":"sj_0123"},"missing":true}` + "\n" +
+ `{"meta":{"work_id":"sj_ab01"},"missing":true}`,
+ }, nil
+ }
+ snapshots, errs := svc.batchReadViaPinned(context.Background(), ref,
+ []string{"sj_0123", "sj_ab01"},
+ map[string]string{"sj_0123": "sbx-a", "sj_ab01": "sbx-b"})
+ require.Empty(t, errs)
+ require.Len(t, snapshots, 2)
+ require.Len(t, calls, 2, "按 (会话, 沙盒) 分组,每组一次 exec")
+ seen := map[string]bool{}
+ for _, opts := range calls {
+ require.NotEmpty(t, opts.ExpectedSandboxID,
+ "D1:维护 exec 绝不带空 expected——空值走 resolveSession,无 turn 时立即销毁重建沙盒")
+ require.True(t, opts.SkipWorkspacePrep, "维护路径不重做目录准备(session_manager.go SkipWorkspacePrep 注释)")
+ seen[opts.ExpectedSandboxID] = true
+ }
+ require.True(t, seen["sbx-a"] && seen["sbx-b"])
+}
+
+// A confirmed replacement is per-item lost; transient errors remain retryable.
+func TestBatchReadSandboxChangedIsPerIDLost(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.remoteExec = func(_ context.Context, _ string, _ string,
+ opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ if opts.ExpectedSandboxID == "sbx-gone" {
+ // SessionBoundManager 对 expected 不匹配返回真 Go 错误
+ //(session_manager.go:843-845),不包装进 ExecuteResult。
+ return nil, sandbox.ErrBoundSandboxChanged
+ }
+ return &sandbox.ExecuteResult{
+ ExitCode: 0,
+ Stdout: `{"meta":{"work_id":"sj_0123"},"missing":true}` + "\n" +
+ `{"meta":{"work_id":"sj_ab01"},"missing":true}`,
+ }, nil
+ }
+ snapshots, errs := svc.batchReadViaPinned(context.Background(), ref,
+ []string{"sj_0123", "sj_ab01"},
+ map[string]string{"sj_0123": "sbx-gone", "sj_ab01": "sbx-ok"})
+ require.Len(t, snapshots, 2)
+ require.ErrorContains(t, errs["sj_0123"], "bound sandbox changed")
+ require.NoError(t, errs["sj_ab01"], "组失败只判组内任务,不连累同批健康任务")
+
+ // Status 出口的翻译:沙盒被换的任务逐条 lost,健康任务照常,绝不整批报错。
+ svc.remoteRead = func(_ context.Context, _ interfaces.SessionRef, ids []string,
+ _ map[string]string,
+ ) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ return []sandbox.ShellJobSnapshot{
+ {Meta: sandbox.ShellJobMeta{ID: ids[0]}},
+ {
+ Meta: sandbox.ShellJobMeta{ID: ids[1]},
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ },
+ }, map[string]error{ids[0]: sandbox.ErrBoundSandboxChanged}
+ }
+ results, err := svc.Status(context.Background(), ref,
+ []string{"sj_0123", "sj_ab01"},
+ interfaces.StatusOptions{Detailed: true})
+ require.NoError(t, err, "confirmed loss must not obscure the healthy item")
+ require.Equal(t, interfaces.WorkStateLost, results[0].State)
+ require.Equal(t, interfaces.WorkStateRunning, results[1].State)
+}
+
+func TestBatchReadEmptySandboxIsPerIDLostWithoutExec(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ var calls int
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ calls++
+ return &sandbox.ExecuteResult{
+ ExitCode: 0,
+ Stdout: `{"meta":{"work_id":"sj_0123"},"missing":true}` + "\n" +
+ `{"meta":{"work_id":"sj_ab01"},"missing":true}`,
+ }, nil
+ }
+ snapshots, errs := svc.batchReadViaPinned(context.Background(), ref,
+ []string{"sj_0123", "sj_ab01"},
+ map[string]string{"sj_0123": "", "sj_ab01": "sbx-ok"})
+ require.Len(t, snapshots, 2)
+ require.Equal(t, 1, calls, "空 sandbox id 不发 exec")
+ require.ErrorContains(t, errs["sj_0123"], "no bound sandbox")
+ require.NoError(t, errs["sj_ab01"])
+}
+
+func TestStatusCheapReturnsUnknownPerIDAndNeverExecs(t *testing.T) {
+ svc, reader, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ // 登记表里放一个 running;信箱里放一条已结束未 Ack 的通知;第三个 ID
+ // 两处都没有(已 Ack 的终态)——三种廉价结果必须在一次调用里各自命中。
+ err := svc.bg.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell", Title: "build"}, time.Minute, 0)
+ require.NoError(t, err)
+ _, err = svc.bg.Inbox.Push(context.Background(), ref, interfaces.BackgroundNotice{
+ ID: "sj_ab01", Kind: "shell",
+ Text: "Background shell sj_ab01 succeeded: tests. Use work_wait(wait_sec=0) to read its output.",
+ FinishedAt: time.Now().UTC(),
+ })
+ require.NoError(t, err)
+
+ results, err := svc.Status(context.Background(), ref,
+ []string{"sj_0123", "sj_ab01", "sj_fedc"},
+ interfaces.StatusOptions{})
+ require.NoError(t, err, "廉价查不到不是整批错误:work_wait 只对结果里的 unknown 补详细读")
+ require.Len(t, results, 3)
+ require.Equal(t, interfaces.WorkStateRunning, results[0].State)
+ require.Equal(t, "build", results[0].Title)
+ require.Equal(t, interfaces.WorkStateSucceeded, results[1].State, "从通知文本解析终态")
+ require.True(t, results[1].Terminal)
+ require.Equal(t, interfaces.WorkStateUnknown, results[2].State,
+ "两处都没有 → 逐条 unknown,work_wait 的 poll 立即补一次 Detailed=true(work.go 只认这个)")
+ require.Zero(t, reader.execCalls, "Detailed=false 绝不发 exec")
+
+ // 格式非法才整批拒绝(接口契约允许的 ErrWorkNotFound 用法)。
+ _, err = svc.Status(context.Background(), ref, []string{"sa_0123456789ab"}, interfaces.StatusOptions{})
+ require.ErrorIs(t, err, interfaces.ErrWorkNotFound)
+}
+
+func TestApplyOutputFilesReportsNewDeliverables(t *testing.T) {
+ mod := time.Unix(100, 0)
+ svc := &ShellJobService{
+ listFiles: func(context.Context, string, string) ([]sandbox.RemoteDirEntry, error) {
+ return []sandbox.RemoteDirEntry{{
+ Name: "b.txt", Path: "/workspace/output/b.txt",
+ Type: sandbox.RemoteEntryFile, Size: 3, ModTime: mod,
+ }}, nil
+ },
+ }
+ meta := sandbox.ShellJobMeta{OutputDir: "/workspace/output", OutputSeen: true}
+ result := interfaces.WorkResult{
+ State: interfaces.WorkStateSucceeded, Terminal: true, Payload: ShellJobPayload{},
+ }
+ got := svc.applyOutputFiles(context.Background(), interfaces.SessionRef{SessionID: "s"}, result, meta)
+ require.Equal(t, []string{"sandbox:b.txt"}, got.OutputFiles)
+}
+
+func TestShellJobResultNotes(t *testing.T) {
+ snap := sandbox.ShellJobSnapshot{
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ }
+ r := ShellJobResult(snap, false)
+ require.Equal(t, interfaces.WorkStateRunning, r.State)
+ require.False(t, r.Terminal)
+ payload := r.Payload.(ShellJobPayload)
+ require.Contains(t, payload.Note, "Call work_wait only when you cannot continue")
+ require.Contains(t, payload.Note, "Do not pipe the command through tail or head")
+ r = ShellJobResult(snap, true)
+ payload = r.Payload.(ShellJobPayload)
+ require.Contains(t, payload.Note, "WeKnora Lite is closed")
+ snap.Status.State = interfaces.WorkStateLost
+ payload = ShellJobResult(snap, false).Payload.(ShellJobPayload)
+ require.Contains(t, payload.Note, "watcher died")
+}
+
+func TestShellJobDurationKeepsFractionalSeconds(t *testing.T) {
+ start, err := time.Parse(time.RFC3339Nano, "2026-10-18T04:39:01.100Z")
+ require.NoError(t, err)
+ end, err := time.Parse(time.RFC3339Nano, "2026-10-18T04:39:01.340Z")
+ require.NoError(t, err)
+ got := shellJobDurationMS(sandbox.ShellJobStatus{StartedAt: start, EndedAt: &end})
+ require.Equal(t, int64(240), got)
+}
+
+func TestSkillEnvCaptureRunsOnlyAfterSuccess(t *testing.T) {
+ svc := &ShellJobService{}
+ ref := interfaces.SessionRef{TenantID: 1, SessionID: "session-1"}
+ calls := 0
+ hold := func() {
+ svc.HoldSkillEnvCapture(context.Background(), ref, "sj_ab12", func(context.Context) { calls++ })
+ }
+ hold()
+ svc.skillCapture.settle(context.Background(), ref, "sj_ab12", interfaces.WorkStateFailed)
+ require.Zero(t, calls)
+ hold()
+ svc.skillCapture.drop(ref, "sj_ab12")
+ svc.skillCapture.settle(context.Background(), ref, "sj_ab12", interfaces.WorkStateSucceeded)
+ require.Zero(t, calls, "lost or cancelled jobs drop the pending write")
+ hold()
+ svc.skillCapture.dropSession(ref)
+ svc.skillCapture.settle(context.Background(), ref, "sj_ab12", interfaces.WorkStateSucceeded)
+ require.Zero(t, calls, "deleting the session drops the pending write")
+ hold()
+ svc.skillCapture.settle(context.Background(), ref, "sj_ab12", interfaces.WorkStateSucceeded)
+ require.Equal(t, 1, calls)
+}
+
+// A lost submit epilogue must not turn an already-finished job into background
+// work. That used to hand the model the result inline and then deliver a
+// completion notice for the same id, which cost it an extra reasoning round.
+func TestSubmitReportsAFinishedJobInlineWhenTheEpilogueIsLost(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ exit := 0
+ svc.remoteRead = func(_ context.Context, _ interfaces.SessionRef,
+ ids []string, _ map[string]string,
+ ) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ out := make([]sandbox.ShellJobSnapshot, len(ids))
+ for i, id := range ids {
+ out[i] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: id, TenantID: ref.TenantID, SessionID: ref.SessionID, Title: "t",
+ },
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateSucceeded, ExitCode: &exit,
+ },
+ WatcherAlive: true,
+ }
+ }
+ return out, nil
+ }
+
+ // No marker is ever fed to OnOutput, so ParseShellJobEpilogue fails and the
+ // submit cannot know whether the job finished inside the yield window.
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.False(t, sub.Background, "a finished job must be answered inline")
+ require.Equal(t, interfaces.WorkStateSucceeded, sub.Result.State)
+
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, items, "no activity registration: the sweeper would report it again")
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Empty(t, due, "no index member: nothing left for the sweeper to find")
+ pending, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Empty(t, pending, "no completion notice for a result already returned")
+}
+
+// The same lost epilogue must still register a job that is genuinely running.
+func TestSubmitRegistersAJobWhoseEpilogueIsLostButStillRuns(t *testing.T) {
+ svc, _, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.remoteRead = func(_ context.Context, _ interfaces.SessionRef,
+ ids []string, _ map[string]string,
+ ) ([]sandbox.ShellJobSnapshot, map[string]error) {
+ out := make([]sandbox.ShellJobSnapshot, len(ids))
+ for i, id := range ids {
+ out[i] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: id, TenantID: ref.TenantID, SessionID: ref.SessionID, Title: "t",
+ },
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, StartedAt: time.Now().UTC(),
+ },
+ WatcherAlive: true,
+ }
+ }
+ return out, nil
+ }
+
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "sleep 30", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.True(t, sub.Background, "a running job must stay background work")
+
+ items, err := bg.Registry.List(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ due, err := index.Due(context.Background(), time.Now().Add(time.Hour))
+ require.NoError(t, err)
+ require.Len(t, due, 1)
+}
+
+// stderr that looks like an epilogue must not be parsed as the job's status.
+// A later replica-silent backend (Cube) also has to be parsed from the
+// buffered exec result, and that buffer must not be fed a second time.
+func TestSubmitIgnoresStderrWhenParsingTheEpilogue(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.remoteRead = func(context.Context, interfaces.SessionRef, []string, map[string]string) (
+ []sandbox.ShellJobSnapshot, map[string]error,
+ ) {
+ t.Fatal("a parsed epilogue must not fall through to a status read")
+ return nil, nil
+ }
+ svc.remoteExec = func(
+ _ context.Context, _ string, script string, opts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ marker := shellJobMarkerInScript.FindString(script)
+ require.NotEmpty(t, marker)
+ stdout := "hello\n" + marker + "\n" + `{"state":"succeeded","exit_code":0}` + "\n4 5\n"
+ if opts.OnOutput != nil {
+ opts.OnOutput("stdout", []byte(stdout))
+ opts.OnOutput("stderr", []byte("noise\n1 2\n"))
+ }
+ // Already delivered via OnOutput. Re-feeding this would replace the offsets.
+ return &sandbox.ExecuteResult{ExitCode: 0, Stdout: "9 9\n", Stderr: "1 2\n"}, nil
+ }
+
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.False(t, sub.Background)
+ require.Equal(t, interfaces.WorkStateSucceeded, sub.Result.State)
+ payload := sub.Result.Payload.(ShellJobPayload)
+ require.Contains(t, payload.Stdout, "hello")
+ require.NotContains(t, payload.Stdout, "noise")
+ require.Contains(t, payload.Stderr, "noise")
+ require.Equal(t, int64(4), payload.TotalBytes.Stdout)
+ require.Equal(t, int64(5), payload.TotalBytes.Stderr)
+ require.NotNil(t, payload.ExitCode)
+ require.Zero(t, *payload.ExitCode)
+}
+
+func TestSubmitParsesEpilogueFromExecResultWhenOnOutputIsSilent(t *testing.T) {
+ svc, _, _, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ svc.remoteRead = func(context.Context, interfaces.SessionRef, []string, map[string]string) (
+ []sandbox.ShellJobSnapshot, map[string]error,
+ ) {
+ t.Fatal("buffered stdout must be parsed without a status read")
+ return nil, nil
+ }
+ svc.remoteExec = func(
+ _ context.Context, _ string, script string, _ sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ marker := shellJobMarkerInScript.FindString(script)
+ require.NotEmpty(t, marker)
+ stdout := "hello\n" + marker + "\n" + `{"state":"succeeded","exit_code":0}` + "\n4 5\n"
+ return &sandbox.ExecuteResult{ExitCode: 0, Stdout: stdout, Stderr: "noise\n"}, nil
+ }
+
+ sub, err := svc.SubmitShellJob(context.Background(), ref, tools.ShellJobSubmitRequest{
+ Command: "true", YieldTime: time.Second, RunID: "m1",
+ })
+ require.NoError(t, err)
+ require.False(t, sub.Background)
+ require.Equal(t, interfaces.WorkStateSucceeded, sub.Result.State)
+ payload := sub.Result.Payload.(ShellJobPayload)
+ require.Contains(t, payload.Stdout, "hello")
+ require.Contains(t, payload.Stderr, "noise")
+ require.Equal(t, int64(4), payload.TotalBytes.Stdout)
+}
+
+func TestSweepSettlesSkillCaptureFromRecordedOutcome(t *testing.T) {
+ sweeper, reader, index, bg := newRemoteServiceFixture(t)
+ submitter := NewShellJobService(nil, bg, index, nil, keeperFunc(
+ func(context.Context, interfaces.SessionRef) error { return nil }))
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ ctx := context.Background()
+ calls := 0
+ submitter.HoldSkillEnvCapture(ctx, ref, "sj_0123", func(context.Context) { calls++ })
+
+ exit := 0
+ now := time.Now().UTC()
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "sbx1"},
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateSucceeded, ExitCode: &exit, StartedAt: now,
+ },
+ }
+ require.NoError(t, index.Add(ctx, ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: now.Add(-time.Minute),
+ }))
+ require.NoError(t, sweeper.SweepOnce(ctx))
+ require.Zero(t, calls, "the replica that finishes the job does not hold the credential callback")
+ require.NoError(t, submitter.SweepOnce(ctx))
+ require.Equal(t, 1, calls, "the submitting replica applies the recorded success")
+ require.NoError(t, submitter.SweepOnce(ctx))
+ require.Equal(t, 1, calls, "a later sweep must not run the write-back again")
+
+ failed := 0
+ submitter.HoldSkillEnvCapture(ctx, ref, "sj_ab12", func(context.Context) { failed++ })
+ require.NoError(t, index.RecordOutcome(ctx, ref, "sj_ab12", interfaces.WorkStateFailed))
+ require.NoError(t, submitter.SweepOnce(ctx))
+ require.Zero(t, failed, "a failed outcome drops the write-back")
+ require.NoError(t, index.RecordOutcome(ctx, ref, "sj_ab12", interfaces.WorkStateSucceeded))
+ require.NoError(t, submitter.SweepOnce(ctx))
+ require.Zero(t, failed, "the dropped write-back is not revived by a later success")
+}
+
+func TestShellJobCursorIsSharedAcrossReplicas(t *testing.T) {
+ svcA, _, index, bg := newRemoteServiceFixture(t)
+ svcB := NewShellJobService(nil, bg, index, nil, keeperFunc(
+ func(context.Context, interfaces.SessionRef) error { return nil }))
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ const id = "sj_0123"
+ svcA.remoteRead = func(_ context.Context, _ interfaces.SessionRef, ids []string, _ map[string]string) (
+ []sandbox.ShellJobSnapshot, map[string]error,
+ ) {
+ out := make([]sandbox.ShellJobSnapshot, len(ids))
+ for i, jobID := range ids {
+ out[i] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{
+ ID: jobID, TenantID: ref.TenantID, SessionID: ref.SessionID, SandboxID: "sbx1",
+ },
+ Status: sandbox.ShellJobStatus{State: interfaces.WorkStateRunning},
+ Stdout: sandbox.ShellJobStream{Text: "hello", TotalBytes: 5},
+ WatcherAlive: true,
+ }
+ }
+ return out, nil
+ }
+ _, err := svcA.Status(context.Background(), ref, []string{id}, interfaces.StatusOptions{
+ Detailed: true, RunID: "m1",
+ })
+ require.NoError(t, err)
+
+ var script string
+ var opts sandbox.ShellExecOptions
+ svcB.remoteExec = func(
+ _ context.Context, _ string, got string, gotOpts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ script = got
+ opts = gotOpts
+ return &sandbox.ExecuteResult{ExitCode: 0, Stdout: `{"meta":{"work_id":"sj_0123"},"missing":true}`}, nil
+ }
+ readCtx := context.WithValue(context.Background(), shellReadRunKey{}, "m1")
+ _, errs := svcB.batchReadViaPinned(readCtx, ref, []string{id}, map[string]string{id: "sbx1"})
+ require.Empty(t, errs)
+ require.Contains(t, script, "SF='5'", "the other replica must resume at the cursor this one stored")
+ require.Equal(t, "sbx1", opts.ExpectedSandboxID)
+ require.True(t, opts.SkipWorkspacePrep)
+}
+
+func TestSweepKillsOrphansWhenTheWatcherDies(t *testing.T) {
+ svc, reader, index, bg := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "sbx1"},
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, StartedAt: now, PGID: 99, TouchPID: 77,
+ },
+ WatcherAlive: false,
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "sbx1", YieldUntil: now.Add(-time.Minute),
+ }))
+ var script string
+ var opts sandbox.ShellExecOptions
+ svc.remoteExec = func(
+ _ context.Context, _ string, got string, gotOpts sandbox.ShellExecOptions,
+ ) (*sandbox.ExecuteResult, error) {
+ script = got
+ opts = gotOpts
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ require.NoError(t, svc.SweepOnce(context.Background()))
+ require.Contains(t, script, "kill -TERM")
+ require.Contains(t, script, "99")
+ require.Contains(t, script, "77")
+ require.Contains(t, script, "weknora-sandbox-activity")
+ require.Equal(t, "sbx1", opts.ExpectedSandboxID)
+ require.True(t, opts.SkipWorkspacePrep)
+ notices, err := bg.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ require.Len(t, notices, 1)
+ require.Contains(t, notices[0].Text, "lost")
+}
+
+func TestSweepDoesNotKillInsideAReplacedSandbox(t *testing.T) {
+ svc, reader, index, _ := newRemoteServiceFixture(t)
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ now := time.Now().UTC()
+ reader.snapshots["sj_0123"] = sandbox.ShellJobSnapshot{
+ Meta: sandbox.ShellJobMeta{ID: "sj_0123", TenantID: 7, SessionID: "s", SandboxID: "old"},
+ Status: sandbox.ShellJobStatus{
+ State: interfaces.WorkStateRunning, StartedAt: now, PGID: 99, TouchPID: 77,
+ },
+ WatcherAlive: false,
+ }
+ require.NoError(t, index.Add(context.Background(), ShellJobIndexEntry{
+ SessionRef: ref, WorkID: "sj_0123", SandboxID: "new", YieldUntil: now.Add(-time.Minute),
+ }))
+ called := false
+ svc.remoteExec = func(context.Context, string, string, sandbox.ShellExecOptions) (*sandbox.ExecuteResult, error) {
+ called = true
+ return &sandbox.ExecuteResult{ExitCode: 0}, nil
+ }
+ require.NoError(t, svc.SweepOnce(context.Background()))
+ require.False(t, called, "a replaced sandbox is already gone; do not exec a kill into the new one")
+}
+
+func miniredisRunT(t *testing.T) *miniredis.Miniredis { return miniredis.RunT(t) }
+
+func redisNewClient(mr *miniredis.Miniredis) *redis.Client {
+ return redis.NewClient(&redis.Options{Addr: mr.Addr()})
+}
diff --git a/internal/background/listeners.go b/internal/background/listeners.go
index 457841535..cdd7da66e 100644
--- a/internal/background/listeners.go
+++ b/internal/background/listeners.go
@@ -2,6 +2,7 @@ package background
import (
"context"
+ "errors"
"sync"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -54,13 +55,19 @@ func (l *Listeners) NotifyStop(ctx context.Context, s interfaces.SessionRef) {
}
// NotifyDelete calls OnDelete on every listener, then clears the session's
-// activity items and notices. Errors are logged, never returned.
-func (l *Listeners) NotifyDelete(ctx context.Context, s interfaces.SessionRef) {
+// activity items and notices. A failed stop preserves shared state and keeps
+// the session available for a retry, especially on a retained shared machine.
+func (l *Listeners) NotifyDelete(ctx context.Context, s interfaces.SessionRef) error {
+ var failures []error
for _, listener := range l.snapshot() {
if err := listener.OnDelete(ctx, s); err != nil {
l.logf(ctx, "[Background] OnDelete listener failed session=%s: %v", s.SessionID, err)
+ failures = append(failures, err)
}
}
+ if err := errors.Join(failures...); err != nil {
+ return err
+ }
if l.registry != nil {
items, err := l.registry.List(ctx, s)
if err != nil {
@@ -78,4 +85,19 @@ func (l *Listeners) NotifyDelete(ctx context.Context, s interfaces.SessionRef) {
l.logf(ctx, "[Background] clear inbox on delete failed session=%s: %v", s.SessionID, err)
}
}
+ return nil
+}
+
+// NotifyDeleteAborted reopens submission gates when a listener or the session
+// repository rejected deletion. Already stopped jobs remain stopped.
+func (l *Listeners) NotifyDeleteAborted(ctx context.Context, s interfaces.SessionRef) {
+ for _, listener := range l.snapshot() {
+ if rollback, ok := listener.(interface {
+ OnDeleteAborted(context.Context, interfaces.SessionRef) error
+ }); ok {
+ if err := rollback.OnDeleteAborted(ctx, s); err != nil {
+ l.logf(ctx, "[Background] reopen session after failed deletion session=%s: %v", s.SessionID, err)
+ }
+ }
+ }
}
diff --git a/internal/container/background.go b/internal/container/background.go
index 57f4c8b2c..f8a36b6e5 100644
--- a/internal/container/background.go
+++ b/internal/container/background.go
@@ -2,6 +2,14 @@
package container
import (
+ "context"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/redis/go-redis/v9"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"
"github.com/Tencent/WeKnora/internal/background"
"github.com/Tencent/WeKnora/internal/sandbox"
@@ -38,3 +46,55 @@ func newBackgroundNoticeInbox(streams interfaces.StreamManager) interfaces.Notic
}
return background.NewMemoryNoticeInbox(nil)
}
+
+// redisNamespace matches the session-binding store: WEKNORA_REDIS_NAMESPACE,
+// or "weknora" when that variable is empty.
+func redisNamespace() string {
+ namespace := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_NAMESPACE"))
+ if namespace == "" {
+ return "weknora"
+ }
+ return namespace
+}
+
+// newRemoteShellJobService wires the shell feature into BackgroundWork:
+// Handler, the throttled keep-aliver, and (when Redis is configured) the job
+// index. Dig provides the build-tagged newShellJobService; the web stub
+// forwards here. The agent service depends on that constructor, so dig builds
+// it first.
+func newRemoteShellJobService(
+ rdb *redis.Client,
+ pinned *service.PinnedSessionSandbox,
+ bg *service.BackgroundWork,
+ streams interfaces.StreamManager,
+) *service.ShellJobService {
+ var index *service.ShellJobIndex
+ if rdb != nil {
+ index = service.NewShellJobIndex(rdb, redisNamespace())
+ }
+ // A nil stream manager must stay a nil pointer. Storing (*Emitter)(nil)
+ // in the lifecycle's event interface panics on the first call.
+ var events *service.BackgroundEventEmitter
+ if streams != nil {
+ events = service.NewBackgroundEventEmitter(streams)
+ }
+ svc := service.NewShellJobService(pinned, bg, index, events, service.NewBackgroundKeepAliver(pinned))
+ if bg != nil {
+ bg.Handler = svc
+ }
+ return svc
+}
+
+// startShellJobSweeper runs the remote handoff loop. Lite has no Redis index,
+// so RemoteAvailable is false and nothing is started.
+func startShellJobSweeper(svc *service.ShellJobService, cleaner interfaces.ResourceCleaner) {
+ if svc == nil || !svc.RemoteAvailable() || cleaner == nil {
+ return
+ }
+ sweeperCtx, stop := context.WithCancel(context.Background())
+ svc.StartSweeper(sweeperCtx, 5*time.Second)
+ cleaner.RegisterWithName("ShellJobSweeper", func() error {
+ stop()
+ return nil
+ })
+}
diff --git a/internal/container/background_shell_desktop.go b/internal/container/background_shell_desktop.go
new file mode 100644
index 000000000..916188fbd
--- /dev/null
+++ b/internal/container/background_shell_desktop.go
@@ -0,0 +1,61 @@
+//go:build desktop
+
+package container
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/redis/go-redis/v9"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/localsandbox"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// newShellJobService wires Lite shell jobs. Remote sandboxes are hidden, so
+// the index stays nil. A nil stream manager stays a nil emitter pointer:
+// storing (*BackgroundEventEmitter)(nil) in the lifecycle event interface
+// panics on the first call. The keep-aliver matches the web wiring;
+// NewBackgroundKeepAliver is a no-op for a host sandbox, and a nil keeper
+// fails the remote submit path.
+func newShellJobService(
+ _ *redis.Client,
+ pinned *service.PinnedSessionSandbox,
+ bg *service.BackgroundWork,
+ streams interfaces.StreamManager,
+ cleaner interfaces.ResourceCleaner,
+ host service.HostSandboxManager,
+) (*service.ShellJobService, error) {
+ var events *service.BackgroundEventEmitter
+ if streams != nil {
+ events = service.NewBackgroundEventEmitter(streams)
+ }
+ jobs := service.NewShellJobService(pinned, bg, nil, events, service.NewBackgroundKeepAliver(pinned))
+ if bg != nil {
+ bg.Handler = jobs
+ }
+ provider, ok := host.Manager.(interface{ LocalSandboxService() *localsandbox.Service })
+ if !ok || provider.LocalSandboxService() == nil {
+ return jobs, nil
+ }
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return nil, err
+ }
+ local, err := service.NewLocalShellJobService(
+ provider.LocalSandboxService(), bg, events, filepath.Join(home, ".weknora", "jobs"),
+ )
+ if err != nil {
+ return nil, err
+ }
+ jobs.WithLocal(local)
+ cleaner.RegisterWithName("LocalShellJobs", func() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ return local.Shutdown(ctx)
+ })
+ return jobs, nil
+}
diff --git a/internal/container/background_shell_desktop_test.go b/internal/container/background_shell_desktop_test.go
new file mode 100644
index 000000000..6d7449479
--- /dev/null
+++ b/internal/container/background_shell_desktop_test.go
@@ -0,0 +1,20 @@
+//go:build desktop
+
+package container
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/background"
+)
+
+func TestDesktopWithoutHostManagerStaysRemoteOnly(t *testing.T) {
+ bg := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ jobs, err := newShellJobService(nil, nil, bg, nil, NewResourceCleaner(),
+ service.HostSandboxManager{Desktop: true})
+ require.NoError(t, err)
+ require.False(t, jobs.LocalAvailable())
+}
diff --git a/internal/container/background_shell_stub.go b/internal/container/background_shell_stub.go
new file mode 100644
index 000000000..58b5e663a
--- /dev/null
+++ b/internal/container/background_shell_stub.go
@@ -0,0 +1,24 @@
+//go:build !desktop
+
+package container
+
+import (
+ "github.com/redis/go-redis/v9"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// newShellJobService is the web constructor dig provides. Host wiring and the
+// local-job cleaner are desktop-only, so this build ignores them and forwards
+// to the remote service.
+func newShellJobService(
+ rdb *redis.Client,
+ pinned *service.PinnedSessionSandbox,
+ bg *service.BackgroundWork,
+ streams interfaces.StreamManager,
+ _ interfaces.ResourceCleaner,
+ _ service.HostSandboxManager,
+) (*service.ShellJobService, error) {
+ return newRemoteShellJobService(rdb, pinned, bg, streams), nil
+}
diff --git a/internal/container/background_test.go b/internal/container/background_test.go
index 6f6cdc593..749bf80d0 100644
--- a/internal/container/background_test.go
+++ b/internal/container/background_test.go
@@ -5,9 +5,13 @@ import (
"testing"
"time"
+ "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"
+ "github.com/Tencent/WeKnora/internal/background"
"github.com/Tencent/WeKnora/internal/sandbox"
"github.com/Tencent/WeKnora/internal/sandbox/allocation"
+ "github.com/Tencent/WeKnora/internal/stream"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/alicebob/miniredis/v2"
@@ -70,3 +74,91 @@ func TestBackgroundActivityRegistryUsesRecordedUserAllocation(t *testing.T) {
_, err = store.TryLockRewind(ctx, key)
require.ErrorIs(t, err, sandbox.ErrSessionWorkActive)
}
+
+func TestBackgroundActivityRegistryUsesRedisBindingStore(t *testing.T) {
+ mini := miniredis.RunT(t)
+ client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
+ t.Cleanup(func() { _ = client.Close() })
+
+ store := newSessionBindingStore(client)
+ require.IsType(t, &sandbox.RedisSessionSandboxBindingStore{}, store)
+ require.IsType(t, &sandbox.RedisActivityRegistry{}, newBackgroundActivityRegistry(store, nil))
+}
+
+func TestBackgroundActivityRegistryIsProcessLocalWithoutRedis(t *testing.T) {
+ store := newSessionBindingStore(nil)
+ require.IsType(t, &sandbox.MemorySessionSandboxBindingStore{}, store)
+ require.IsType(t, &sandbox.MemoryActivityRegistry{}, newBackgroundActivityRegistry(store, nil))
+}
+
+func TestBackgroundActivityRegistryWithoutBindingStoreIsProcessLocal(t *testing.T) {
+ require.IsType(t, &background.MemoryActivityRegistry{}, newBackgroundActivityRegistry(nil, nil))
+}
+
+func TestBackgroundNoticeInboxFollowsTheStreamManager(t *testing.T) {
+ mini := miniredis.RunT(t)
+ redisStreams, err := stream.NewRedisStreamManager(mini.Addr(), "", "", 0, "test", time.Hour)
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = redisStreams.Close() })
+ require.IsType(t, &stream.RedisNoticeInbox{}, newBackgroundNoticeInbox(redisStreams))
+
+ require.IsType(t, &background.MemoryNoticeInbox{}, newBackgroundNoticeInbox(stream.NewMemoryStreamManager()))
+ require.IsType(t, &background.MemoryNoticeInbox{}, newBackgroundNoticeInbox(nil))
+}
+
+type nameCleaner struct {
+ names []string
+ stops []types.CleanupFunc
+}
+
+func (c *nameCleaner) Register(fn types.CleanupFunc) { c.RegisterWithName("", fn) }
+
+func (c *nameCleaner) RegisterWithName(name string, fn types.CleanupFunc) {
+ c.names = append(c.names, name)
+ c.stops = append(c.stops, fn)
+}
+
+func (c *nameCleaner) Promote(string) {}
+
+func (c *nameCleaner) Cleanup(context.Context) []error {
+ var errs []error
+ for _, fn := range c.stops {
+ if err := fn(); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errs
+}
+
+func TestNewShellJobServiceSetsHandler(t *testing.T) {
+ bg := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ svc := newRemoteShellJobService(nil, nil, bg, nil)
+ require.Equal(t, svc, bg.Handler)
+ require.False(t, svc.RemoteAvailable())
+ _, err := svc.SubmitShellJob(context.Background(), interfaces.SessionRef{TenantID: 1, SessionID: "s"},
+ tools.ShellJobSubmitRequest{Command: "true"})
+ require.Error(t, err)
+}
+
+func TestShellJobSweeperStartsOnlyWhenRemoteAvailable(t *testing.T) {
+ bg := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ local := newRemoteShellJobService(nil, nil, bg, stream.NewMemoryStreamManager())
+ cleaner := &nameCleaner{}
+ startShellJobSweeper(local, cleaner)
+ require.Empty(t, cleaner.names)
+
+ mini := miniredis.RunT(t)
+ client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
+ t.Cleanup(func() { _ = client.Close() })
+ remoteBG := service.NewBackgroundWork(
+ background.NewMemoryActivityRegistry(nil),
+ background.NewMemoryNoticeInbox(nil),
+ )
+ remote := newRemoteShellJobService(client, &service.PinnedSessionSandbox{}, remoteBG, nil)
+ require.True(t, remote.RemoteAvailable())
+ startShellJobSweeper(remote, cleaner)
+ require.Equal(t, []string{"ShellJobSweeper"}, cleaner.names)
+ for _, err := range cleaner.Cleanup(context.Background()) {
+ require.NoError(t, err)
+ }
+}
diff --git a/internal/container/container.go b/internal/container/container.go
index 3151e2875..5c185161b 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -437,6 +437,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
) *service.PinnedSessionSandbox {
return service.NewPinnedSessionSandbox(pinner, resolver, mgr, host)
}))
+ must(container.Provide(newShellJobService))
must(container.Provide(func(
mgr sandbox.Manager,
pinned *service.PinnedSessionSandbox,
@@ -810,6 +811,14 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Invoke(func(cleaner interfaces.ResourceCleaner, _ *browserskill.Manager) {
cleaner.Promote("BrowserSkill")
}))
+ must(container.Invoke(startShellJobSweeper))
+ // Background shell jobs shut down before the database and streams: their
+ // watchers write terminal state locally and must not race the cleanup that
+ // follows. Cleanup runs in reverse, so promote runs them first. A missing
+ // name is a no-op, so the web build can invoke this without a local hook.
+ must(container.Invoke(func(cleaner interfaces.ResourceCleaner, _ *service.ShellJobService) {
+ cleaner.Promote("LocalShellJobs")
+ }))
logger.Infof(ctx, "[Container] Container initialization completed successfully")
return container
diff --git a/internal/handler/session/background_http.go b/internal/handler/session/background_http.go
new file mode 100644
index 000000000..93055b69c
--- /dev/null
+++ b/internal/handler/session/background_http.go
@@ -0,0 +1,230 @@
+package session
+
+import (
+ "context"
+ "net/http"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/errors"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/gin-gonic/gin"
+)
+
+type backgroundWorkItem struct {
+ WorkID string `json:"work_id"`
+ Kind string `json:"kind"`
+ Title string `json:"title"`
+ State string `json:"state"`
+ StartedAt string `json:"started_at,omitempty"`
+ FinishedAt string `json:"finished_at,omitempty"`
+ ParentToolCallID string `json:"parent_tool_call_id,omitempty"`
+}
+
+// ListBackgroundWork returns the session's background items: live ones from the
+// activity registry plus notices finished within the last 24h. Terminal states
+// come from Handler.Status with Detailed false. Nothing here execs a sandbox.
+func (h *Handler) ListBackgroundWork(c *gin.Context) {
+ sessionID := paramSessionID(c)
+ ctx := c.Request.Context()
+ session, err := h.sessionService.GetSession(ctx, sessionID)
+ if err != nil || session == nil {
+ writeNotFound(c, "session not found")
+ return
+ }
+ items := make([]backgroundWorkItem, 0)
+ if h.background == nil || !h.background.Enabled() {
+ c.JSON(http.StatusOK, gin.H{"success": true, "data": items})
+ return
+ }
+ ref := sessionWorkRef(session)
+ live, err := h.background.Registry.List(ctx, ref)
+ if err != nil {
+ logger.Warnf(ctx, "list background work session=%s: %v", sessionID, err)
+ writeUnavailable(c, "background work is temporarily unavailable")
+ return
+ }
+ liveSet := make(map[string]struct{}, len(live))
+ for _, activity := range live {
+ liveSet[activity.ID] = struct{}{}
+ items = append(items, backgroundWorkItem{
+ WorkID: activity.ID,
+ Kind: activity.Kind,
+ Title: activity.Title,
+ State: interfaces.WorkStateRunning,
+ StartedAt: formatWorkTime(activity.StartedAt),
+ })
+ }
+ notices, err := h.background.Inbox.Pending(ctx, ref)
+ if err != nil {
+ logger.Warnf(ctx, "pending background notices session=%s: %v", sessionID, err)
+ writeUnavailable(c, "background work is temporarily unavailable")
+ return
+ }
+ terminalIDs := make([]string, 0, len(notices))
+ for _, notice := range notices {
+ if _, ok := liveSet[notice.ID]; ok || notice.ID == "" {
+ continue
+ }
+ terminalIDs = append(terminalIDs, notice.ID)
+ }
+ resolved := h.terminalWorkResults(ctx, ref, terminalIDs)
+ for _, notice := range notices {
+ if _, ok := liveSet[notice.ID]; ok {
+ continue
+ }
+ result, ok := resolved[notice.ID]
+ state, title := terminalWorkView(notice, result, ok)
+ items = append(items, backgroundWorkItem{
+ WorkID: notice.ID,
+ Kind: notice.Kind,
+ Title: title,
+ State: state,
+ FinishedAt: formatWorkTime(notice.FinishedAt),
+ })
+ }
+ c.JSON(http.StatusOK, gin.H{"success": true, "data": items})
+}
+
+// GetBackgroundWork returns one item's detail with output. RunID stays empty:
+// a user-triggered read must not move the model's incremental cursor.
+func (h *Handler) GetBackgroundWork(c *gin.Context) {
+ sessionID := paramSessionID(c)
+ workID := c.Param("work_id")
+ ctx := c.Request.Context()
+ session, err := h.sessionService.GetSession(ctx, sessionID)
+ if err != nil || session == nil {
+ writeNotFound(c, "session not found")
+ return
+ }
+ if h.background == nil || !h.background.Enabled() {
+ writeNotFound(c, "background work is not available")
+ return
+ }
+ ref := sessionWorkRef(session)
+ results, err := h.background.Handler.Status(ctx, ref, []string{workID}, interfaces.StatusOptions{
+ Detailed: true,
+ RunID: "",
+ })
+ if err != nil || len(results) != 1 {
+ writeNotFound(c, "background work not found")
+ return
+ }
+ id := results[0].ID
+ if id == "" {
+ id = workID
+ }
+ c.JSON(http.StatusOK, gin.H{"success": true, "data": gin.H{
+ "work_id": id,
+ "state": results[0].State,
+ "title": results[0].Title,
+ "result": results[0].Payload,
+ }})
+}
+
+// CancelBackgroundWork stops one item. Strictly owner-only: mutations must not
+// honour admin read access (same GetOwnedSession rule as steer).
+func (h *Handler) CancelBackgroundWork(c *gin.Context) {
+ sessionID := paramSessionID(c)
+ workID := c.Param("work_id")
+ ctx := c.Request.Context()
+ session, err := h.sessionService.GetOwnedSession(ctx, sessionID)
+ if err != nil || session == nil {
+ writeNotFound(c, "session not found")
+ return
+ }
+ if h.background == nil || !h.background.Enabled() {
+ writeNotFound(c, "background work is not available")
+ return
+ }
+ ref := sessionWorkRef(session)
+ results, err := h.background.Handler.Cancel(ctx, ref, []string{workID})
+ if err != nil || len(results) != 1 {
+ writeNotFound(c, "background work not found")
+ return
+ }
+ id := results[0].ID
+ if id == "" {
+ id = workID
+ }
+ c.JSON(http.StatusOK, gin.H{"success": true, "data": gin.H{
+ "work_id": id,
+ "state": results[0].State,
+ }})
+}
+
+func sessionWorkRef(session *types.Session) interfaces.SessionRef {
+ return interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID}
+}
+
+func (h *Handler) terminalWorkResults(
+ ctx context.Context, ref interfaces.SessionRef, ids []string,
+) map[string]interfaces.WorkResult {
+ out := make(map[string]interfaces.WorkResult, len(ids))
+ if len(ids) == 0 || h.background == nil || h.background.Handler == nil {
+ return out
+ }
+ results, err := h.background.Handler.Status(ctx, ref, ids, interfaces.StatusOptions{
+ Detailed: false,
+ RunID: "",
+ })
+ if err != nil {
+ logger.Warnf(ctx, "background work status session=%s: %v", ref.SessionID, err)
+ return out
+ }
+ for _, result := range results {
+ out[result.ID] = result
+ }
+ return out
+}
+
+func terminalWorkView(
+ notice interfaces.BackgroundNotice, result interfaces.WorkResult, ok bool,
+) (string, string) {
+ if ok && result.State != "" && result.State != interfaces.WorkStateUnknown {
+ return result.State, result.Title
+ }
+ if parsed := service.ParseShellJobNoticeState(notice.Text); parsed != "" {
+ if ok {
+ return parsed, result.Title
+ }
+ return parsed, ""
+ }
+ if ok {
+ return interfaces.WorkStateLost, result.Title
+ }
+ return interfaces.WorkStateLost, ""
+}
+
+func formatWorkTime(t time.Time) string {
+ if t.IsZero() {
+ return ""
+ }
+ return t.UTC().Format(time.RFC3339)
+}
+
+// writeNotFound returns HTTP 404 itself. Session routes in production also
+// install ErrorHandler, which only writes when c.Errors is set; writing here
+// keeps a router that has no error middleware on the same status.
+func writeNotFound(c *gin.Context, message string) {
+ writeAppError(c, errors.NewNotFoundError(message))
+}
+
+// writeUnavailable is a retryable failure. An empty 200 would clear the chip
+// and stop the client's poll, while the jobs are still running.
+func writeUnavailable(c *gin.Context, message string) {
+ writeAppError(c, errors.NewServiceUnavailableError(message))
+}
+
+func writeAppError(c *gin.Context, appErr *errors.AppError) {
+ c.AbortWithStatusJSON(appErr.HTTPCode, gin.H{
+ "success": false,
+ "error": gin.H{
+ "code": appErr.Code,
+ "message": appErr.Message,
+ "details": appErr.Details,
+ },
+ })
+}
diff --git a/internal/handler/session/background_http_test.go b/internal/handler/session/background_http_test.go
new file mode 100644
index 000000000..f4a8890f0
--- /dev/null
+++ b/internal/handler/session/background_http_test.go
@@ -0,0 +1,253 @@
+package session
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+)
+
+type httpWorkHandler struct {
+ interfaces.WorkHandler
+ status map[string]interfaces.WorkResult
+ cancel []string
+ refs []interfaces.SessionRef
+ opts []interfaces.StatusOptions
+}
+
+func (h *httpWorkHandler) Status(_ context.Context, ref interfaces.SessionRef,
+ ids []string, opts interfaces.StatusOptions,
+) ([]interfaces.WorkResult, error) {
+ h.refs = append(h.refs, ref)
+ h.opts = append(h.opts, opts)
+ out := make([]interfaces.WorkResult, 0, len(ids))
+ for _, id := range ids {
+ if r, ok := h.status[id]; ok {
+ out = append(out, r)
+ continue
+ }
+ // A live id is not in the terminal map. Detail still has to resolve.
+ out = append(out, interfaces.WorkResult{ID: id, State: interfaces.WorkStateRunning})
+ }
+ return out, nil
+}
+
+func (h *httpWorkHandler) Cancel(
+ _ context.Context, ref interfaces.SessionRef, ids []string,
+) ([]interfaces.WorkResult, error) {
+ h.refs = append(h.refs, ref)
+ h.cancel = append(h.cancel, ids...)
+ out := make([]interfaces.WorkResult, 0, len(ids))
+ for _, id := range ids {
+ out = append(out, interfaces.WorkResult{ID: id, State: interfaces.WorkStateCancelled, Terminal: true})
+ }
+ return out, nil
+}
+
+func (s *stubSessionService) GetOwnedSession(ctx context.Context, id string) (*types.Session, error) {
+ session, err := s.GetSession(ctx, id)
+ if err != nil || session == nil {
+ if err == nil {
+ err = errors.New("session not found")
+ }
+ return nil, err
+ }
+ // Strict owner: a caller executing as another tenant must not mutate.
+ if tenant, ok := types.TenantIDFromContext(ctx); ok && tenant != session.TenantID {
+ return nil, errors.New("session not found")
+ }
+ return session, nil
+}
+
+func newBackgroundHTTPRouter(t *testing.T, work *service.BackgroundWork,
+ sessionSvc interfaces.SessionService,
+) *gin.Engine {
+ t.Helper()
+ gin.SetMode(gin.TestMode)
+ h := &Handler{background: work, sessionService: sessionSvc}
+ r := gin.New()
+ r.GET("/api/v1/sessions/:id/background-work", h.ListBackgroundWork)
+ r.GET("/api/v1/sessions/:id/background-work/:work_id", h.GetBackgroundWork)
+ r.POST("/api/v1/sessions/:session_id/background-work/:work_id/cancel", h.CancelBackgroundWork)
+ return r
+}
+
+func ownedSessionContext(t *testing.T, tenant uint64, sessionID string) context.Context {
+ t.Helper()
+ ctx := types.WithExecutionTenant(context.Background(), tenant)
+ ctx = types.WithSandboxTenantID(ctx, tenant)
+ return types.WithSessionID(ctx, sessionID)
+}
+
+// splitCallerContext is a read whose execution tenant and sandbox-pin tenant
+// both differ from the session row. The handler must still use session.TenantID.
+func splitCallerContext(t *testing.T, sessionID string) context.Context {
+ t.Helper()
+ ctx := types.WithExecutionTenant(context.Background(), 99)
+ ctx = types.WithSandboxTenantID(ctx, 100)
+ return types.WithSessionID(ctx, sessionID)
+}
+
+func TestListMergesRegistryAndInboxWithoutExec(t *testing.T) {
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ work := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ handler := &httpWorkHandler{status: map[string]interfaces.WorkResult{
+ "sj_ab01": {
+ ID: "sj_ab01", State: interfaces.WorkStateSucceeded, Terminal: true, Title: "build",
+ },
+ }}
+ work.Handler = handler
+ err := work.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell", Title: "tests"}, time.Minute, 0)
+ require.NoError(t, err)
+ finished := time.Now().UTC()
+ _, err = work.Inbox.Push(context.Background(), ref, interfaces.BackgroundNotice{
+ ID: "sj_ab01", Kind: "shell", Text: "Background shell sj_ab01 succeeded: build.",
+ FinishedAt: finished,
+ })
+ require.NoError(t, err)
+
+ r := newBackgroundHTTPRouter(t, work, &stubSessionService{session: &types.Session{ID: "s", TenantID: 7}})
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/sessions/s/background-work", nil).
+ WithContext(splitCallerContext(t, "s"))
+ rec := httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code)
+ var body struct {
+ Data []map[string]any `json:"data"`
+ }
+ require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body))
+ require.Len(t, body.Data, 2, "运行中(登记表)+ 24h 内终态(信箱)")
+ states := map[string]any{}
+ titles := map[string]any{}
+ for _, item := range body.Data {
+ states[item["work_id"].(string)] = item["state"]
+ titles[item["work_id"].(string)] = item["title"]
+ }
+ require.Equal(t, "running", states["sj_0123"])
+ require.Equal(t, "succeeded", states["sj_ab01"])
+ require.Equal(t, "tests", titles["sj_0123"])
+ require.Equal(t, "build", titles["sj_ab01"])
+ require.NotEmpty(t, handler.opts, "终态 state 由 Status(Detailed=false) 批量补齐")
+ for _, opts := range handler.opts {
+ require.False(t, opts.Detailed)
+ require.Empty(t, opts.RunID)
+ }
+ require.NotEmpty(t, handler.refs)
+ for _, got := range handler.refs {
+ require.Equal(t, uint64(7), got.TenantID)
+ require.Equal(t, "s", got.SessionID)
+ }
+}
+
+func TestDetailAndCancelEnforceOwnership(t *testing.T) {
+ work := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ handler := &httpWorkHandler{status: map[string]interfaces.WorkResult{}}
+ work.Handler = handler
+ svc := &stubSessionService{session: &types.Session{ID: "s", TenantID: 7}}
+ r := newBackgroundHTTPRouter(t, work, svc)
+
+ // Owner read goes through GetSession. Caller tenant and sandbox pin differ
+ // from the session row; the status ref must still be the session tenant.
+ req := httptest.NewRequest(http.MethodGet,
+ "/api/v1/sessions/s/background-work/sj_0123", nil).
+ WithContext(splitCallerContext(t, "s"))
+ rec := httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Contains(t, rec.Body.String(), `"state":"running"`)
+ require.NotEmpty(t, handler.opts)
+ last := handler.opts[len(handler.opts)-1]
+ require.True(t, last.Detailed)
+ require.Empty(t, last.RunID, "a user read must not move the model cursor")
+ require.Equal(t, interfaces.SessionRef{TenantID: 7, SessionID: "s"}, handler.refs[len(handler.refs)-1])
+
+ // 取消走严格 owner(stub 拒绝非 owner)
+ req = httptest.NewRequest(http.MethodPost,
+ "/api/v1/sessions/s/background-work/sj_0123/cancel", nil).
+ WithContext(ownedSessionContext(t, 8, "s"))
+ rec = httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusNotFound, rec.Code, "GetOwnedSession 失败 → 404,不区分 403 防探测")
+ require.Empty(t, handler.cancel)
+}
+
+type errActivityRegistry struct {
+ interfaces.ActivityRegistry
+}
+
+func (errActivityRegistry) List(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundActivity, error) {
+ return nil, errors.New("redis down")
+}
+
+func TestListStorageErrorIsNotAnEmptySuccess(t *testing.T) {
+ work := service.NewBackgroundWork(errActivityRegistry{}, background.NewMemoryNoticeInbox(nil))
+ work.Handler = &httpWorkHandler{}
+ r := newBackgroundHTTPRouter(t, work, &stubSessionService{session: &types.Session{ID: "s", TenantID: 7}})
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/sessions/s/background-work", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec := httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.GreaterOrEqual(t, rec.Code, 500)
+ require.NotContains(t, rec.Body.String(), `"data":[]`)
+}
+
+func TestDisabledBackgroundWorkIsEmptyListAnd404(t *testing.T) {
+ sessionSvc := &stubSessionService{session: &types.Session{ID: "s", TenantID: 7}}
+ r := newBackgroundHTTPRouter(t, nil, sessionSvc)
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/sessions/s/background-work", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec := httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Contains(t, rec.Body.String(), `"data":[]`)
+ req = httptest.NewRequest(http.MethodGet,
+ "/api/v1/sessions/s/background-work/sj_0123", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec = httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusNotFound, rec.Code)
+ req = httptest.NewRequest(http.MethodPost,
+ "/api/v1/sessions/s/background-work/sj_0123/cancel", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec = httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusNotFound, rec.Code)
+
+ // Enabled() is false until Handler, Registry, and Inbox are all set.
+ // Rows already stored must not leak through a disabled bundle.
+ work := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ ref := interfaces.SessionRef{TenantID: 7, SessionID: "s"}
+ err := work.Registry.Add(context.Background(), ref,
+ interfaces.BackgroundActivity{ID: "sj_0123", Kind: "shell", Title: "tests"}, time.Minute, 0)
+ require.NoError(t, err)
+ _, err = work.Inbox.Push(context.Background(), ref, interfaces.BackgroundNotice{
+ ID: "sj_ab01", Kind: "shell", Text: "Background shell sj_ab01 succeeded: build.",
+ FinishedAt: time.Now().UTC(),
+ })
+ require.NoError(t, err)
+ require.False(t, work.Enabled())
+ r = newBackgroundHTTPRouter(t, work, sessionSvc)
+ req = httptest.NewRequest(http.MethodGet, "/api/v1/sessions/s/background-work", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec = httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Contains(t, rec.Body.String(), `"data":[]`)
+ req = httptest.NewRequest(http.MethodGet,
+ "/api/v1/sessions/s/background-work/sj_0123", nil).
+ WithContext(ownedSessionContext(t, 7, "s"))
+ rec = httptest.NewRecorder()
+ r.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusNotFound, rec.Code)
+}
diff --git a/internal/handler/session/continue_stream_resource_urls_test.go b/internal/handler/session/continue_stream_resource_urls_test.go
index 83f986b93..3a1eec64e 100644
--- a/internal/handler/session/continue_stream_resource_urls_test.go
+++ b/internal/handler/session/continue_stream_resource_urls_test.go
@@ -22,9 +22,15 @@ import (
type stubSessionService struct {
interfaces.SessionService
+ // session, when set, is the row GetSession returns. Background-work tests
+ // use it so the handler must key work by session.TenantID.
+ session *types.Session
}
func (s *stubSessionService) GetSession(_ context.Context, id string) (*types.Session, error) {
+ if s != nil && s.session != nil {
+ return s.session, nil
+ }
return &types.Session{ID: id, TenantID: 1}, nil
}
diff --git a/internal/localsandbox/adapter/adapter.go b/internal/localsandbox/adapter/adapter.go
index fd99f73df..8478aba6b 100644
--- a/internal/localsandbox/adapter/adapter.go
+++ b/internal/localsandbox/adapter/adapter.go
@@ -32,6 +32,15 @@ type Adapter struct {
// New wraps a localsandbox Service as the session-scoped Manager the agent uses.
func New(svc *localsandbox.Service) *Adapter { return &Adapter{svc: svc} }
+// LocalSandboxService exposes the wrapped service for desktop composition
+// (local shell jobs). Not part of the remote-facing capability interfaces.
+func (a *Adapter) LocalSandboxService() *localsandbox.Service {
+ if a == nil {
+ return nil
+ }
+ return a.svc
+}
+
// maxListSessionFiles stops WalkDir once the listing is as large as the
// list_sandbox_files tool will ever keep (500). The tool also truncates,
// but that is after the whole tree is already in memory.
diff --git a/internal/localsandbox/core/policy_builder.go b/internal/localsandbox/core/policy_builder.go
index b1e9f3561..cfe8b6b26 100644
--- a/internal/localsandbox/core/policy_builder.go
+++ b/internal/localsandbox/core/policy_builder.go
@@ -127,6 +127,7 @@ type PolicyBuilder struct {
homeDir string
appDataDir string
skillsRoot string
+ jobsRoot string
}
// NewPolicyBuilder returns a builder scoped to the user's home and app-data dirs.
@@ -134,6 +135,7 @@ func NewPolicyBuilder(homeDir, appDataDir string) *PolicyBuilder {
return &PolicyBuilder{
homeDir: filepath.Clean(homeDir),
appDataDir: filepath.Clean(appDataDir),
+ jobsRoot: filepath.Join(filepath.Clean(homeDir), ".weknora", "jobs"),
}
}
@@ -293,6 +295,11 @@ func (b *PolicyBuilder) rejectBroadWorkspace(root string) error {
return fmt.Errorf("%w: %q", ErrWorkspaceTooBroad, root)
}
}
+ // Background jobs live under ~/.weknora/jobs. A workspace that covers that
+ // directory, or sits inside it, can read or replace another session's output.
+ if b.jobsRoot != "" && (PathUnder(b.jobsRoot, root) || PathUnder(root, b.jobsRoot)) {
+ return fmt.Errorf("%w: %q overlaps the background jobs directory", ErrWorkspaceTooBroad, root)
+ }
if isWellKnownWideRoot(root) {
return fmt.Errorf("%w: %q", ErrWorkspaceTooBroad, root)
}
@@ -307,7 +314,7 @@ func samePath(a, b string) bool {
}
func (b *PolicyBuilder) denyRead() []string {
- deny := make([]string, 0, len(credentialDirNames)+1)
+ deny := make([]string, 0, len(credentialDirNames)+2)
for _, name := range credentialDirNames {
deny = append(deny, filepath.Join(b.homeDir, filepath.FromSlash(name)))
}
@@ -317,6 +324,9 @@ func (b *PolicyBuilder) denyRead() []string {
if b.appDataDir != "" && b.appDataDir != string(filepath.Separator) {
deny = append(deny, b.appDataDir)
}
+ if b.jobsRoot != "" && !isFilesystemRoot(b.jobsRoot) {
+ deny = append(deny, b.jobsRoot)
+ }
return deny
}
diff --git a/internal/localsandbox/core/policy_builder_test.go b/internal/localsandbox/core/policy_builder_test.go
index 6edb03256..1b45d89ec 100644
--- a/internal/localsandbox/core/policy_builder_test.go
+++ b/internal/localsandbox/core/policy_builder_test.go
@@ -393,3 +393,16 @@ func TestBuildInstallRequiresASkillsRoot(t *testing.T) {
_, err := b.BuildInstall(filepath.Join(b.homeDir, ".weknora", "skills", ".versions", "pdf-1"))
require.ErrorIs(t, err, ErrInstallDirOutsideSkillsRoot)
}
+
+func TestJobsDirectoryCannotBecomeAWorkspace(t *testing.T) {
+ home := "/Users/local-test"
+ jobs := filepath.Join(home, ".weknora", "jobs")
+ b := NewPolicyBuilder(home, filepath.Join(home, "Library", "Application Support", "WeKnora Lite"))
+ for _, root := range []string{filepath.Dir(jobs), jobs, filepath.Join(jobs, "sj_0123")} {
+ _, err := b.Build(ModeAuto, Workspace{Root: root})
+ require.Error(t, err, root)
+ }
+ p, err := b.Build(ModeAuto, Workspace{Root: filepath.Join(home, "project")})
+ require.NoError(t, err)
+ require.Contains(t, p.DenyRead, jobs)
+}
diff --git a/internal/localsandbox/seatbelt/backend.go b/internal/localsandbox/seatbelt/backend.go
index 9f9646069..157f94174 100644
--- a/internal/localsandbox/seatbelt/backend.go
+++ b/internal/localsandbox/seatbelt/backend.go
@@ -133,6 +133,7 @@ func (b *seatbeltBackend) Spawn(
proc := &seatbeltProcess{
cmd: execCmd,
+ pid: pid,
stdout: stdout,
stderr: stderr,
started: started,
@@ -246,6 +247,7 @@ func withSandboxTemp(env []string, cwd string, explicit map[string]string) []str
type seatbeltProcess struct {
cmd *exec.Cmd
+ pid int
stdout io.ReadCloser
stderr io.ReadCloser
started time.Time
@@ -258,7 +260,7 @@ type seatbeltProcess struct {
func (p *seatbeltProcess) Stdout() io.Reader { return p.stdout }
func (p *seatbeltProcess) Stderr() io.Reader { return p.stderr }
-func (p *seatbeltProcess) PID() int { return p.cmd.Process.Pid }
+func (p *seatbeltProcess) PID() int { return p.pid }
func (p *seatbeltProcess) watchContext(ctx context.Context) {
select {
@@ -280,17 +282,14 @@ func (p *seatbeltProcess) watchContext(ctx context.Context) {
// descendants running after a timeout.
func (p *seatbeltProcess) Kill() error {
p.mu.Lock()
- // ProcessState is written before cmd.Wait returns, so this closes the
- // window between Wait returning and reaped being set.
- if p.reaped || p.cmd.ProcessState != nil {
+ // reaped is set under this lock as soon as Wait returns. Cmd.ProcessState
+ // is written by Wait itself, so another goroutine must not read it.
+ if p.reaped {
p.mu.Unlock()
return nil
}
p.killed = true
- pid := 0
- if p.cmd.Process != nil {
- pid = p.cmd.Process.Pid
- }
+ pid := p.pid
p.mu.Unlock()
if pid == 0 {
diff --git a/internal/localsandbox/service_test.go b/internal/localsandbox/service_test.go
index c2db80a1d..eb21473c1 100644
--- a/internal/localsandbox/service_test.go
+++ b/internal/localsandbox/service_test.go
@@ -8,6 +8,7 @@ import (
"path/filepath"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -27,6 +28,7 @@ type fakeBackend struct {
calls []string
proc Process
onSpawn func()
+ preps []*fakePrepared
}
func (f *fakeBackend) Name() string { return "fake" }
@@ -47,17 +49,25 @@ func (f *fakeBackend) EnsureReady(context.Context) error {
}
func (f *fakeBackend) TearDown(context.Context) error { return nil }
-type fakePrepared struct{ fp string }
+type fakePrepared struct {
+ fp string
+ closed atomic.Bool
+}
-func (p fakePrepared) Fingerprint() string { return p.fp }
-func (p fakePrepared) Close() error { return nil }
+func (p *fakePrepared) Fingerprint() string { return p.fp }
+func (p *fakePrepared) Close() error {
+ p.closed.Store(true)
+ return nil
+}
func (f *fakeBackend) Prepare(_ context.Context, p Policy) (Prepared, error) {
+ prep := &fakePrepared{fp: p.Fingerprint()}
f.mu.Lock()
f.calls = append(f.calls, "prepare")
f.prepared = append(f.prepared, p.Fingerprint())
+ f.preps = append(f.preps, prep)
f.mu.Unlock()
- return fakePrepared{fp: p.Fingerprint()}, nil
+ return prep, nil
}
func (f *fakeBackend) Spawn(_ context.Context, _ Prepared, cmd Command) (Process, error) {
diff --git a/internal/localsandbox/start.go b/internal/localsandbox/start.go
new file mode 100644
index 000000000..44bdc48e1
--- /dev/null
+++ b/internal/localsandbox/start.go
@@ -0,0 +1,71 @@
+package localsandbox
+
+import (
+ "context"
+ "fmt"
+ "sync"
+)
+
+// StartForSession resolves the session policy and starts a managed job
+// process. Unlike Run it takes no root lock, applies no timeout and reads no
+// pipes — the caller owns draining and Wait.
+func (s *Service) StartForSession(ctx context.Context, req RunRequest) (Process, error) {
+ _, policy, err := s.workspaceFor(ctx, req.SessionID)
+ if err != nil {
+ return nil, err
+ }
+ return s.Start(ctx, policy, req)
+}
+
+// Start spawns one command under a prepared policy. The returned Process wraps
+// the prepared handle so its resources are released exactly once at Wait; a
+// failed Spawn closes it here. ctx cancellation kills the whole process group
+// (seatbelt watchContext), so callers must pass an application-level ctx.
+func (s *Service) Start(ctx context.Context, policy Policy, req RunRequest) (Process, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if err := s.ensureBackend(ctx); err != nil {
+ return nil, err
+ }
+ if err := policy.Validate(); err != nil {
+ return nil, err
+ }
+ cwd := policy.Cwd
+ if req.WorkDir != "" {
+ resolved, err := NewPathGuard(policy).CheckDir(req.WorkDir)
+ if err != nil {
+ return nil, err
+ }
+ cwd = resolved
+ }
+ prep, err := s.backend.Prepare(ctx, policy)
+ if err != nil {
+ return nil, fmt.Errorf("prepare sandbox: %w", err)
+ }
+ argv := append(append([]string(nil), s.shell...), req.Command)
+ var bins []string
+ if s.builder != nil {
+ bins = s.builder.ToolchainBins()
+ }
+ proc, err := s.backend.Spawn(ctx, prep, Command{
+ Argv: argv, Env: BuildCommandEnv(req.Env, bins), Cwd: cwd,
+ })
+ if err != nil {
+ _ = prep.Close()
+ return nil, fmt.Errorf("spawn sandboxed command: %w", err)
+ }
+ return &preparedProcess{Process: proc, prepared: prep}, nil
+}
+
+type preparedProcess struct {
+ Process
+ prepared Prepared
+ once sync.Once
+}
+
+func (p *preparedProcess) Wait(ctx context.Context) (ExitStatus, error) {
+ status, err := p.Process.Wait(ctx)
+ p.once.Do(func() { _ = p.prepared.Close() })
+ return status, err
+}
diff --git a/internal/localsandbox/start_test.go b/internal/localsandbox/start_test.go
new file mode 100644
index 000000000..f345db90c
--- /dev/null
+++ b/internal/localsandbox/start_test.go
@@ -0,0 +1,74 @@
+package localsandbox
+
+import (
+ "context"
+ "errors"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+var errFakeSpawn = errors.New("spawn boom")
+
+func TestStartDoesNotTakeRootLock(t *testing.T) {
+ backend := &fakeBackend{stdout: "ok", exit: ExitStatus{Code: 0}}
+ svc := serviceFixture(t, backend, ModeAuto)
+ _, ws, err := svc.GuardForSession(context.Background(), "s1")
+ require.NoError(t, err)
+ unlock := svc.LockRoot(ws.Root)
+ defer unlock()
+ ready := make(chan Process, 1)
+ failed := make(chan error, 1)
+ go func() {
+ proc, err := svc.StartForSession(context.Background(), RunRequest{
+ SessionID: "s1", Command: "true", Env: map[string]string{"K": "v"},
+ })
+ if err != nil {
+ failed <- err
+ return
+ }
+ ready <- proc
+ }()
+ select {
+ case err := <-failed:
+ t.Fatal(err)
+ case proc := <-ready:
+ _, err := io.Copy(io.Discard, proc.Stdout())
+ require.NoError(t, err)
+ _, err = io.Copy(io.Discard, proc.Stderr())
+ require.NoError(t, err)
+ _, err = proc.Wait(context.Background())
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("Start blocked behind the synchronous root lock")
+ }
+ require.Equal(t, "v", backend.spawnCmd.Env["K"], "env 经 BuildCommandEnv 传给 Spawn")
+ require.Len(t, backend.preps, 1)
+ require.True(t, backend.preps[0].closed.Load(), "Wait closes the prepared handle")
+}
+
+func TestStartValidatesWorkDirAndMode(t *testing.T) {
+ backend := &fakeBackend{}
+ svc := serviceFixture(t, backend, ModeFull)
+ _, err := svc.StartForSession(context.Background(), RunRequest{SessionID: "s", Command: "true"})
+ require.ErrorIs(t, err, ErrApprovalModeNotShipped)
+ require.Empty(t, backend.prepared)
+ svc = serviceFixture(t, backend, ModeAuto)
+ _, err = svc.StartForSession(context.Background(), RunRequest{
+ SessionID: "s", Command: "true", WorkDir: "/etc",
+ })
+ require.Error(t, err)
+ require.Empty(t, backend.prepared)
+}
+
+func TestStartSpawnFailureClosesPrepared(t *testing.T) {
+ backend := &fakeBackend{spawnErr: errFakeSpawn}
+ svc := serviceFixture(t, backend, ModeAuto)
+ _, err := svc.StartForSession(context.Background(), RunRequest{SessionID: "s", Command: "true"})
+ require.ErrorIs(t, err, errFakeSpawn)
+ require.Len(t, backend.prepared, 1, "Prepare 被调用过;Close 由 Start 负责")
+ require.Len(t, backend.preps, 1)
+ require.True(t, backend.preps[0].closed.Load(), "Start closes the prepared handle when Spawn fails")
+}
diff --git a/internal/router/background_routes_test.go b/internal/router/background_routes_test.go
new file mode 100644
index 000000000..1f15dd8b8
--- /dev/null
+++ b/internal/router/background_routes_test.go
@@ -0,0 +1,40 @@
+// Package router registers the HTTP API, including background-work routes.
+package router
+
+import (
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/config"
+ "github.com/Tencent/WeKnora/internal/handler"
+ sessionhandler "github.com/Tencent/WeKnora/internal/handler/session"
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+)
+
+// registerRoutesForTest builds the smallest router that exercises session
+// registration. Tenant is set so a nil config does not log a startup error.
+func registerRoutesForTest(t *testing.T) *gin.Engine {
+ t.Helper()
+ gin.SetMode(gin.TestMode)
+ g := &rbacGuards{cfg: &config.Config{Tenant: &config.TenantConfig{}}}
+ engine := gin.New()
+ RegisterSessionRoutes(engine.Group("/api/v1"), &sessionhandler.Handler{}, &handler.MessageSuggestionHandler{}, g)
+ return engine
+}
+
+// Gin 同一 verb 的通配符名必须一致:GET 树已用 :id,POST 树用 :session_id。
+func TestBackgroundWorkRoutesRegisterWithoutWildcardConflict(t *testing.T) {
+ var engine *gin.Engine
+ require.NotPanics(t, func() {
+ engine = registerRoutesForTest(t) // 构建最小 RouterGroup 并调用 RegisterSessionRoutes;
+ // 覆盖点是“注册不 panic”。参数解析由 handler 测试负责。
+ })
+ found := map[string]bool{}
+ for _, route := range engine.Routes() {
+ found[route.Method+" "+route.Path] = true
+ }
+ require.True(t, found["GET /api/v1/sessions/:id/background-work"])
+ require.True(t, found["GET /api/v1/sessions/:id/background-work/:work_id"])
+ require.True(t, found["POST /api/v1/sessions/:session_id/background-work/:work_id/cancel"])
+ require.False(t, found["GET /api/v1/sessions/:session_id/background-work"])
+}
diff --git a/internal/router/routes_chat.go b/internal/router/routes_chat.go
index 43f06bcf0..a6a1e1e32 100644
--- a/internal/router/routes_chat.go
+++ b/internal/router/routes_chat.go
@@ -79,6 +79,13 @@ func RegisterSessionRoutes(
sessions.GET("/:id/steer", handler.ListSteerMessages)
sessions.DELETE("/:id/steer/:steer_id", handler.DeleteSteerMessage)
sessions.POST("/:session_id/steer/:steer_id/inject", handler.PromoteSteerMessage)
+ // Background work: cheap list for the chip, on-demand output for the
+ // user, owner-only cancel. Same WorkHandler the model tools use.
+ // GET stays on :id and POST on :session_id so each verb's radix tree
+ // keeps the wildcard name it already uses.
+ sessions.GET("/:id/background-work", handler.ListBackgroundWork)
+ sessions.GET("/:id/background-work/:work_id", handler.GetBackgroundWork)
+ sessions.POST("/:session_id/background-work/:work_id/cancel", handler.CancelBackgroundWork)
// POST and DELETE share this path but gin maintains a separate radix tree
// per HTTP verb, and the existing trees use different wildcard names
// (POST uses :session_id, DELETE uses :id). Use whatever matches each
diff --git a/internal/sandbox/session_file_operation_test.go b/internal/sandbox/session_file_operation_test.go
index 43d178341..a5afde764 100644
--- a/internal/sandbox/session_file_operation_test.go
+++ b/internal/sandbox/session_file_operation_test.go
@@ -100,7 +100,7 @@ func TestExpectedSandboxMaintenanceDoesNotPrepareOrProvision(t *testing.T) {
_, err := mgr.ExecShellCommandWithOptions(ctx, tc.session, "git status", ShellExecOptions{
ExpectedSandboxID: tc.expected, SkipWorkspacePrep: true,
})
- require.Error(t, err)
+ require.ErrorIs(t, err, ErrBoundSandboxChanged)
}
require.Len(t, client.connectIDs, 1, "do not resume a mismatched sandbox")
require.Len(t, client.execRequests, 1)
diff --git a/internal/sandbox/session_manager.go b/internal/sandbox/session_manager.go
index 5408c23bb..1fb308c89 100644
--- a/internal/sandbox/session_manager.go
+++ b/internal/sandbox/session_manager.go
@@ -89,6 +89,10 @@ const sessionArtifactDirBootstrapTimeout = 15 * time.Second
// disappearance).
const sessionLifecycleCleanupTimeout = 30 * time.Second
+// ErrBoundSandboxChanged confirms that a maintenance command's original
+// sandbox is gone. Transport failures must not be treated as this condition.
+var ErrBoundSandboxChanged = errors.New("sandbox: bound sandbox changed before maintenance command")
+
// SessionBoundManager is a sandbox.Manager that binds one remote sandbox per
// tenant session. Concrete provider work is delegated to RemoteSandboxClient;
// this type owns validation and the mapping between application concepts
@@ -1090,7 +1094,7 @@ func (m *SessionBoundManager) execShellCommandWithOutputSnapshot(
return nil, nil, err
}
if !found || handle.ID() != opts.ExpectedSandboxID {
- return nil, nil, errors.New("sandbox: bound sandbox changed before maintenance command")
+ return nil, nil, ErrBoundSandboxChanged
}
} else {
handle, err = m.resolveScope(ctx, sc)
@@ -1789,7 +1793,7 @@ func (m *SessionBoundManager) lookupSessionHandleForKey(
return nil, false, nil
}
if expectedID != "" && binding.SandboxID != expectedID {
- return nil, false, errors.New("sandbox: bound sandbox changed before maintenance command")
+ return nil, false, ErrBoundSandboxChanged
}
handle, err := m.client.Connect(ctx, RemoteConnectRequest{
SandboxID: binding.SandboxID,
diff --git a/internal/sandbox/shell_job.go b/internal/sandbox/shell_job.go
new file mode 100644
index 000000000..91a44d8ce
--- /dev/null
+++ b/internal/sandbox/shell_job.go
@@ -0,0 +1,542 @@
+// Package sandbox reads and writes the shell-job directory protocol.
+package sandbox
+
+import (
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+)
+
+// ShellJobsRoot is where job directories live inside a remote sandbox, outside
+// the /workspace git worktree so checkpoints never see them (shell 稿 §9.1).
+const ShellJobsRoot = "/var/lib/weknora/jobs"
+
+const (
+ shellJobSegmentBytes = 16 * 1024 * 1024
+ shellJobHeadBytes = 8 * 1024
+ // Model-facing caps (shell 稿 §9.3). A kept window is 25% head and 75% tail.
+ shellJobStdoutBudget = 16 * 1024
+ shellJobStderrBudget = 8 * 1024
+)
+
+// ShellJobDirExistsExit is the submit script's exit when the job directory
+// is already there. The caller draws another id and tries again.
+const ShellJobDirExistsExit = 17
+
+var shellJobIDRE = regexp.MustCompile(`^sj_[0-9a-f]{4}$`)
+
+// ValidShellJobID reports whether id is a well-formed shell job handle.
+func ValidShellJobID(id string) bool { return shellJobIDRE.MatchString(id) }
+
+// NewShellJobID draws sj_ plus 4 lowercase hex digits (2 random bytes).
+func NewShellJobID() (string, error) {
+ var token [2]byte
+ if _, err := rand.Read(token[:]); err != nil {
+ return "", err
+ }
+ return "sj_" + hex.EncodeToString(token[:]), nil
+}
+
+// ShellJobMeta is the persisted identity of one shell job.
+type ShellJobMeta struct {
+ ID string `json:"work_id"`
+ TenantID uint64 `json:"tenant_id"`
+ SandboxTenantID uint64 `json:"sandbox_tenant_id"` // config owner, NOT the binding-key owner
+ SessionID string `json:"session_id"`
+ Root string `json:"root"`
+ Command string `json:"command"` // masked; credentials never persist
+ WorkDir string `json:"work_dir"`
+ SandboxID string `json:"sandbox_id"`
+ CreatedAt time.Time `json:"created_at"`
+ YieldUntil time.Time `json:"yield_until"`
+ ParentMessageID string `json:"parent_message_id"`
+ ToolCallID string `json:"tool_call_id"`
+ Title string `json:"title"`
+ OutputDir string `json:"output_dir,omitempty"`
+ OutputSeen bool `json:"output_seen,omitempty"`
+ OutputBefore []OutputFileStamp `json:"output_before,omitempty"`
+}
+
+// OutputFileStamp is one file in the output directory before a job starts.
+// A later listing is compared with it to find deliverables.
+type OutputFileStamp struct {
+ Path string `json:"path"`
+ Size int64 `json:"size"`
+ ModUnixNano int64 `json:"mod_unix_nano"`
+}
+
+// StampsFromEntries keeps file entries only, capped so the job meta stays small.
+// ok is false when the listing is too large to compare reliably.
+func StampsFromEntries(entries []RemoteDirEntry) (stamps []OutputFileStamp, ok bool) {
+ const maxOutputStamps = 200
+ files := 0
+ for _, entry := range entries {
+ if entry.Type == RemoteEntryFile {
+ files++
+ }
+ }
+ if files > maxOutputStamps {
+ return nil, false
+ }
+ stamps = make([]OutputFileStamp, 0, files)
+ for _, entry := range entries {
+ if entry.Type != RemoteEntryFile {
+ continue
+ }
+ stamps = append(stamps, OutputFileStamp{
+ Path: entry.Path, Size: entry.Size, ModUnixNano: entry.ModTime.UnixNano(),
+ })
+ }
+ return stamps, true
+}
+
+// EntriesFromStamps restores stamps for the same comparison shell_exec uses.
+func EntriesFromStamps(stamps []OutputFileStamp) []RemoteDirEntry {
+ out := make([]RemoteDirEntry, 0, len(stamps))
+ for _, stamp := range stamps {
+ out = append(out, RemoteDirEntry{
+ Path: stamp.Path, Type: RemoteEntryFile, Size: stamp.Size,
+ ModTime: time.Unix(0, stamp.ModUnixNano),
+ })
+ }
+ return out
+}
+
+// ShellJobStatus is the watcher's latest state for one shell job.
+type ShellJobStatus struct {
+ State string `json:"state"`
+ WatcherPID int `json:"watcher_pid"`
+ PGID int `json:"pgid"`
+ TouchPID int `json:"touch_pid,omitempty"`
+ ProcStartedAt int64 `json:"proc_started_at,omitempty"`
+ ExitCode *int `json:"exit_code,omitempty"`
+ StartedAt time.Time `json:"started_at"`
+ EndedAt *time.Time `json:"ended_at,omitempty"`
+}
+
+// ShellJobOffsets is an absolute byte cursor into the two output streams.
+type ShellJobOffsets struct {
+ Stdout int64 `json:"stdout"`
+ Stderr int64 `json:"stderr"`
+}
+
+// ShellJobStream is one bounded view of a job output stream.
+type ShellJobStream struct {
+ Text string `json:"text"`
+ FromBytes int64 `json:"from_bytes"`
+ TotalBytes int64 `json:"total_bytes"`
+ OmittedBytes int64 `json:"omitted_bytes"`
+ Truncated bool `json:"truncated"`
+ // NextBytes is the absolute end of the bytes consumed by this view, before
+ // UTF-8 sanitization. Nil is reserved for older snapshot producers.
+ NextBytes *int64 `json:"next_bytes,omitempty"`
+}
+
+// ShellJobSnapshot is a point-in-time read of one job directory.
+type ShellJobSnapshot struct {
+ Meta ShellJobMeta `json:"meta"`
+ Status ShellJobStatus `json:"status"`
+ Stdout ShellJobStream `json:"stdout"`
+ Stderr ShellJobStream `json:"stderr"`
+ WatcherAlive bool `json:"watcher_alive"`
+ Missing bool `json:"missing"`
+ Preview string `json:"preview,omitempty"`
+}
+
+// ShellJobRead selects a job directory and how much of each stream to return.
+type ShellJobRead struct {
+ ID string `json:"id"`
+ From *ShellJobOffsets `json:"from,omitempty"`
+ OutputBudget int `json:"output_budget"`
+ Preview bool `json:"preview"`
+}
+
+// WriteShellJobJSON writes v atomically (tmp + rename). The remote wrapper does
+// the same dance in sh; Lite uses this helper.
+func WriteShellJobJSON(path string, value any) error {
+ raw, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+ tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+"-")
+ if err != nil {
+ return err
+ }
+ name := tmp.Name()
+ committed := false
+ defer func() {
+ if !committed {
+ _ = os.Remove(name)
+ }
+ }()
+ if _, err := tmp.Write(raw); err != nil {
+ _ = tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ if err := os.Rename(name, path); err != nil {
+ return err
+ }
+ committed = true
+ return nil
+}
+
+// ReadShellJob reads one job directory with bounded IO. A nil cursor is the
+// first read of a run, or an out-of-run "查看输出" read: head plus tail, with
+// the tail padded from the end of .1 when the current segment is short
+// (shell 稿 §9.1, §6.3). A cursor returns only bytes after that absolute
+// offset. A cursor that falls in bytes rotation already discarded starts at
+// the earliest byte still on disk, and OmittedBytes counts the gap.
+func ReadShellJob(root string, req ShellJobRead) (ShellJobSnapshot, error) {
+ if !ValidShellJobID(req.ID) {
+ return ShellJobSnapshot{}, fmt.Errorf("sandbox: invalid shell job id %q", req.ID)
+ }
+ dir := filepath.Join(root, req.ID)
+ snap := ShellJobSnapshot{}
+ if err := readShellJobMeta(dir, &snap); err != nil {
+ return snap, err
+ }
+ statusRaw, err := os.ReadFile(filepath.Join(dir, "status.json"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return ShellJobSnapshot{Missing: true}, nil
+ }
+ return snap, err
+ }
+ if err := json.Unmarshal(statusRaw, &snap.Status); err != nil {
+ return snap, fmt.Errorf("sandbox: read job status: %w", err)
+ }
+ snap.WatcherAlive = watcherAlive(dir, snap.Status.WatcherPID, snap.Meta.ID)
+ snap.Stdout = readShellStream(dir, "stdout", req.From, "Stdout", req.OutputBudget)
+ snap.Stderr = readShellStream(dir, "stderr", req.From, "Stderr", req.OutputBudget)
+ return snap, nil
+}
+
+// readShellJobMeta loads meta.json when the watcher has written it. A job
+// directory can be readable from status and stream files alone; only a missing
+// status.json means the job itself is absent.
+func readShellJobMeta(dir string, snap *ShellJobSnapshot) error {
+ metaRaw, err := os.ReadFile(filepath.Join(dir, "meta.json"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ if err := json.Unmarshal(metaRaw, &snap.Meta); err != nil {
+ return fmt.Errorf("sandbox: read job meta: %w", err)
+ }
+ return nil
+}
+
+// watcherAlive reports whether status.json still names a watcher. The remote
+// cmdline check runs inside the sandbox (the batch read script sees that
+// /proc); this host-side reader only has the job directory.
+func watcherAlive(dir string, pid int, jobID string) bool {
+ _ = dir
+ _ = jobID
+ return pid > 0
+}
+
+func streamBudget(which string, budget int) int {
+ if budget > 0 {
+ stderr := budget / 3
+ if which == "Stderr" {
+ return stderr
+ }
+ return budget - stderr
+ }
+ if which == "Stderr" {
+ return shellJobStderrBudget
+ }
+ return shellJobStdoutBudget
+}
+
+func readShellStream(dir, name string, from *ShellJobOffsets, which string, budget int) ShellJobStream {
+ budget = streamBudget(which, budget)
+ current := readFileOrEmpty(filepath.Join(dir, name))
+ rotated := readRotated(filepath.Join(dir, name+".rotated"))
+ previous := readFileOrEmpty(filepath.Join(dir, name+".1"))
+ headFile := readFileOrEmpty(filepath.Join(dir, name+".head"))
+ total := rotated + int64(len(current))
+ out := ShellJobStream{TotalBytes: total}
+ var text []byte
+ if from == nil {
+ text = readShellHeadTail(&out, current, previous, headFile, rotated, total, budget)
+ } else {
+ text = readShellIncremental(&out, current, previous, headFile, rotated, shellOffset(from, which), budget)
+ }
+ if out.OmittedBytes > 0 || int64(len(text)) != total-out.FromBytes {
+ out.Truncated = true
+ }
+ out.Text = sanitizeText(text)
+ // The local reader selects head and tail when over budget, so it has
+ // consumed the end even if some middle bytes were deliberately omitted.
+ next := total
+ out.NextBytes = &next
+ return out
+}
+
+// readShellHeadTail returns the head and the tail of one stream. The head is
+// .head when the watcher saved it, otherwise the start of the current segment.
+// The tail is the end of the current segment, padded from the end of .1 when
+// that segment is shorter than the tail budget. Overlap is returned once.
+func readShellHeadTail(
+ out *ShellJobStream, current, previous, headFile []byte, rotated, total int64, budget int,
+) []byte {
+ headBudget := budget / 4
+ tailBudget := budget - headBudget
+ head, headStart, headFromFile := selectHead(current, headFile, rotated, headBudget)
+ tail, tailStart := selectTail(previous, current, total, tailBudget)
+ text, textStart, gap := joinHeadTail(head, headStart, headFromFile, tail, tailStart)
+ out.FromBytes = textStart
+ out.OmittedBytes = textStart + gap + (total - (textStart + int64(len(text)) + gap))
+ if out.OmittedBytes < 0 {
+ out.OmittedBytes = 0
+ }
+ return text
+}
+
+func selectHead(current, headFile []byte, rotated int64, headBudget int) ([]byte, int64, bool) {
+ if headBudget <= 0 {
+ return nil, 0, false
+ }
+ if len(headFile) > 0 {
+ return copyFirst(headFile, headBudget), 0, true
+ }
+ return copyFirst(current, headBudget), rotated, false
+}
+
+func selectTail(previous, current []byte, total int64, tailBudget int) ([]byte, int64) {
+ if tailBudget <= 0 {
+ return nil, total
+ }
+ if len(current) >= tailBudget {
+ return copyLast(current, tailBudget), total - int64(tailBudget)
+ }
+ need := tailBudget - len(current)
+ prevSuffix := copyLast(previous, need)
+ text := make([]byte, 0, len(prevSuffix)+len(current))
+ text = append(text, prevSuffix...)
+ text = append(text, current...)
+ return text, total - int64(len(text))
+}
+
+func joinHeadTail(
+ head []byte, headStart int64, headFromFile bool, tail []byte, tailStart int64,
+) ([]byte, int64, int64) {
+ if len(head) == 0 {
+ return tail, tailStart, 0
+ }
+ if len(tail) == 0 {
+ return head, headStart, 0
+ }
+ headEnd := headStart + int64(len(head))
+ tailEnd := tailStart + int64(len(tail))
+ // A head taken from the current segment is already inside a tail that
+ // reaches back into .1. Keep that pad. A .head file overrides the bytes
+ // at the start of the stream instead.
+ if !headFromFile && tailStart <= headStart && tailEnd >= headEnd {
+ return tail, tailStart, 0
+ }
+ if headStart <= tailStart && headEnd >= tailEnd {
+ return head, headStart, 0
+ }
+ if tailStart >= headEnd {
+ out := make([]byte, 0, len(head)+len(tail))
+ out = append(out, head...)
+ out = append(out, tail...)
+ return out, headStart, tailStart - headEnd
+ }
+ skip := int(headEnd - tailStart)
+ if skip < 0 {
+ skip = 0
+ }
+ if skip > len(tail) {
+ skip = len(tail)
+ }
+ out := make([]byte, 0, len(head)+len(tail)-skip)
+ out = append(out, head...)
+ out = append(out, tail[skip:]...)
+ return out, headStart, 0
+}
+
+// readShellIncremental returns bytes at and after an absolute cursor. A cursor
+// in a hole left by rotation starts at the next byte still on disk.
+func readShellIncremental(
+ out *ShellJobStream, current, previous, headFile []byte, rotated, offset int64, budget int,
+) []byte {
+ total := out.TotalBytes
+ if offset < 0 {
+ offset = 0
+ }
+ if offset > total {
+ offset = total
+ }
+ prevStart := rotated
+ if len(previous) > 0 {
+ prevStart = rotated - int64(len(previous))
+ }
+ headEnd := int64(len(headFile))
+ start := offset
+ if !streamByteExists(start, headEnd, prevStart, rotated, total) && start < total {
+ start = earliestByteAtOrAfter(start, headEnd, prevStart, rotated)
+ out.OmittedBytes = start - offset
+ }
+ out.FromBytes = start
+ text, gap := collectStream(start, headFile, previous, current, prevStart, rotated, headEnd, total)
+ out.OmittedBytes += gap
+ return capToBudget(out, text, budget)
+}
+
+func streamByteExists(offset, headEnd, prevStart, rotated, total int64) bool {
+ if offset >= total {
+ return true
+ }
+ if offset >= rotated {
+ return true
+ }
+ if offset >= prevStart && offset < rotated && prevStart < rotated {
+ return true
+ }
+ return offset < headEnd
+}
+
+func earliestByteAtOrAfter(offset, headEnd, prevStart, rotated int64) int64 {
+ if offset < headEnd {
+ return offset
+ }
+ if offset < prevStart {
+ return prevStart
+ }
+ if offset < rotated {
+ return offset
+ }
+ return offset
+}
+
+func collectStream(
+ start int64, headFile, previous, current []byte, prevStart, rotated, headEnd, total int64,
+) ([]byte, int64) {
+ var text []byte
+ var gap int64
+ if len(headFile) > 0 && start < headEnd {
+ text = append(text, headFile[start:]...)
+ start = headEnd
+ }
+ if len(previous) > 0 && start < rotated {
+ if start < prevStart {
+ gap += prevStart - start
+ start = prevStart
+ }
+ if start < rotated {
+ text = append(text, previous[start-prevStart:]...)
+ start = rotated
+ }
+ } else if start < rotated {
+ gap += rotated - start
+ start = rotated
+ }
+ if start < total {
+ off := start - rotated
+ if off < 0 {
+ off = 0
+ }
+ if off < int64(len(current)) {
+ text = append(text, current[off:]...)
+ }
+ }
+ return text, gap
+}
+
+func capToBudget(out *ShellJobStream, text []byte, budget int) []byte {
+ if len(text) <= budget {
+ return text
+ }
+ headN := budget / 4
+ tailN := budget - headN
+ head := copyFirst(text, headN)
+ tail := copyLast(text, tailN)
+ kept := make([]byte, 0, len(head)+len(tail))
+ kept = append(kept, head...)
+ kept = append(kept, tail...)
+ out.OmittedBytes += int64(len(text) - len(kept))
+ return kept
+}
+
+func shellOffset(from *ShellJobOffsets, which string) int64 {
+ if which == "Stdout" {
+ return from.Stdout
+ }
+ return from.Stderr
+}
+
+func sanitizeText(b []byte) string {
+ clean := make([]byte, 0, len(b))
+ for _, r := range string(b) {
+ if r == 0xFFFD {
+ continue
+ }
+ clean = append(clean, string(r)...)
+ }
+ return string(clean)
+}
+
+func readFileOrEmpty(path string) []byte {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return nil
+ }
+ return b
+}
+
+func readRotated(path string) int64 {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return 0
+ }
+ s := strings.TrimSpace(string(b))
+ if s == "" {
+ return 0
+ }
+ var v int64
+ for _, c := range s {
+ if c < '0' || c > '9' {
+ return 0
+ }
+ v = v*10 + int64(c-'0')
+ }
+ return v
+}
+
+func copyFirst(b []byte, n int) []byte {
+ if n <= 0 || len(b) == 0 {
+ return nil
+ }
+ if len(b) <= n {
+ n = len(b)
+ }
+ out := make([]byte, n)
+ copy(out, b[:n])
+ return out
+}
+
+func copyLast(b []byte, n int) []byte {
+ if n <= 0 || len(b) == 0 {
+ return nil
+ }
+ if len(b) <= n {
+ return copyFirst(b, len(b))
+ }
+ out := make([]byte, n)
+ copy(out, b[len(b)-n:])
+ return out
+}
diff --git a/internal/sandbox/shell_job_integration_test.go b/internal/sandbox/shell_job_integration_test.go
new file mode 100644
index 000000000..08e05dbad
--- /dev/null
+++ b/internal/sandbox/shell_job_integration_test.go
@@ -0,0 +1,538 @@
+//go:build docker_integration
+
+// Shell-job protocol against a real Docker daemon.
+//
+// Covers the remote rows of the shell spec's test matrix: a short command
+// finishes inside one exec, a long command returns a running handle, rotation
+// keeps an 8 KiB .head, TERM cancels the command process group while the
+// wrapper still writes the terminal record, and a submit-exec timeout does
+// not kill the setsid wrapper.
+//
+// The suite uses the same daemon gate as docker_integration_test.go
+// (DOCKER_INTEGRATION_IMAGE, DOCKER_INTEGRATION_HOST). Without the image it
+// skips; it does not fail.
+package sandbox
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestShellJobShortCommandOneRoundTrip: the command finishes inside the yield
+// window, so the same exec that submitted it returns the terminal record.
+func TestShellJobShortCommandOneRoundTrip(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ job := ShellJobMeta{ID: "sj_0123", Command: "echo hi", WorkDir: "/workspace"}
+ script, _ := ShellJobSubmitScript(job, job.Command, 5000)
+ res := dockerIntegrationExec(t, client, handle, script, 16*time.Second)
+ require.Equal(t, 0, res.ExitCode, "stderr=%s", res.Stderr)
+ require.False(t, res.Killed)
+ require.Contains(t, res.Stdout, "hi", "stdout=%s", clipForLog(res.Stdout))
+ require.Contains(t, res.Stdout, `"state":"succeeded"`, "窗口内结束直接带回终态")
+}
+
+// TestShellJobLongCommandReturnsRunningThenTerminal: the yield window expires
+// while the command is still running, so the submit exec returns a handle.
+// The wrapper keeps the command and later writes the terminal record itself.
+func TestShellJobLongCommandReturnsRunningThenTerminal(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ const id = "sj_ab01"
+ job := ShellJobMeta{ID: id, Command: "sleep 3; echo done", WorkDir: "/workspace"}
+ script, marker := ShellJobSubmitScript(job, job.Command, 1000)
+ res := dockerIntegrationExec(t, client, handle, script, 12*time.Second)
+ require.Equal(t, 0, res.ExitCode, "stderr=%s stdout=%s", res.Stderr, clipForLog(res.Stdout))
+ require.Contains(t, res.Stdout, marker)
+ require.Contains(t, res.Stdout, `"state":"running"`, "窗口到期仍是 running")
+ require.NotContains(t, res.Stdout, "done", "yield 返回时命令尚未结束")
+
+ var snap ShellJobSnapshot
+ require.Eventually(t, func() bool {
+ loaded, err := readRemoteJob(t, client, handle, id)
+ if err != nil || !WorkStateSucceededLike(loaded.Status.State) {
+ return false
+ }
+ snap = loaded
+ return true
+ }, 15*time.Second, 200*time.Millisecond)
+ require.True(t, WorkStateSucceededLike(snap.Status.State))
+ require.Equal(t, "done\n", snap.Stdout.Text)
+}
+
+// TestShellJobCancelKillsGroupWritesCancelled sends TERM to the watcher.
+// The trap kills the command process group and the wrapper still writes
+// cancelled (exit 143) afterwards.
+func TestShellJobCancelKillsGroupWritesCancelled(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ const id = "sj_fedc"
+ job := ShellJobMeta{ID: id, Command: "sleep 60", WorkDir: "/workspace"}
+ script, _ := ShellJobSubmitScript(job, job.Command, 1000)
+ res := dockerIntegrationExec(t, client, handle, script, 12*time.Second)
+ require.Equal(t, 0, res.ExitCode, "stderr=%s", res.Stderr)
+ snap, err := readRemoteJob(t, client, handle, id)
+ require.NoError(t, err)
+ require.Equal(t, "running", snap.Status.State)
+ require.NotZero(t, snap.Status.WatcherPID)
+ alive, ok := remoteCommandCount(t, client, handle, "sleep 60")
+ require.True(t, ok, "process probe failed before TERM")
+ require.Greater(t, alive, 0, "sleep 60 must be visible before TERM")
+
+ killed := dockerIntegrationExec(t, client, handle,
+ "kill -TERM "+strconv.Itoa(snap.Status.WatcherPID)+"; sleep 1",
+ 10*time.Second)
+ require.Equal(t, 0, killed.ExitCode, "stderr=%s", killed.Stderr)
+
+ require.Eventually(t, func() bool {
+ loaded, loadErr := readRemoteJob(t, client, handle, id)
+ if loadErr != nil || loaded.Status.State != "cancelled" {
+ return false
+ }
+ snap = loaded
+ n, countOK := remoteCommandCount(t, client, handle, "sleep 60")
+ return countOK && n == 0
+ }, 10*time.Second, 200*time.Millisecond)
+ require.Equal(t, "cancelled", snap.Status.State)
+ require.NotNil(t, snap.Status.ExitCode)
+ require.Equal(t, 143, *snap.Status.ExitCode, "wrapper 应活到写完 cancelled")
+ gone, ok := remoteCommandCount(t, client, handle, "sleep 60")
+ require.True(t, ok)
+ require.Zero(t, gone, "TERM 应杀掉命令进程组")
+}
+
+// TestShellJobRotationKeepsHeadAndCommandSurvives writes 20 MiB plus a tail
+// marker. The watcher rotates at 16 MiB, keeps the first 8 KiB in .head, and
+// the command still reaches the marker. Totals are .rotated plus the live
+// segment, not "one segment plus a 16-byte fudge".
+func TestShellJobRotationKeepsHeadAndCommandSurvives(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ const id = "sj_aa01"
+ // 20 MiB of x, then a marker that can only appear if the command keeps
+ // running across rotation. tr '\0' is the command the wrapper receives.
+ const command = "head -c 20971520 /dev/zero | tr '\\0' 'x'; echo TAILMARK"
+ job := ShellJobMeta{ID: id, Command: command, WorkDir: "/workspace"}
+ script, _ := ShellJobSubmitScript(job, job.Command, 1000)
+ // The submit loop forwards at most 1 MiB. 60s lets that copy finish; a
+ // kill here must not be treated as a rotation failure because the wrapper
+ // is already detached.
+ res, err := execInSandbox(client, handle, script, 60*time.Second)
+ require.NoError(t, err)
+ require.NotNil(t, res)
+
+ require.Eventually(t, func() bool {
+ state, ok := remoteJobState(client, handle, id)
+ return ok && state == "succeeded"
+ }, 90*time.Second, time.Second)
+
+ snap, root, err := loadRemoteJob(t, client, handle, id)
+ require.NoError(t, err)
+ require.Equal(t, "succeeded", snap.Status.State)
+ dir := filepath.Join(root, id)
+ head, err := os.ReadFile(filepath.Join(dir, "stdout.head"))
+ require.NoError(t, err)
+ require.Equal(t, shellJobHeadBytes, len(head), "rotation writes an 8 KiB .head")
+ require.Equal(t, bytes.Repeat([]byte{'x'}, shellJobHeadBytes), head, "开头保留")
+ info, err := os.Stat(filepath.Join(dir, "stdout.1"))
+ require.NoError(t, err)
+ require.Positive(t, info.Size())
+
+ rotatedRaw, err := os.ReadFile(filepath.Join(dir, "stdout.rotated"))
+ require.NoError(t, err)
+ rotated, err := strconv.ParseInt(strings.TrimSpace(string(rotatedRaw)), 10, 64)
+ require.NoError(t, err)
+ live, err := os.ReadFile(filepath.Join(dir, "stdout"))
+ require.NoError(t, err)
+ require.GreaterOrEqual(t, rotated, int64(shellJobSegmentBytes), ".rotated 至少跨过一个 16 MiB 段")
+ require.Equal(t, rotated+int64(len(live)), snap.Stdout.TotalBytes, "总字节 = .rotated + 当前段")
+ const written = int64(20971520 + len("TAILMARK\n"))
+ require.LessOrEqual(t, snap.Stdout.TotalBytes, written, "计数不超过命令写过的字节")
+ require.True(t, strings.HasPrefix(snap.Stdout.Text, "xxxx"), "开头保留")
+ require.Contains(t, streamTail(t, dir, "stdout"), "TAILMARK", "轮转后命令仍写完")
+ require.Contains(t, snap.Stdout.Text, "TAILMARK")
+}
+
+// TestShellJobSubmitExecTimeoutLeavesWrapperRunning kills the submit exec
+// with the in-container timeout. The wrapper was started under setsid, so
+// the command process group is still running afterwards.
+func TestShellJobSubmitExecTimeoutLeavesWrapperRunning(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ const id = "sj_cc03"
+ job := ShellJobMeta{ID: id, Command: "sleep 90", WorkDir: "/workspace"}
+ script, _ := ShellJobSubmitScript(job, job.Command, 30000)
+ res := dockerIntegrationExec(t, client, handle, script, 3*time.Second)
+ require.True(t, res.Killed, "submit exec must hit its own timeout, exit=%d stdout=%s stderr=%s",
+ res.ExitCode, clipForLog(res.Stdout), res.Stderr)
+
+ var snap ShellJobSnapshot
+ require.Eventually(t, func() bool {
+ loaded, loadErr := readRemoteJob(t, client, handle, id)
+ if loadErr != nil || loaded.Status.State != "running" || loaded.Status.WatcherPID == 0 {
+ return false
+ }
+ if !remoteWatcherAlive(t, client, handle, loaded.Status.WatcherPID, id) {
+ return false
+ }
+ n, ok := remoteCommandCount(t, client, handle, "sleep 90")
+ if !ok || n == 0 {
+ return false
+ }
+ snap = loaded
+ return true
+ }, 15*time.Second, 300*time.Millisecond)
+ require.Equal(t, "running", snap.Status.State)
+ require.NotZero(t, snap.Status.WatcherPID)
+ require.True(t, remoteWatcherAlive(t, client, handle, snap.Status.WatcherPID, id),
+ "timeout 不应杀掉 setsid wrapper")
+ n, ok := remoteCommandCount(t, client, handle, "sleep 90")
+ require.True(t, ok)
+ require.Greater(t, n, 0, "timeout 不应杀掉命令进程组")
+}
+
+// WorkStateSucceededLike reports the terminal state of a command that exited 0.
+func WorkStateSucceededLike(state string) bool {
+ return state == "succeeded"
+}
+
+func dockerIntegrationClient(t *testing.T) RemoteSandboxClient {
+ t.Helper()
+ cfg := dockerIntegrationConfig(t)
+ client, err := NewDockerRemoteClient(cfg)
+ require.NoError(t, err)
+ // This process must not sweep unrelated containers on a shared daemon.
+ client.sweeper = nil
+ probeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+ if err := client.Health(probeCtx); err != nil {
+ t.Skipf("docker daemon unreachable: %v", err)
+ }
+ return client
+}
+
+func dockerIntegrationHandle(t *testing.T, client RemoteSandboxClient) RemoteSandboxHandle {
+ t.Helper()
+ if os.Getenv("DOCKER_INTEGRATION_IMAGE") == "" {
+ t.Skip("DOCKER_INTEGRATION_IMAGE not set")
+ }
+ return dockerIntegrationHandleFrom(t, client)
+}
+
+func dockerIntegrationHandleFrom(t *testing.T, client RemoteSandboxClient) RemoteSandboxHandle {
+ t.Helper()
+ image := strings.TrimSpace(os.Getenv("DOCKER_INTEGRATION_IMAGE"))
+ if image == "" {
+ t.Skip("DOCKER_INTEGRATION_IMAGE not set")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ handle, err := client.Create(ctx, RemoteCreateRequest{
+ TemplateID: image,
+ Metadata: map[string]string{"weknora.test": "shell-job"},
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cleanupCancel()
+ if err := client.Delete(cleanupCtx, handle.ID()); err != nil {
+ t.Errorf("delete shell-job sandbox: %v", err)
+ }
+ })
+ return handle
+}
+
+func dockerIntegrationExec(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ script string,
+ timeout time.Duration,
+) *RemoteExecResult {
+ t.Helper()
+ res, err := execInSandbox(client, handle, script, timeout)
+ require.NoError(t, err)
+ require.NotNil(t, res)
+ return res
+}
+
+func execInSandbox(
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ script string,
+ timeout time.Duration,
+) (*RemoteExecResult, error) {
+ return client.Exec(context.Background(), handle, RemoteExecRequest{
+ Command: script,
+ Shell: true,
+ Timeout: timeout,
+ })
+}
+
+// readRemoteJob copies one job directory out of the sandbox and reads it with
+// ReadShellJob. The wrapper's echo writes ".rotated" with a trailing newline;
+// ReadShellJob rejects that newline, so the local copy is trimmed to the same
+// integer the in-sandbox shell reader already accepts.
+func readRemoteJob(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ id string,
+) (ShellJobSnapshot, error) {
+ t.Helper()
+ snap, _, err := loadRemoteJob(t, client, handle, id)
+ return snap, err
+}
+
+func loadRemoteJob(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ id string,
+) (ShellJobSnapshot, string, error) {
+ t.Helper()
+ root, err := fetchRemoteJobRoot(t, client, handle, id)
+ if err != nil {
+ return ShellJobSnapshot{}, "", err
+ }
+ if err := trimRotatedCounters(filepath.Join(root, id)); err != nil {
+ return ShellJobSnapshot{}, "", err
+ }
+ snap, err := ReadShellJob(root, ShellJobRead{ID: id})
+ if err != nil {
+ return ShellJobSnapshot{}, "", err
+ }
+ return snap, root, nil
+}
+
+func fetchRemoteJobRoot(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ id string,
+) (string, error) {
+ t.Helper()
+ if !ValidShellJobID(id) {
+ return "", fmt.Errorf("sandbox: invalid shell job id %q", id)
+ }
+ res, err := execInSandbox(client, handle,
+ "tar -C "+shQuote(ShellJobsRoot)+" -cf - "+shQuote(id),
+ time.Minute)
+ if err != nil {
+ return "", err
+ }
+ if res == nil {
+ return "", fmt.Errorf("tar job %s: no result", id)
+ }
+ if res.ExitCode != 0 {
+ return "", fmt.Errorf("tar job %s: exit %d stderr=%s", id, res.ExitCode, clipForLog(res.Stderr))
+ }
+ root := t.TempDir()
+ cmd := exec.Command("tar", "-C", root, "-xf", "-")
+ cmd.Stdin = strings.NewReader(res.Stdout)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return "", fmt.Errorf("untar job %s: %w %s", id, err, clipForLog(string(out)))
+ }
+ return root, nil
+}
+
+// trimRotatedCounters strips the trailing newline echo writes into
+// stdout.rotated and stderr.rotated. readRotated treats any non-digit as a
+// corrupt counter and would otherwise report 0 after a real rotation.
+func trimRotatedCounters(dir string) error {
+ for _, name := range []string{"stdout.rotated", "stderr.rotated"} {
+ path := filepath.Join(dir, name)
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ return err
+ }
+ trimmed := strings.TrimSpace(string(raw))
+ if trimmed == string(raw) {
+ continue
+ }
+ if err := os.WriteFile(path, []byte(trimmed), 0o644); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func remoteJobState(client RemoteSandboxClient, handle RemoteSandboxHandle, id string) (string, bool) {
+ if !ValidShellJobID(id) {
+ return "", false
+ }
+ res, err := execInSandbox(client, handle,
+ "cat "+shQuote(path.Join(ShellJobsRoot, id, "status.json")),
+ 15*time.Second)
+ if err != nil || res == nil || res.ExitCode != 0 {
+ return "", false
+ }
+ var status ShellJobStatus
+ if err := json.Unmarshal([]byte(res.Stdout), &status); err != nil || status.State == "" {
+ return "", false
+ }
+ return status.State, true
+}
+
+// remoteCommandCount counts processes whose cmdline is exactly argv or starts
+// with "argv ". The probe's own shell contains the text but does not start
+// with it, so it cannot match itself.
+func remoteCommandCount(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ argv string,
+) (int, bool) {
+ t.Helper()
+ script := fmt.Sprintf(`n=0
+for p in /proc/[0-9]*; do
+ cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null || true)
+ case "$cmd" in
+ %s|%s*) n=$((n+1));;
+ esac
+done
+echo "$n"
+`, shQuote(argv), shQuote(argv+" "))
+ res, err := execInSandbox(client, handle, script, 15*time.Second)
+ if err != nil || res == nil || res.ExitCode != 0 {
+ return 0, false
+ }
+ n, err := strconv.Atoi(strings.TrimSpace(res.Stdout))
+ if err != nil {
+ return 0, false
+ }
+ return n, true
+}
+
+func remoteWatcherAlive(
+ t *testing.T,
+ client RemoteSandboxClient,
+ handle RemoteSandboxHandle,
+ pid int,
+ jobID string,
+) bool {
+ t.Helper()
+ if pid <= 0 || !ValidShellJobID(jobID) {
+ return false
+ }
+ script := fmt.Sprintf(
+ `if [ -r /proc/%d/cmdline ]; then tr '\0' ' ' < /proc/%d/cmdline; fi`,
+ pid, pid)
+ res, err := execInSandbox(client, handle, script, 15*time.Second)
+ if err != nil || res == nil || res.ExitCode != 0 {
+ return false
+ }
+ return strings.Contains(res.Stdout, jobID)
+}
+
+// streamTail is the retained end of a stream: the live segment when it holds
+// the marker, otherwise the rotated-away .1 copy.
+func streamTail(t *testing.T, dir, name string) string {
+ t.Helper()
+ live, err := os.ReadFile(filepath.Join(dir, name))
+ require.NoError(t, err)
+ if strings.Contains(string(live), "TAILMARK") {
+ return string(live)
+ }
+ prev, err := os.ReadFile(filepath.Join(dir, name+".1"))
+ require.NoError(t, err)
+ return string(prev)
+}
+
+func clipForLog(s string) string {
+ const limit = 400
+ if len(s) <= limit {
+ return s
+ }
+ return s[:limit] + "...(truncated)"
+}
+
+func TestShellJobSharedMachineCancelChecksOwnerAndStopsStubbornProcess(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ jobs := []ShellJobMeta{
+ {
+ ID: "sj_a111", TenantID: 7, SessionID: "session-a", SandboxID: handle.ID(),
+ Command: `trap '' TERM; while :; do sleep 1; done`,
+ },
+ {ID: "sj_b111", TenantID: 7, SessionID: "session-b", SandboxID: handle.ID(), Command: "sleep 60"},
+ }
+ for _, job := range jobs {
+ script, _ := ShellJobSubmitScript(job, job.Command, 250)
+ res := dockerIntegrationExec(t, client, handle, script, 10*time.Second)
+ require.Equal(t, 0, res.ExitCode, res.Stderr)
+ }
+ // Even if ownership changed after the host-side read, the signal exec must
+ // verify persisted tenant, session and sandbox identity again.
+ for _, wrong := range []ShellJobMeta{
+ {ID: jobs[1].ID, TenantID: 7, SessionID: "session-a", SandboxID: handle.ID()},
+ {ID: jobs[1].ID, TenantID: 70, SessionID: "session-b", SandboxID: handle.ID()},
+ {ID: jobs[1].ID, TenantID: 7, SessionID: "session-b", SandboxID: "replaced"},
+ } {
+ res := dockerIntegrationExec(t, client, handle, ShellJobCancelScript(wrong, true), 10*time.Second)
+ require.Equal(t, 3, res.ExitCode)
+ }
+ res := dockerIntegrationExec(t, client, handle, ShellJobCancelScript(jobs[0], true), 10*time.Second)
+ require.Equal(t, 0, res.ExitCode, res.Stderr)
+ require.Contains(t, res.Stdout, `"state":"cancelled"`)
+ snapshots := ParseShellJobBatch(dockerIntegrationExec(t, client, handle,
+ ShellJobBatchReadScript(handle.ID(), []string{jobs[0].ID, jobs[1].ID}), 10*time.Second).Stdout)
+ require.Len(t, snapshots, 2)
+ require.Equal(t, "cancelled", snapshots[0].Status.State)
+ require.Equal(t, "running", snapshots[1].Status.State)
+ require.True(t, snapshots[1].WatcherAlive, "another chat on the same machine keeps running")
+}
+
+func TestShellJobBudgetedIncrementalReadAfterRotation(t *testing.T) {
+ client := dockerIntegrationClient(t)
+ handle := dockerIntegrationHandle(t, client)
+ id := "sj_c111"
+ script := `D=/var/lib/weknora/jobs/sj_c111; mkdir -p "$D"
+printf '%s' '{"work_id":"sj_c111"}' > "$D/meta.json"
+printf '%s' '{"state":"succeeded"}' > "$D/status.json"
+printf 65536 > "$D/stdout.rotated"
+head -c 20000 /dev/zero | tr '\0' x > "$D/stdout.1"
+printf FINAL-DIAGNOSTIC > "$D/stdout"
+printf STDERR > "$D/stderr"
+`
+ require.Equal(t, 0, dockerIntegrationExec(t, client, handle, script, 10*time.Second).ExitCode)
+ offsets := ShellJobOffsets{Stdout: 1}
+ var output strings.Builder
+ for i := 0; i < 5; i++ {
+ readScript := ShellJobBatchReadScriptWithBudget(
+ handle.ID(), []string{id}, map[string]ShellJobOffsets{id: offsets}, 12000,
+ )
+ res := dockerIntegrationExec(t, client, handle, readScript, 10*time.Second)
+ require.Equal(t, 0, res.ExitCode, res.Stderr)
+ snapshots := ParseShellJobBatch(res.Stdout)
+ require.Len(t, snapshots, 1)
+ snap := snapshots[0]
+ require.LessOrEqual(t, len(snap.Stdout.Text)+len(snap.Stderr.Text), 12000)
+ if i == 0 {
+ require.Positive(t, snap.Stdout.OmittedBytes)
+ }
+ output.WriteString(snap.Stdout.Text)
+ require.NotNil(t, snap.Stdout.NextBytes)
+ offsets = ShellJobOffsets{Stdout: *snap.Stdout.NextBytes, Stderr: *snap.Stderr.NextBytes}
+ if offsets.Stdout == snap.Stdout.TotalBytes {
+ break
+ }
+ }
+ require.Equal(t, strings.Repeat("x", 20000)+"FINAL-DIAGNOSTIC", output.String())
+}
diff --git a/internal/sandbox/shell_job_scripts.go b/internal/sandbox/shell_job_scripts.go
new file mode 100644
index 000000000..a751c8cef
--- /dev/null
+++ b/internal/sandbox/shell_job_scripts.go
@@ -0,0 +1,662 @@
+package sandbox
+
+import (
+ "bytes"
+ "crypto/rand"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+const shellJobMarkerPrefix = "__WEKNORA_SJ__"
+
+// shellJobRotateFunc keeps an 8 KiB head and rotates a stream at 16 MiB.
+// Truncating with `: >` leaves no sparse hole; writers only append.
+var shellJobRotateFunc = fmt.Sprintf(`r(){ F="$D/$1"; S=$(wc -c < "$F" 2>/dev/null || echo 0);
+ if [ "$S" -ge %d ]; then
+ [ -f "$F.head" ] || head -c %d "$F" > "$F.head";
+ cp "$F" "$F.1"; R=$(cat "$F.rotated" 2>/dev/null || echo 0);
+ echo $(( R + S )) > "$F.rotated.tmp" && mv "$F.rotated.tmp" "$F.rotated";
+ : > "$F";
+ fi; }`, shellJobSegmentBytes, shellJobHeadBytes)
+
+// shellJobForwardLoop is the submit-side poll. It copies new bytes until the
+// watcher leaves "running" or the yield deadline passes. It does not wait on
+// the wrapper. Each stream may forward at most 1 MiB for the whole window:
+// a poll larger than the remaining allowance forwards only that remainder,
+// then the read cursor moves to the bytes just observed so the next poll
+// neither rereads them nor jumps over bytes already added to the allowance.
+// The 50ms tick is how soon a finished command is noticed. It is not a wait
+// for the yield deadline, and it is long enough to avoid a tight poll loop.
+const shellJobForwardLoop = `W=$!;
+sleep 0.05;
+SO=0; SE=0; BO=0; BE=0; END=$(( $(date +%s)000 + @@YIELD@@ ));
+p() {
+ if [ "$1" = stdout ]; then
+ CUR=$SO
+ BUD=$BO
+ else
+ CUR=$SE
+ BUD=$BE
+ fi
+ N=$(wc -c < "$D/$1" 2>/dev/null || echo 0)
+ N=$(printf '%s' "$N" | tr -d '[:space:]')
+ [ -n "$N" ] || N=0
+ if [ "$N" -gt "$CUR" ]; then
+ DLT=$(( N - CUR ))
+ ROOM=$(( 1048576 - BUD ))
+ if [ "$ROOM" -gt 0 ]; then
+ TAKE=$DLT
+ [ "$TAKE" -gt "$ROOM" ] && TAKE=$ROOM
+ if [ "$1" = stdout ]; then
+ dd if="$D/$1" iflag=skip_bytes,count_bytes skip="$CUR" count="$TAKE" 2>/dev/null
+ else
+ dd if="$D/$1" iflag=skip_bytes,count_bytes skip="$CUR" count="$TAKE" 2>/dev/null >&2
+ fi
+ BUD=$(( BUD + TAKE ))
+ fi
+ CUR=$N
+ fi
+ if [ "$1" = stdout ]; then
+ SO=$CUR
+ BO=$BUD
+ else
+ SE=$CUR
+ BE=$BUD
+ fi
+}
+while :; do
+ if [ -f "$D/status.json" ]; then
+ p stdout
+ p stderr
+ grep -q '"state":"running"' "$D/status.json" || break
+ fi
+ [ "$(date +%s)000" -ge "$END" ] && break
+ sleep 0.05
+done
+p stdout
+p stderr
+`
+
+// shellJobAbsoluteSizes publishes rotated-away bytes plus the live segment.
+// wc -c of the live file alone shrinks the reported total after rotation.
+const shellJobAbsoluteSizes = `RO=$(cat "$D/stdout.rotated" 2>/dev/null || echo 0)
+RE=$(cat "$D/stderr.rotated" 2>/dev/null || echo 0)
+LN=$(wc -c < "$D/stdout" 2>/dev/null || echo 0)
+LE=$(wc -c < "$D/stderr" 2>/dev/null || echo 0)
+RO=$(printf '%s' "$RO" | tr -d '[:space:]'); [ -n "$RO" ] || RO=0
+RE=$(printf '%s' "$RE" | tr -d '[:space:]'); [ -n "$RE" ] || RE=0
+LN=$(printf '%s' "$LN" | tr -d '[:space:]'); [ -n "$LN" ] || LN=0
+LE=$(printf '%s' "$LE" | tr -d '[:space:]'); [ -n "$LE" ] || LE=0
+SO=$(( RO + LN ))
+SE=$(( RE + LE ))
+`
+
+const shellJobEpiloguePrint = `printf '\n%s\n' "$M"; cat "$D/status.json"; printf '%s %s\n' "$SO" "$SE"
+`
+
+// shellJobTermTrap is installed before the command starts. $C is expanded
+// when the trap runs, not when it is installed.
+const shellJobTermTrap = `trap 'ST=cancelled; kill -TERM -$C 2>/dev/null' TERM;`
+
+// shellJobTimeStamp is UTC with milliseconds. Duration is shown in ms, so a
+// whole-second clock would report 0 for any command that starts and ends
+// inside the same second.
+const shellJobTimeStamp = `date -u +%Y-%m-%dT%H:%M:%S.%3NZ`
+
+const shellJobRunningStatus = `START=$(` + shellJobTimeStamp + `);
+echo '{"state":"running","watcher_pid":'"$$"',"pgid":'"$C"',"touch_pid":'"$TOUCH"',"started_at":"'"$START"'"}' \
+ > "$D/status.json.tmp" && mv "$D/status.json.tmp" "$D/status.json";`
+
+// shellJobRotateLoop is backgrounded. The parent must wait $C itself: on
+// dash, kill -0 stays true for a zombie and only wait reaps, so a foreground
+// `while kill -0; do sleep; done; wait` never reaches wait. sleep is itself
+// backgrounded so the parent's TERM ends the tick instead of waiting it out.
+const shellJobRotateLoop = `rot() {
+ SP=
+ trap '[ -n "$SP" ] && kill $SP 2>/dev/null; exit 0' TERM
+ while kill -0 "$C" 2>/dev/null; do
+ sleep 5 &
+ SP=$!
+ wait $SP
+ SP=
+ kill -0 "$C" 2>/dev/null || exit 0
+ r stdout
+ r stderr
+ done
+}
+rot &
+ROT=$!;
+`
+
+// shellJobReap waits in the parent, then rotates once more so a command that
+// exits before the first 5s tick still rotates a segment past 16 MiB.
+// A TERM trap can make the first wait return before dash reaps; kill -0 is
+// still true for that zombie, so wait once more. The body is wait, not sleep.
+const shellJobReap = `wait "$C"; X=$?;
+if kill -0 "$C" 2>/dev/null; then
+ wait "$C"; X=$?;
+fi
+kill -TERM $ROT 2>/dev/null
+wait $ROT 2>/dev/null
+r stdout
+r stderr
+`
+
+// shellJobTerminalStatus writes the terminal state exactly once.
+// A prior printf-to-/dev/null draft of this record was a build leftover.
+const shellJobTerminalStatus = `if [ "$ST" = cancelled ]; then
+ X=143
+ STATE=cancelled
+elif [ "$X" = 0 ]; then
+ STATE=succeeded
+else
+ STATE=failed
+fi
+E=$(` + shellJobTimeStamp + `);
+echo "{\"state\":\"$STATE\",\"watcher_pid\":$$,\"pgid\":$C,\"exit_code\":$X,\
+\"started_at\":\"$START\",\"ended_at\":\"$E\"}" \
+ > "$D/status.json.tmp" && mv "$D/status.json.tmp" "$D/status.json";`
+
+// One batch-read body. $J and $D are set by the caller. A missing status
+// produces an explicit Missing record, distinct from a broken response. Totals are
+// .rotated plus the live segment (shellJobAbsoluteSizes).
+const shellJobBatchHead = `if [ -f "$D/status.json" ]; then
+ META=$(cat "$D/meta.json" 2>/dev/null || printf '%s' '{}')
+ ST=$(cat "$D/status.json")
+`
+
+const shellJobBatchTail = `
+ PID=$(sed -n 's/.*"watcher_pid"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' "$D/status.json")
+ ALIVE=false
+ if [ -n "$PID" ] && [ -r "/proc/$PID/cmdline" ]; then
+ if tr '\0' ' ' < "/proc/$PID/cmdline" | grep -F -q -- "$J"; then
+ ALIVE=true
+ fi
+ fi
+ printf '%s' '{"meta":'
+ printf '%s' "$META"
+ printf '%s' ',"status":'
+ printf '%s' "$ST"
+ printf '%s' ',"stdout":'
+ stream_json stdout "$OH" "$OT" "$SF"
+ printf '%s' ',"stderr":'
+ stream_json stderr "$EH" "$ET" "$EF"
+ printf '%s' ',"watcher_alive":'
+ printf '%s' "$ALIVE"
+ printf '}\n'
+else
+ printf '{"meta":{"work_id":"%s"},"missing":true}\n' "$J"
+fi
+`
+
+// shellJobBatchPrelude defines stream_json. A first read (empty cursor) sends
+// the head slice and the tail slice. A cursor sends only the bytes after it,
+// capped at the stream budget. The host assembles those slices; the script
+// does not decide truncation.
+const shellJobBatchPrelude = `b64() { base64 2>/dev/null | tr -d '\n'; }
+ntrim() { printf '%s' "$1" | tr -d '[:space:]'; }
+stream_json() {
+ NAME=$1; HB=$2; TB=$3; FROM=$4
+ F="$D/$NAME"; ROTF="$F.rotated"; PREVF="$F.1"; HEADF="$F.head"
+ RO=0
+ if [ -f "$ROTF" ]; then RO=$(ntrim "$(cat "$ROTF" 2>/dev/null || echo 0)"); [ -n "$RO" ] || RO=0; fi
+ LN=0
+ if [ -f "$F" ]; then LN=$(ntrim "$(wc -c < "$F" || echo 0)"); [ -n "$LN" ] || LN=0; fi
+ PN=0
+ if [ -f "$PREVF" ]; then PN=$(ntrim "$(wc -c < "$PREVF" || echo 0)"); [ -n "$PN" ] || PN=0; fi
+ TOTAL=$(( RO + LN ))
+ if [ -n "$FROM" ]; then
+ BUD=$(( HB + TB ))
+ PREV_START=$(( RO - PN ))
+ OMITTED=0
+ if [ "$FROM" -ge "$TOTAL" ]; then
+ BODY=""; FROM=$TOTAL
+ elif [ "$FROM" -ge "$RO" ]; then
+ BODY=$(dd if="$F" iflag=skip_bytes,count_bytes skip=$(( FROM - RO )) count=$BUD 2>/dev/null | b64)
+ elif [ "$PN" -gt 0 ] && [ "$FROM" -ge "$PREV_START" ]; then
+ SKIP=$(( FROM - PREV_START ))
+ BODY=$( { tail -c +$(( SKIP + 1 )) "$PREVF"; cat "$F"; } | head -c $BUD | b64)
+ elif [ "$PN" -gt 0 ]; then
+ OMITTED=$(( PREV_START - FROM )); FROM=$PREV_START
+ BODY=$( { cat "$PREVF"; cat "$F"; } | head -c $BUD | b64)
+ else
+ OMITTED=$(( RO - FROM )); FROM=$RO
+ BODY=$(head -c $BUD "$F" 2>/dev/null | b64)
+ fi
+ printf '%s' "{\"total_bytes\":$TOTAL,\"slices\":true,\"from\":$FROM,"
+ printf '%s' "\"omitted_bytes\":$OMITTED,\"body_b64\":\"$BODY\"}"
+ return
+ fi
+ if [ -f "$HEADF" ]; then
+ HEAD_FROM=true; HEAD_START=0
+ HEADB=$(head -c "$HB" "$HEADF" | b64)
+ else
+ HEAD_FROM=false; HEAD_START=$RO
+ HEADB=$(head -c "$HB" "$F" 2>/dev/null | b64)
+ fi
+ if [ "$LN" -ge "$TB" ]; then
+ TAILB=$(tail -c "$TB" "$F" | b64)
+ TAIL_START=$(( TOTAL - TB ))
+ else
+ PAD=$(( TB - LN ))
+ TAKE=0
+ if [ -f "$PREVF" ]; then
+ TAKE=$PAD; [ "$PN" -lt "$TAKE" ] && TAKE=$PN
+ TAILB=$( { tail -c "$TAKE" "$PREVF"; cat "$F"; } | b64)
+ else
+ TAILB=$(cat "$F" 2>/dev/null | b64)
+ fi
+ TAIL_START=$(( TOTAL - TAKE - LN ))
+ fi
+ printf '%s' "{\"total_bytes\":$TOTAL,\"slices\":true,\"head_b64\":\"$HEADB\","
+ printf '%s' "\"head_from_file\":$HEAD_FROM,\"head_start\":$HEAD_START,"
+ printf '%s' "\"tail_b64\":\"$TAILB\",\"tail_start\":$TAIL_START}"
+}
+`
+
+// ShellJobSubmitScript builds the one Exec that (1) creates the job dir and
+// meta.json, (2) launches the setsid wrapper without waiting, (3) forwards new
+// output bytes to its own stdout/stderr until the job ends or yieldMS elapses,
+// then emits the marker line, the final status.json and the absolute offsets.
+// A short command therefore costs exactly one exec round-trip.
+// meta.json is the masked job. execCommand is only the wrapper argument.
+func ShellJobSubmitScript(job ShellJobMeta, execCommand string, yieldMS int) (string, string) {
+ return ShellJobSubmitScriptWithBackend(job, execCommand, yieldMS, false)
+}
+
+// ShellJobSubmitScriptWithBackend is ShellJobSubmitScript with the backend
+// chosen by the caller. Docker is the only backend whose wrapper must touch
+// the activity marker; the sandbox id is not a backend name.
+func ShellJobSubmitScriptWithBackend(job ShellJobMeta, execCommand string, yieldMS int, docker bool) (string, string) {
+ marker := shellJobMarkerPrefix + "_" + randomHex(6) + "__"
+ meta := mustJSON(job)
+ wrapper := shellJobWrapperScript(job, execCommand, docker)
+ var b strings.Builder
+ fmt.Fprintf(&b, "J=%q; R=%s; mkdir -p \"$R\" || exit 9; D=\"$R/$J\"; %s\n",
+ job.ID, shQuote(ShellJobsRoot), shellJobCreateDir())
+ fmt.Fprintf(&b, "printf '%%s' %s > \"$D/meta.json\";\n", shQuote(meta))
+ fmt.Fprintf(&b, "M=%s;\n", shQuote(marker))
+ fmt.Fprintf(&b, "setsid /bin/sh -c %s >/dev/null 2>&1 </dev/null &\n", shSingleQuote(wrapper))
+ b.WriteString(strings.ReplaceAll(shellJobForwardLoop, "@@YIELD@@", strconv.Itoa(yieldMS)))
+ b.WriteString(shellJobAbsoluteSizes)
+ b.WriteString(shellJobEpiloguePrint)
+ return b.String(), marker
+}
+
+// shellJobWrapperScript is the watcher: the ONLY writer of status.json. It
+// starts the command in its own session (setsid), appends both streams to
+// files, rotates at 16 MiB keeping an 8 KiB head, optionally touches the
+// docker activity marker, traps TERM into "cancelled", and writes the terminal
+// state exactly once. The parent shell waits on the command; the 5s rotator
+// runs beside that wait.
+func shellJobWrapperScript(job ShellJobMeta, execCommand string, docker bool) string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "D=%s/%s; J=%s; ST=running;\n", ShellJobsRoot, job.ID, job.ID)
+ b.WriteString(shellJobRotateFunc)
+ b.WriteByte('\n')
+ b.WriteString(shellJobTermTrap)
+ b.WriteByte('\n')
+ fmt.Fprintf(&b, "setsid /bin/bash -c %s >>\"$D/stdout\" 2>>\"$D/stderr\" </dev/null &\n",
+ shSingleQuote(execCommand))
+ b.WriteString("C=$!;\n")
+ if docker {
+ b.WriteString("( while :; do sleep 30; touch /var/lib/weknora-sandbox-activity; done ) &\n")
+ b.WriteString("TOUCH=$!;\n")
+ } else {
+ b.WriteString("TOUCH=0;\n")
+ }
+ b.WriteString(shellJobRunningStatus)
+ b.WriteByte('\n')
+ b.WriteString(shellJobRotateLoop)
+ b.WriteString(shellJobReap)
+ b.WriteString(shellJobTerminalStatus)
+ b.WriteByte('\n')
+ b.WriteString(touchKill(docker))
+ b.WriteByte('\n')
+ return b.String()
+}
+
+// shellJobCreateDir refuses to reuse a live or recent job directory. A
+// terminal directory whose status is older than the notice retention may be
+// removed so the 16-bit id space can be used again.
+func shellJobCreateDir() string {
+ return fmt.Sprintf(`mkdir "$D" || {
+ if [ -f "$D/status.json" ] && ! grep -q '"state":"running"' "$D/status.json" &&
+ [ -n "$(find "$D/status.json" -mmin +1440 2>/dev/null)" ]; then
+ rm -rf "$D" && mkdir "$D" || exit %d
+ else
+ exit %d
+ fi
+};`, ShellJobDirExistsExit, ShellJobDirExistsExit)
+}
+
+// ShellJobKillOrphansScript stops a command whose watcher has died, and the
+// docker activity loop that would otherwise keep the sandbox awake. Pid 1 is
+// never a target. A missing process is not a failed exec.
+func ShellJobKillOrphansScript(pgid, touchPID int) string {
+ if pgid < 0 {
+ pgid = 0
+ }
+ if touchPID < 0 {
+ touchPID = 0
+ }
+ return fmt.Sprintf(`PGID=%d; TOUCH=%d;
+if [ "$PGID" -gt 1 ] && [ -d "/proc/$PGID" ]; then
+ kill -TERM -$PGID 2>/dev/null || true
+ sleep 0.2
+ kill -KILL -$PGID 2>/dev/null || true
+fi
+if [ "$TOUCH" -gt 1 ] && [ -r "/proc/$TOUCH/cmdline" ] &&
+ tr '\0' ' ' < "/proc/$TOUCH/cmdline" | grep -F -q weknora-sandbox-activity; then
+ kill -KILL "$TOUCH" 2>/dev/null || true
+fi
+true
+`, pgid, touchPID)
+}
+
+func touchKill(docker bool) string {
+ if docker {
+ return "kill $TOUCH 2>/dev/null"
+ }
+ return ":"
+}
+
+func shSingleQuote(s string) string { return shQuote(s) }
+
+func mustJSON(v any) string {
+ raw, err := json.Marshal(v)
+ if err != nil {
+ panic(err)
+ }
+ return string(raw)
+}
+
+func randomHex(n int) string {
+ buf := make([]byte, n)
+ if _, err := rand.Read(buf); err != nil {
+ return "000000000000"
+ }
+ return hex.EncodeToString(buf)
+}
+
+func shQuote(s string) string {
+ return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// ShellJobBatchReadScript is the sweeper / detail-read script. For each valid
+// id it cats meta.json and status.json, sends a bounded head and tail (or the
+// bytes after a cursor), checks /proc/<watcher_pid>/cmdline for the job id,
+// and prints one JSON object. Ids with no status.json produce a Missing record.
+// sandboxID is the caller's exec target; the script runs inside that sandbox
+// and does not embed it. from is nil for a first read.
+func ShellJobBatchReadScript(sandboxID string, ids []string) string {
+ return ShellJobBatchReadScriptFor(sandboxID, ids, nil)
+}
+
+// ShellJobBatchReadScriptFor is ShellJobBatchReadScript with a per-id cursor.
+// A missing id in from is a first read. A present offset, including 0, is incremental.
+func ShellJobBatchReadScriptFor(sandboxID string, ids []string, from map[string]ShellJobOffsets) string {
+ return ShellJobBatchReadScriptWithBudget(sandboxID, ids, from, 0)
+}
+
+// ShellJobBatchReadScriptWithBudget bounds the combined stream bytes per job.
+func ShellJobBatchReadScriptWithBudget(
+ sandboxID string, ids []string, from map[string]ShellJobOffsets, budget int,
+) string {
+ _ = sandboxID
+ var b strings.Builder
+ wrote := false
+ for _, id := range ids {
+ if !ValidShellJobID(id) {
+ continue
+ }
+ if !wrote {
+ b.WriteString(shellJobBatchPrelude)
+ stdout, stderr := streamBudget("Stdout", budget), streamBudget("Stderr", budget)
+ fmt.Fprintf(&b, "OH=%d; OT=%d; EH=%d; ET=%d;\n",
+ stdout/4, stdout-stdout/4, stderr/4, stderr-stderr/4)
+ wrote = true
+ }
+ sf, ef := "", ""
+ if from != nil {
+ if off, ok := from[id]; ok {
+ sf = strconv.FormatInt(off.Stdout, 10)
+ ef = strconv.FormatInt(off.Stderr, 10)
+ }
+ }
+ fmt.Fprintf(&b, "J=%s; D=%s; SF=%s; EF=%s;\n",
+ shQuote(id), shQuote(ShellJobsRoot+"/"+id), shQuote(sf), shQuote(ef))
+ b.WriteString(shellJobBatchHead)
+ b.WriteString(shellJobBatchTail)
+ }
+ if !wrote {
+ return "true\n"
+ }
+ return b.String()
+}
+
+// ParseShellJobBatch decodes one JSON snapshot per non-empty line.
+// Lines that are not objects are skipped. Missing records are explicit; an
+// absent or malformed line is a read failure for the caller. Slice fields are
+// assembled with the same head/tail rules as ReadShellJob.
+func ParseShellJobBatch(stdout string) []ShellJobSnapshot {
+ out := make([]ShellJobSnapshot, 0)
+ for _, line := range strings.Split(stdout, "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || line == "null" {
+ continue
+ }
+ var raw batchLine
+ if err := json.Unmarshal([]byte(line), &raw); err != nil {
+ continue
+ }
+ snap := ShellJobSnapshot{
+ Meta: raw.Meta, Status: raw.Status, WatcherAlive: raw.WatcherAlive, Missing: raw.Missing,
+ }
+ snap.Stdout = raw.Stdout.stream()
+ snap.Stderr = raw.Stderr.stream()
+ out = append(out, snap)
+ }
+ return out
+}
+
+type batchLine struct {
+ Missing bool `json:"missing"`
+ Meta ShellJobMeta `json:"meta"`
+ Status ShellJobStatus `json:"status"`
+ WatcherAlive bool `json:"watcher_alive"`
+ Stdout batchStream `json:"stdout"`
+ Stderr batchStream `json:"stderr"`
+}
+
+type batchStream struct {
+ TotalBytes int64 `json:"total_bytes"`
+ Slices bool `json:"slices"`
+ HeadB64 string `json:"head_b64"`
+ HeadFromFile bool `json:"head_from_file"`
+ HeadStart int64 `json:"head_start"`
+ TailB64 string `json:"tail_b64"`
+ TailStart int64 `json:"tail_start"`
+ From *int64 `json:"from"`
+ BodyB64 string `json:"body_b64"`
+ Omitted int64 `json:"omitted_bytes"`
+}
+
+func (b batchStream) stream() ShellJobStream {
+ out := ShellJobStream{TotalBytes: b.TotalBytes}
+ if !b.Slices {
+ return out
+ }
+ if b.From != nil {
+ body := decodeB64(b.BodyB64)
+ next := min(b.TotalBytes, *b.From+int64(len(body)))
+ out.NextBytes = &next
+ out.FromBytes = *b.From
+ out.OmittedBytes = b.Omitted
+ out.Text = sanitizeText(body)
+ if out.OmittedBytes > 0 || int64(len(body)) != out.TotalBytes-out.FromBytes {
+ out.Truncated = true
+ }
+ return out
+ }
+ head := decodeB64(b.HeadB64)
+ next := b.TotalBytes
+ out.NextBytes = &next
+ tail := decodeB64(b.TailB64)
+ text, textStart, gap := joinHeadTail(head, b.HeadStart, b.HeadFromFile, tail, b.TailStart)
+ out.FromBytes = textStart
+ out.OmittedBytes = textStart + gap + (b.TotalBytes - (textStart + int64(len(text)) + gap))
+ if out.OmittedBytes < 0 {
+ out.OmittedBytes = 0
+ }
+ if out.OmittedBytes > 0 || int64(len(text)) != b.TotalBytes-out.FromBytes {
+ out.Truncated = true
+ }
+ out.Text = sanitizeText(text)
+ return out
+}
+
+func decodeB64(s string) []byte {
+ if s == "" {
+ return nil
+ }
+ raw, err := base64.StdEncoding.DecodeString(s)
+ if err != nil {
+ return nil
+ }
+ return raw
+}
+
+// ShellJobMarkerStream strips the trailing marker line and everything after it
+// from forwarded output, holding back at most len(marker)-1 bytes so a marker
+// split across chunks is never leaked into the preview or the result body.
+type ShellJobMarkerStream struct {
+ marker string
+ held []byte
+ found bool
+ after []byte
+}
+
+// NewShellJobMarkerStream splits forwarded output on marker.
+func NewShellJobMarkerStream(marker string) *ShellJobMarkerStream {
+ return &ShellJobMarkerStream{marker: marker}
+}
+
+// Feed returns output that is safe to forward and any bytes still held
+// because they might be the start of the marker.
+func (s *ShellJobMarkerStream) Feed(chunk []byte) (body, tail []byte) {
+ if s.found {
+ s.after = append(s.after, chunk...)
+ return nil, nil
+ }
+ buf := append(append([]byte{}, s.held...), chunk...)
+ s.held = nil
+ if i := indexOf(buf, s.marker); i >= 0 {
+ s.found = true
+ body = buf[:i]
+ s.after = append(s.after, buf[i+len(s.marker):]...)
+ return body, nil
+ }
+ // Hold back the longest suffix that could still become the marker.
+ limit := len(s.marker) - 1
+ if len(buf) < limit {
+ limit = len(buf)
+ }
+ for keep := limit; keep > 0; keep-- {
+ suffix := buf[len(buf)-keep:]
+ if strings.HasPrefix(s.marker, string(suffix)) {
+ s.held = append([]byte{}, suffix...)
+ return buf[:len(buf)-keep], s.held
+ }
+ }
+ return buf, nil
+}
+
+// Final returns the held body when no marker arrived, or the epilogue after it.
+func (s *ShellJobMarkerStream) Final() (body, epilogue string) {
+ if !s.found {
+ return string(s.held), ""
+ }
+ return "", string(s.after)
+}
+
+func indexOf(buf []byte, marker string) int {
+ return bytes.Index(buf, []byte(marker))
+}
+
+// ParseShellJobEpilogue reads the status JSON and the offset line that follow
+// a shell-job marker. ok is false when the epilogue is not that pair.
+func ParseShellJobEpilogue(epilogue string) (ShellJobStatus, ShellJobOffsets, bool) {
+ var lines []string
+ for _, line := range strings.Split(epilogue, "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+ lines = append(lines, line)
+ }
+ if len(lines) < 2 {
+ return ShellJobStatus{}, ShellJobOffsets{}, false
+ }
+ var status ShellJobStatus
+ statusLine := strings.TrimSpace(lines[len(lines)-2])
+ if err := json.Unmarshal([]byte(statusLine), &status); err != nil {
+ return ShellJobStatus{}, ShellJobOffsets{}, false
+ }
+ fields := strings.Fields(strings.TrimSpace(lines[len(lines)-1]))
+ if len(fields) != 2 {
+ return ShellJobStatus{}, ShellJobOffsets{}, false
+ }
+ so, err1 := strconv.ParseInt(fields[0], 10, 64)
+ se, err2 := strconv.ParseInt(fields[1], 10, 64)
+ if err1 != nil || err2 != nil {
+ return ShellJobStatus{}, ShellJobOffsets{}, false
+ }
+ return status, ShellJobOffsets{Stdout: so, Stderr: se}, true
+}
+
+// ShellJobCancelScript rechecks ownership in the exec that signals the process. The
+// directory is global on shared machines; a valid handle alone is insufficient.
+func ShellJobCancelScript(meta ShellJobMeta, force bool) string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "D=%s/%s; [ -f \"$D/status.json\" ] || exit 3;\n", ShellJobsRoot, meta.ID)
+ for _, field := range []struct {
+ name string
+ value any
+ }{
+ {"work_id", meta.ID},
+ {"tenant_id", meta.TenantID},
+ {"session_id", meta.SessionID},
+ {"sandbox_id", meta.SandboxID},
+ } {
+ raw, _ := json.Marshal(field.value)
+ // Meta is written atomically with json.Marshal, and each of these
+ // fields is followed by a comma. This also separates tenant 7 from 70.
+ pattern := fmt.Sprintf("%q:%s,", field.name, raw)
+ fmt.Fprintf(&b, "grep -F -q -- %s \"$D/meta.json\" || exit 3;\n", ShellQuote(pattern))
+ }
+ b.WriteString(`if grep -q '"state":"running"' "$D/status.json"; then
+ W=$(sed -n 's/.*"watcher_pid":\([0-9]*\).*/\1/p' "$D/status.json")
+`)
+ fmt.Fprintf(&b, ` if [ -n "$W" ] && grep -F -q -- %s "/proc/$W/cmdline" 2>/dev/null; then
+ kill -TERM "$W" 2>/dev/null || true
+ sleep 0.3
+`, ShellQuote(meta.ID))
+ if force {
+ b.WriteString(` if grep -q '"state":"running"' "$D/status.json"; then
+ C=$(sed -n 's/.*"pgid":\([0-9]*\).*/\1/p' "$D/status.json")
+ if [ -n "$C" ] && [ "$C" -gt 1 ]; then kill -KILL -"$C" 2>/dev/null || true; fi
+ sleep 0.3
+ fi
+`)
+ }
+ b.WriteString(` fi
+fi
+cat "$D/status.json"
+`)
+ return b.String()
+}
diff --git a/internal/sandbox/shell_job_scripts_test.go b/internal/sandbox/shell_job_scripts_test.go
new file mode 100644
index 000000000..a36451a35
--- /dev/null
+++ b/internal/sandbox/shell_job_scripts_test.go
@@ -0,0 +1,365 @@
+package sandbox
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func requireShellSyntax(t *testing.T, script string) {
+ t.Helper()
+ cmd := exec.Command("sh", "-n")
+ cmd.Stdin = strings.NewReader(script)
+ out, err := cmd.CombinedOutput()
+ require.NoError(t, err, "sh -n: %s", out)
+}
+
+func TestSubmitScriptEmbedsJobAndYield(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", Command: "TOKEN=1 make build", WorkDir: "/workspace"}
+ script, marker := ShellJobSubmitScript(job, "TOKEN=1 make build", 10000)
+ require.True(t, strings.HasPrefix(marker, shellJobMarkerPrefix))
+ require.Contains(t, script, "/var/lib/weknora/jobs/sj_0123")
+ require.Contains(t, script, marker) // 结尾行标记
+ require.Contains(t, script, "10000") // yield 轮询上限
+ require.Contains(t, script, "setsid") // wrapper 脱管启动
+ require.Contains(t, script, `find "$D/status.json" -mmin +1440`)
+ require.Contains(t, script, "exit 17")
+ require.Contains(t, script, `mkdir "$D" || {`)
+ require.NotContains(t, script, `D="$R/$J"; mkdir "$D" || exit 17;`)
+ require.NotContains(t, script, `mkdir -p "$D"`)
+ require.Contains(t, script, "iflag=skip_bytes,count_bytes")
+ require.NotContains(t, script, "bs=1")
+ require.Contains(t, strings.Repeat(script, 1), "meta.json")
+ // 提交脚本绝不能 wait wrapper
+ require.NotContains(t, script, "wait $wrapper")
+ requireShellSyntax(t, script)
+}
+
+func TestSubmitScriptPersistsMaskedMetaNotExecCommand(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", Command: "make build", WorkDir: "/workspace"}
+ script, _ := ShellJobSubmitScript(job, "SECRET=1 make build", 1000)
+ require.Contains(t, script, `"command":"make build"`)
+ require.Equal(t, 1, strings.Count(script, "SECRET=1 make build"))
+}
+
+func TestSubmitScriptDockerFollowsBackendNotSandboxID(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", SandboxID: "abc123def"}
+ script, _ := ShellJobSubmitScript(job, "true", 1)
+ require.NotContains(t, script, "weknora-sandbox-activity")
+ script, _ = ShellJobSubmitScriptWithBackend(job, "true", 1, true)
+ require.Contains(t, script, "weknora-sandbox-activity")
+ script, _ = ShellJobSubmitScriptWithBackend(job, "true", 1, false)
+ require.NotContains(t, script, "weknora-sandbox-activity")
+}
+
+func TestWrapperScriptRotatesAndTrapsTerm(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", Command: "sleep 100"}
+ script := shellJobWrapperScript(job, "sleep 100", true)
+ require.Contains(t, script, "trap") // TERM → 杀进程组 + cancelled
+ require.Contains(t, script, "kill -") // 杀命令进程组
+ require.Contains(t, script, "8192") // .head 保留 8 KiB
+ require.Contains(t, script, "16777216") // 16 MiB 轮转阈值
+ require.Contains(t, script, "weknora-sandbox-activity") // docker touch
+ require.Contains(t, script, "touch_pid")
+ require.Contains(t, script, ">>") // 追加写,轮转截断无空洞
+ // 终态只写一次:构建残留的 printf … > /dev/null 不能留下。
+ require.NotContains(t, script, "> /dev/null")
+ require.Contains(t, script, "cancelled")
+ require.Contains(t, script, `%Y-%m-%dT%H:%M:%S.%3NZ`)
+ plain := shellJobWrapperScript(job, "sleep 100", false)
+ require.NotContains(t, plain, "weknora-sandbox-activity")
+ require.Contains(t, plain, "TOUCH=0")
+ require.NotContains(t, plain, "> /dev/null")
+ requireShellSyntax(t, script)
+ requireShellSyntax(t, plain)
+}
+
+func TestMarkerFilterHoldsPartialPrefixAcrossChunks(t *testing.T) {
+ marker := shellJobMarkerPrefix + "_abc123__"
+ s := NewShellJobMarkerStream(marker)
+ body, tail := s.Feed([]byte("out1"))
+ require.Equal(t, "out1", string(body))
+ require.Empty(t, tail)
+ // 分块恰好把标记劈开
+ half := len(marker) / 2
+ body, tail = s.Feed([]byte(marker[:half]))
+ require.Empty(t, body)
+ require.Equal(t, marker[:half], string(tail))
+ body, _ = s.Feed([]byte(marker[half:] + `{"state":"succeeded"}` + "\n" + "0 0"))
+ require.Empty(t, body)
+ // Final returns strings; body is already the []byte from Feed, so := cannot reuse it.
+ finalBody, epilogue := s.Final()
+ require.Empty(t, finalBody)
+ require.Contains(t, epilogue, `"state":"succeeded"`)
+}
+
+func TestMarkerFilterHoldsOnlyPrefixLengthTail(t *testing.T) {
+ s := NewShellJobMarkerStream(shellJobMarkerPrefix + "_xyz__")
+ body, tail := s.Feed([]byte("keep__WEK"))
+ require.Equal(t, "keep", string(body))
+ require.Equal(t, "__WEK", string(tail))
+ body, tail = s.Feed([]byte("NORA_SJ"))
+ require.Equal(t, "__WEKNORA_SJ", string(tail), "非标记前缀最终全部放行")
+ require.Empty(t, body)
+}
+
+func TestParseShellJobEpilogue(t *testing.T) {
+ status, offsets, ok := ParseShellJobEpilogue(
+ `{"state":"running","watcher_pid":7,"pgid":8,"started_at":"2026-10-16T00:00:00Z"}` + "\n123 45")
+ require.True(t, ok)
+ require.Equal(t, 7, status.WatcherPID)
+ require.Equal(t, int64(123), offsets.Stdout)
+ require.Equal(t, int64(45), offsets.Stderr)
+ _, _, ok = ParseShellJobEpilogue("garbage")
+ require.False(t, ok)
+ // A blank line between the status record and the offsets is ignored.
+ status, offsets, ok = ParseShellJobEpilogue(
+ "\n" + `{"state":"running","watcher_pid":7,"pgid":8,"started_at":"2026-10-16T00:00:00Z"}` + "\n\n123 45\n")
+ require.True(t, ok)
+ require.Equal(t, int64(123), offsets.Stdout)
+ require.Equal(t, int64(45), offsets.Stderr)
+}
+
+func TestShellJobBatchReadScriptReadsEachJob(t *testing.T) {
+ script := ShellJobBatchReadScript("docker-box", []string{
+ "sj_0123",
+ "sj_abef",
+ "sj_bad",
+ "$(reboot)",
+ })
+ require.Contains(t, script, ShellJobsRoot+"/sj_0123")
+ require.Contains(t, script, ShellJobsRoot+"/sj_abef")
+ require.NotContains(t, script, "sj_bad")
+ require.NotContains(t, script, "reboot")
+ require.Contains(t, script, "cat")
+ require.Contains(t, script, "meta.json")
+ require.Contains(t, script, "status.json")
+ require.Contains(t, script, "wc -c")
+ require.Contains(t, script, "stdout")
+ require.Contains(t, script, "stderr")
+ require.Contains(t, script, "/proc/")
+ require.Contains(t, script, "cmdline")
+ require.Contains(t, script, "watcher_alive")
+ require.Contains(t, script, `"meta"`)
+ require.Contains(t, script, `"status"`)
+ require.Contains(t, script, "total_bytes")
+ require.Contains(t, script, "iflag=skip_bytes,count_bytes")
+ require.NotContains(t, script, "bs=1")
+ require.Contains(t, script, "sj_0123")
+ require.Contains(t, script, "sj_abef")
+ requireShellSyntax(t, script)
+}
+
+func TestShellJobBatchReadScriptEmitsDecodableJSON(t *testing.T) {
+ root := t.TempDir()
+ id := "sj_0123"
+ dir := filepath.Join(root, id)
+ require.NoError(t, os.MkdirAll(dir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "meta.json"),
+ []byte(`{"work_id":"sj_0123","command":"make"}`), 0o644))
+ status := "{\"state\":\"running\",\"watcher_pid\":999999,\"pgid\":8," +
+ "\"started_at\":\"2026-10-16T00:00:00Z\"}\n"
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "status.json"), []byte(status), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout"), []byte("hello"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stderr"), []byte("xyz"), 0o644))
+
+ script := strings.ReplaceAll(
+ ShellJobBatchReadScript("docker-box", []string{id, "sj_abef"}),
+ ShellJobsRoot, root)
+ cmd := exec.Command("sh")
+ cmd.Stdin = strings.NewReader(script)
+ out, err := cmd.CombinedOutput()
+ require.NoError(t, err, string(out))
+ snaps := ParseShellJobBatch(string(out))
+ require.Len(t, snaps, 2)
+ require.True(t, snaps[1].Missing)
+ require.Equal(t, id, snaps[0].Meta.ID)
+ require.Equal(t, "make", snaps[0].Meta.Command)
+ require.Equal(t, "running", snaps[0].Status.State)
+ require.Equal(t, 999999, snaps[0].Status.WatcherPID)
+ require.Equal(t, int64(5), snaps[0].Stdout.TotalBytes)
+ require.Equal(t, int64(3), snaps[0].Stderr.TotalBytes)
+ require.Equal(t, "hello", snaps[0].Stdout.Text)
+ require.Equal(t, "xyz", snaps[0].Stderr.Text)
+ require.False(t, snaps[0].WatcherAlive)
+ require.False(t, snaps[0].Missing)
+}
+
+func TestShellJobBatchReadScriptReturnsHeadAndTail(t *testing.T) {
+ root := t.TempDir()
+ id := "sj_0123"
+ dir := filepath.Join(root, id)
+ require.NoError(t, os.MkdirAll(dir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "status.json"),
+ []byte(`{"state":"running","watcher_pid":1,"started_at":"2026-10-16T00:00:00Z"}`), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "meta.json"),
+ []byte(`{"work_id":"sj_0123"}`), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout"), []byte("TAIL"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.1"), []byte("0123456789"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.head"), []byte("HEAD8K"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.rotated"), []byte("10\n"), 0o644))
+
+ script := strings.ReplaceAll(ShellJobBatchReadScript("sbx", []string{id}), ShellJobsRoot, root)
+ cmd := exec.Command("sh")
+ cmd.Stdin = strings.NewReader(script)
+ out, err := cmd.CombinedOutput()
+ require.NoError(t, err, string(out))
+ snaps := ParseShellJobBatch(string(out))
+ require.Len(t, snaps, 1)
+ require.Equal(t, "HEAD8K6789TAIL", snaps[0].Stdout.Text)
+ require.Equal(t, int64(14), snaps[0].Stdout.TotalBytes)
+ require.Equal(t, int64(0), snaps[0].Stdout.OmittedBytes)
+}
+
+func TestSubmitScriptCapsForwardingForTheWholeWindow(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", Command: "true"}
+ script, _ := ShellJobSubmitScript(job, "true", 10000)
+ require.NotContains(t, script, "-le 1048576")
+ budget := strings.Index(script, "ROOM=$(( 1048576 - BUD ))")
+ clamp := strings.Index(script, `[ "$TAKE" -gt "$ROOM" ] && TAKE=$ROOM`)
+ add := strings.Index(script, "BUD=$(( BUD + TAKE ))")
+ advance := strings.Index(script, "CUR=$N")
+ require.Greater(t, budget, 0)
+ require.Less(t, budget, clamp)
+ require.Less(t, clamp, add)
+ require.Less(t, add, advance)
+ // The allowance if closes before the cursor moves to the observed end,
+ // so a capped poll still advances and a later poll does not reread it.
+ require.Contains(t, script[add:advance], "fi")
+ require.Contains(t, script, `cat "$D/stdout.rotated"`)
+ require.Contains(t, script, `cat "$D/stderr.rotated"`)
+ require.Contains(t, script, "SO=$(( RO + LN ))")
+ require.Contains(t, script, "SE=$(( RE + LE ))")
+ requireShellSyntax(t, script)
+}
+
+func TestWrapperScriptParentWaitsAndRotatesAfterReap(t *testing.T) {
+ job := ShellJobMeta{ID: "sj_0123", Command: "sleep 100"}
+ script := shellJobWrapperScript(job, "sleep 100", false)
+ trapAt := strings.Index(script, "trap '")
+ cmdAt := strings.Index(script, "setsid /bin/bash")
+ loopAt := strings.Index(script, `while kill -0 "$C"`)
+ rotAt := strings.Index(script, "rot &")
+ waitAt := strings.Index(script, `wait "$C"`)
+ require.GreaterOrEqual(t, trapAt, 0)
+ require.Less(t, trapAt, cmdAt)
+ require.Greater(t, loopAt, cmdAt)
+ require.Less(t, loopAt, rotAt)
+ require.Less(t, rotAt, waitAt)
+ require.NotContains(t, script, "done; wait")
+ require.NotContains(t, script, "done;\nwait")
+ after := script[waitAt:]
+ require.Contains(t, after, "r stdout")
+ require.Contains(t, after, "r stderr")
+ require.NotContains(t, script, "> /dev/null")
+ require.Contains(t, script, "status.json.tmp")
+ requireShellSyntax(t, script)
+}
+
+func TestShellJobBatchReadScriptAddsRotatedToLiveSegment(t *testing.T) {
+ id := "sj_0123"
+ script := ShellJobBatchReadScript("docker-box", []string{id})
+ require.Contains(t, script, "wc -c")
+ require.Contains(t, script, `cat "$ROTF"`)
+ require.Contains(t, script, "TOTAL=$(( RO + LN ))")
+ requireShellSyntax(t, script)
+
+ root := t.TempDir()
+ dir := filepath.Join(root, id)
+ require.NoError(t, os.MkdirAll(dir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "meta.json"),
+ []byte(`{"work_id":"sj_0123","command":"make"}`), 0o644))
+ status := "{\"state\":\"running\",\"watcher_pid\":999999,\"pgid\":8," +
+ "\"started_at\":\"2026-10-16T00:00:00Z\"}\n"
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "status.json"), []byte(status), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout"), []byte("hello"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stderr"), []byte("xyz"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stdout.rotated"), []byte("100\n"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "stderr.rotated"), []byte("7\n"), 0o644))
+
+ script = strings.ReplaceAll(script, ShellJobsRoot, root)
+ cmd := exec.Command("sh")
+ cmd.Stdin = strings.NewReader(script)
+ out, err := cmd.CombinedOutput()
+ require.NoError(t, err, string(out))
+ snaps := ParseShellJobBatch(string(out))
+ require.Len(t, snaps, 1)
+ require.Equal(t, int64(105), snaps[0].Stdout.TotalBytes)
+ require.Equal(t, int64(10), snaps[0].Stderr.TotalBytes)
+}
+
+func TestKillOrphansScriptSkipsPidOne(t *testing.T) {
+ script := ShellJobKillOrphansScript(99, 77)
+ require.Contains(t, script, "PGID=99")
+ require.Contains(t, script, "TOUCH=77")
+ require.Contains(t, script, "kill -TERM -$PGID")
+ require.Contains(t, script, "kill -KILL -$PGID")
+ require.Contains(t, script, "weknora-sandbox-activity")
+ require.Contains(t, script, `-gt 1`)
+ require.Contains(t, script, "\ntrue\n")
+ idle := ShellJobKillOrphansScript(1, 0)
+ require.Contains(t, idle, "PGID=1")
+ require.Contains(t, idle, "TOUCH=0")
+ require.NotContains(t, idle, "kill -TERM -1")
+ require.NotContains(t, idle, "kill -KILL -1")
+ requireShellSyntax(t, script)
+ requireShellSyntax(t, idle)
+}
+
+func TestParseShellJobBatchDecodesLinesWithoutMissingPlaceholders(t *testing.T) {
+ running := `{"meta":{"work_id":"sj_0123","command":"make"},` +
+ `"status":{"state":"running","watcher_pid":7,"pgid":8,"started_at":"2026-10-16T00:00:00Z"},` +
+ `"stdout":{"total_bytes":123},"stderr":{"total_bytes":45},"watcher_alive":true}`
+ done := `{"meta":{"work_id":"sj_abef"},` +
+ `"status":{"state":"succeeded","watcher_pid":9,"pgid":10,"exit_code":0,` +
+ `"started_at":"2026-10-16T00:00:00Z"},` +
+ `"stdout":{"total_bytes":1},"stderr":{"total_bytes":2},"watcher_alive":false}`
+ stdout := strings.Join([]string{running, done, "", "not-json"}, "\n")
+ snaps := ParseShellJobBatch(stdout)
+ require.Len(t, snaps, 2)
+ require.Equal(t, "sj_0123", snaps[0].Meta.ID)
+ require.Equal(t, 7, snaps[0].Status.WatcherPID)
+ require.Equal(t, int64(123), snaps[0].Stdout.TotalBytes)
+ require.Equal(t, int64(45), snaps[0].Stderr.TotalBytes)
+ require.True(t, snaps[0].WatcherAlive)
+ require.False(t, snaps[0].Missing)
+ require.Equal(t, "sj_abef", snaps[1].Meta.ID)
+ require.False(t, snaps[1].WatcherAlive)
+ require.False(t, snaps[1].Missing)
+ require.Empty(t, ParseShellJobBatch(""))
+ require.Empty(t, ParseShellJobBatch("\n\n"))
+}
+
+func TestShellJobCursorCountsRawBytesBeforeSanitizingUTF8(t *testing.T) {
+ // /w== is one invalid UTF-8 byte, followed by a valid 'x' in /3g=.
+ snaps := ParseShellJobBatch(`{"meta":{"work_id":"sj_0123"},"status":{"state":"running"},` +
+ `"stdout":{"slices":true,"total_bytes":1000,"from":100,"omitted_bytes":99,"body_b64":"/3g="}}`)
+ require.Len(t, snaps, 1)
+ require.Equal(t, "x", snaps[0].Stdout.Text)
+ require.NotNil(t, snaps[0].Stdout.NextBytes)
+ require.Equal(t, int64(102), *snaps[0].Stdout.NextBytes)
+}
+
+func TestShellJobLocalOutputBudgetIncludesBothStreams(t *testing.T) {
+ root := t.TempDir()
+ dir := filepath.Join(root, "sj_0123")
+ require.NoError(t, os.Mkdir(dir, 0o700))
+ require.NoError(t, WriteShellJobJSON(filepath.Join(dir, "meta.json"), ShellJobMeta{ID: "sj_0123"}))
+ require.NoError(t, WriteShellJobJSON(filepath.Join(dir, "status.json"), ShellJobStatus{State: "succeeded"}))
+ for _, name := range []string{"stdout", "stderr"} {
+ require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(strings.Repeat(name, 5000)), 0o600))
+ }
+ for _, budget := range []int{1, 2, 3, 2048} {
+ for _, from := range []*ShellJobOffsets{nil, {Stdout: 1, Stderr: 1}} {
+ snap, err := ReadShellJob(root, ShellJobRead{ID: "sj_0123", OutputBudget: budget, From: from})
+ require.NoError(t, err)
+ require.LessOrEqual(t, len(snap.Stdout.Text)+len(snap.Stderr.Text), budget)
+ require.True(t, snap.Stdout.Truncated)
+ require.True(t, snap.Stderr.Truncated)
+ }
+ }
+}
diff --git a/internal/sandbox/shell_job_test.go b/internal/sandbox/shell_job_test.go
new file mode 100644
index 000000000..e02da2ece
--- /dev/null
+++ b/internal/sandbox/shell_job_test.go
@@ -0,0 +1,147 @@
+package sandbox
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func writeJobDir(t *testing.T, files map[string]string) string {
+ t.Helper()
+ root := t.TempDir()
+ for name, body := range files {
+ path := filepath.Join(root, "sj_0123", name)
+ require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+ require.NoError(t, os.WriteFile(path, []byte(body), 0o644))
+ }
+ return root
+}
+
+func TestValidShellJobID(t *testing.T) {
+ require.True(t, ValidShellJobID("sj_0123"))
+ id, err := NewShellJobID()
+ require.NoError(t, err)
+ require.True(t, ValidShellJobID(id))
+ require.False(t, ValidShellJobID("sj_012"))
+ require.False(t, ValidShellJobID("sj_01234"))
+ require.False(t, ValidShellJobID("sj_0123456789ab"))
+ require.False(t, ValidShellJobID("sa_0123"))
+ require.False(t, ValidShellJobID("sj_012Z"))
+ require.False(t, ValidShellJobID(""))
+}
+
+// 结尾 = 当前段(不足补 .1 末尾);开头 = .head(没有则当前段开头);
+// 总字节 = .rotated + 当前段大小(.1 存 [.rotated-size(.1), .rotated))。
+func TestReadShellJobHeadAndTailAcrossRotation(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "TAIL",
+ "stdout.1": "0123456789",
+ "stdout.head": "HEAD8K",
+ "stdout.rotated": "10",
+ "status.json": `{"state":"running","watcher_pid":9,"pgid":9,"started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ snap, err := ReadShellJob(root, ShellJobRead{ID: "sj_0123"})
+ require.NoError(t, err)
+ // .head 占 [0, 6),.1 占 [0, 10),当前段占 [10, 14)。开头用 .head,
+ // 与 .1 重叠的前缀不重复,结尾补上 .1 的剩余部分和当前段。
+ require.Equal(t, "HEAD8K6789TAIL", snap.Stdout.Text)
+ require.Equal(t, int64(0), snap.Stdout.OmittedBytes)
+ require.Equal(t, int64(14), snap.Stdout.TotalBytes)
+ require.Equal(t, int64(0), snap.Stdout.FromBytes)
+ require.False(t, snap.Stdout.Truncated)
+ require.True(t, snap.WatcherAlive)
+}
+
+func TestReadShellJobTailPadWithoutHeadFile(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "TAIL",
+ "stdout.1": "0123456789",
+ "stdout.rotated": "10",
+ "status.json": `{"state":"running","watcher_pid":9,"started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ snap, err := ReadShellJob(root, ShellJobRead{ID: "sj_0123"})
+ require.NoError(t, err)
+ require.Equal(t, "0123456789TAIL", snap.Stdout.Text)
+ require.Equal(t, int64(0), snap.Stdout.FromBytes)
+ require.Equal(t, int64(0), snap.Stdout.OmittedBytes)
+ require.Equal(t, int64(14), snap.Stdout.TotalBytes)
+}
+
+func TestReadShellJobRotatedCountAllowsTrailingNewline(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "TAIL",
+ "stdout.rotated": "10\n",
+ "status.json": `{"state":"running","watcher_pid":9,"started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ snap, err := ReadShellJob(root, ShellJobRead{ID: "sj_0123"})
+ require.NoError(t, err)
+ require.Equal(t, int64(14), snap.Stdout.TotalBytes)
+}
+
+func TestReadShellJobShortStreamIsNotDoubled(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "hello",
+ "status.json": `{"state":"running","watcher_pid":9,"started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ snap, err := ReadShellJob(root, ShellJobRead{ID: "sj_0123"})
+ require.NoError(t, err)
+ require.Equal(t, "hello", snap.Stdout.Text)
+ require.Equal(t, int64(5), snap.Stdout.TotalBytes)
+ require.False(t, snap.Stdout.Truncated)
+}
+
+func TestReadShellJobIncrementalAndOmitted(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "CDEFGH",
+ "stdout.1": "AB",
+ "stdout.rotated": "2",
+ "status.json": `{"state":"running","started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ // 读取位置 4 在当前段 [2, 8) 内,只返回这个位置之后的字节。
+ snap, err := ReadShellJob(root, ShellJobRead{
+ ID: "sj_0123",
+ From: &ShellJobOffsets{Stdout: 4, Stderr: 0},
+ })
+ require.NoError(t, err)
+ require.Equal(t, "EFGH", snap.Stdout.Text)
+ require.Equal(t, int64(0), snap.Stdout.OmittedBytes)
+ require.Equal(t, int64(4), snap.Stdout.FromBytes)
+ require.Equal(t, int64(8), snap.Stdout.TotalBytes)
+ require.False(t, snap.Stdout.Truncated)
+}
+
+func TestReadShellJobCursorInDiscardedRangeStartsAtEarliestByte(t *testing.T) {
+ root := writeJobDir(t, map[string]string{
+ "stdout": "CDEFGH",
+ "stdout.1": "XY",
+ "stdout.rotated": "5",
+ "status.json": `{"state":"running","started_at":"2026-10-16T00:00:00Z"}`,
+ })
+ // .1 占 [3, 5),当前段占 [5, 11)。位置 1 已经不在磁盘上,从 3 开始读。
+ snap, err := ReadShellJob(root, ShellJobRead{
+ ID: "sj_0123",
+ From: &ShellJobOffsets{Stdout: 1},
+ })
+ require.NoError(t, err)
+ require.Equal(t, "XYCDEFGH", snap.Stdout.Text)
+ require.Equal(t, int64(2), snap.Stdout.OmittedBytes)
+ require.Equal(t, int64(3), snap.Stdout.FromBytes)
+ require.Equal(t, int64(11), snap.Stdout.TotalBytes)
+}
+
+func TestReadShellJobMissingDirIsMissingNotError(t *testing.T) {
+ snap, err := ReadShellJob(t.TempDir(), ShellJobRead{ID: "sj_0123"})
+ require.NoError(t, err)
+ require.True(t, snap.Missing)
+}
+
+func TestWriteShellJobJSONIsAtomicAndRFC3339(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "status.json")
+ require.NoError(t, WriteShellJobJSON(path, ShellJobStatus{State: "running"}))
+ require.NoFileExists(t, filepath.Dir(path)+"/.status.json-*")
+ raw, err := os.ReadFile(path)
+ require.NoError(t, err)
+ require.Contains(t, string(raw), `"state":"running"`)
+}