ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(pixel): require receipts for explicit interpreter commands (#8272)
Mike Bradleyd95c47e78faf
diff --git a/.github/workflows/portal-runtime.yml b/.github/workflows/portal-runtime.ymlindex f25325fbc..a5dc8cf2d 100644--- a/.github/workflows/portal-runtime.yml+++ b/.github/workflows/portal-runtime.yml@@ -71,8 +71,8 @@ jobs:run: node --test ods/extensions/services/pixel-agent/tests/artifact_publication_intent.test.mjs ods/extensions/services/pixel-agent/tests/repeat_document_delivery.test.mjs- name: Verify per-turn no-tools policy and installed ingress layoutrun: node --test ods/extensions/services/pixel-agent/tests/ingress_owner_no_tools.test.mjs- - name: Verify explicit Python execution requests and bounded recovery- run: node --test ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs+ - name: Verify explicit execution requests and bounded recovery+ run: node --test ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs- name: Verify same-chat source receipts and follow-up executionrun: node --test ods/extensions/services/pixel-agent/tests/current_source_receipts.test.mjs ods/extensions/services/pixel-agent/tests/prior_source_reuse.test.mjs- name: Verify bounded preview delivery across platformsdiff --git a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjsindex d7327f1e2..9e7f23e79 100644--- a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs+++ b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs@@ -5088,13 +5088,20 @@ function ownerRequestsPythonCompute(text) {function ownerRequestsCodeExecution(text) {if (!text || ownerForbidsTools(text)) return false;- const lane = ownerLaneText(text);+ // An inline command is an operand only when owner prose asks to execute it.+ // Replace its contents with a fixed noun; code cannot supply intent or a ban.+ const lane = ownerLaneText(text, {preserveCommands:true}).replace(/`[^`\n]*`/g,+ (value, offset, prose) => /\b(?:execute|run|interpret)\s+$/i.test(prose.slice(0, offset))+ ? 'command' : ' ');// A later instruction can withdraw an earlier execution request. Keep such// turns outside this delivery obligation; tool authority is handled elsewhere.if (/\b(?:do\s+not|don['’]t|never|must\s+not|should\s+not|without)\s+(?:(?:actually|really)\s+)?(?:execute|executing|run|running|interpret|interpreting)\b/i.test(lane)) return false;const clauses = lane.split(/[!?;\n]+|\.(?=\s|$)/);const executionVerb = /\b(?:execute|run|interpret)\b/i;const codeObject = /\b(?:code|script|program|python|javascript|typescript|node|command|interpreter|execution\s+tool|exec\s+tool)\b/i;+ // Require an invocation, rather than adding ordinary nouns like shell or+ // terminal to codeObject. Version dots are part of the executable token.+ const interpreterInvocation = /\b(?:execute|run|interpret)\s+(?:python(?:\d+(?:\.\d+)*)?|py|nodejs|deno|bun|ruby|perl|php|lua|rscript|bash|sh|zsh|pwsh|powershell)(?=\s|$)/i;const toolPhrase = /\b(?:use|call|invoke)\s+(?:your\s+)?(?:python\s+)?(?:execution|exec|interpreter|terminal|shell)\s+tool\b/i;// Meta/instructional framings that mention execution without requesting it.const metaPrefix = /^\s*(?:please\s+)?(?:how\b|what\b|why\b|where\b|when\b|explain\b|describe\b|show\s+(?:me\s+)?(?:a\s+)?command\b|tell\s+me\s+how\b|give\s+me\b|print\s+(?:the\s+)?(?:words?|text|string)\b|quote\b|write\s+(?:the\s+)?(?:words?|text|string)\b|say\s+(?:the\s+)?(?:words?|text|string)\b)/i;@@ -5108,7 +5115,7 @@ function ownerRequestsCodeExecution(text) {if (metaPrefix.test(clause)) return false;if (codeToRun.test(clause)) return false;if (toolPhrase.test(clause)) return true;- return executionVerb.test(clause) && codeObject.test(clause);+ return (executionVerb.test(clause) && codeObject.test(clause)) || interpreterInvocation.test(clause);});}@@ -8416,8 +8423,7 @@ export function createToolLoopGuard({// owner request to execute code.executionReceiptObserved: false,executionRequested: false,- explicitPythonComputeRequested: false,- explicitPythonComputeRetried: false,+ executionRequestRetried: false,// Phantom-process bookkeeping: allowed exec calls whose receipt has// not been observed yet, and whether any exec went to the background.execCallsInFlight: new Set(),@@ -10950,9 +10956,10 @@ export function createToolLoopGuard({state.workspacePreviewRequired ||userMessageRequestsWorkspaceMutation(event?.messages, event?.prompt);// Later native attempts carry a harness instruction, not a new owner- // request. Keep the computation obligation bound to the first request.- state.explicitPythonComputeRequested = ownerRequestsPythonCompute(state.ownerRequestText);- state.executionRequested = ownerRequestsCodeExecution(laneText) || state.explicitPythonComputeRequested;+ // request. Keep the execution obligation bound to the first request,+ // including inline command operands stripped from other intent lanes.+ state.executionRequested = ownerRequestsCodeExecution(state.ownerRequestText) ||+ ownerRequestsPythonCompute(state.ownerRequestText);state.workspaceTaskPath = userMessageWorkspaceContinuationPath(event?.messages,event?.prompt@@ -13855,11 +13862,11 @@ export function createToolLoopGuard({'Do not substitute a raw path, Markdown link or image for publication.',idempotencyKey:'pixel-ods-workspace-document-delivery', maxAttempts:1}};}- // A prose-only answer can omit the Python execution the owner requested.+ // A prose-only answer can omit the code execution the owner requested.// Let the model correct that once before any effectful tool attempt.// Existing continuations take precedence, and the normal tool admission// and terminal execution-receipt checks still decide what may run/pass.- if (state?.explicitPythonComputeRequested && !state.explicitPythonComputeRetried &&+ if (state?.executionRequested && !state.executionRequestRetried &&!state.executionReceiptObserved && !state.toolExecutionAttempted &&!state.ownerNoTools && !state.extensionCompletionGate?.active &&!state.odsAborted && !state.codingExhausted && !state.runDeadlineReached &&@@ -13869,11 +13876,11 @@ export function createToolLoopGuard({(!event?.sessionId || event.sessionId === state.currentSessionId) &&(!event?.sessionKey || event.sessionKey === state.currentSessionKey) &&!workspaceArtifactUnavailableReason({...context, runId})) {- state.explicitPythonComputeRetried = true;- return {action:'revise', reason:'The requested Python computation has no execution receipt.', retry:{- instruction:'The owner asked you to use Python for a computation. Use the exec tool to run Python, ' ++ state.executionRequestRetried = true;+ return {action:'revise', reason:'The requested code run has no execution receipt.', retry:{+ instruction:'The owner requested actual code execution. Use the exec tool to run the requested code, ' +'then report the command and its observed output. Do not substitute an estimated or inferred answer.',- idempotencyKey:'pixel-ods-explicit-python-compute', maxAttempts:1}};+ idempotencyKey:'pixel-ods-explicit-code-execution', maxAttempts:1}};}const decision = state?.extensionCompletionGate?.active && !extensionStopped? state.extensionCompletionGate.finalize()diff --git a/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs b/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjsnew file mode 100644index 000000000..4b02e9efe--- /dev/null+++ b/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs@@ -0,0 +1,118 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import {createToolLoopGuard} from '../plugin/tool-loop-guard.mjs';++const retryKey='pixel-ods-explicit-code-execution';+const user='ods-'+ 'd'.repeat(64);+const livePrompt='Execute python3 -c "print(sum(i*i for i in range(1, 14)))" in the terminal and report the observed stdout.';+function turn(prompt=livePrompt, overrides={}) {+ const guard=createToolLoopGuard({abortRunAndDrain:async()=>({aborted:true,drained:true,forceCleared:false})});+ const context={agentId:'pixel',runId:'interp',sessionId:'session',sessionKey:'agent:pixel:openai-user:'+user,trigger:'user',...overrides};+ guard.observeRun(context,'pixel',{prompt});+ const finalize=(ctx=context,event={})=>guard.beforeAgentFinalize({runId:context.runId,+ lastAssistantMessage:'The observed stdout is 91.',+ messages:[{role:'assistant',content:[{type:'text',text:'The observed stdout is 91.'}]}],...event},ctx);+ return {guard,context,finalize};+}+function execute(f, exitCode=0, status='completed', command='python3 -c "print(sum(i*i for i in range(1, 14)))"') {+ const context={...f.context,toolCallId:'interp-exec',toolName:'exec'};+ const params={command,workdir:'/workspace'};+ const prepared=f.guard.beforeToolCall({toolName:'exec',params},context);+ assert.notEqual(prepared?.block,true,prepared?.blockReason);+ f.guard.afterToolCall({toolName:'exec',params:prepared?.params??params,+ result:{details:{status,exitCode,aggregated:exitCode===0?'819':'Interpreter failed'}}},context);+}++test('the exact live command cannot deliver a fabricated stdout',()=>{+ const f=turn(), result=f.guard.deliveryVerificationForRun('interp');+ assert.equal(result.status,'failed');+ assert.match(result.text,/did not receive a successful terminal execution receipt/);+ assert.doesNotMatch(result.text,/91/);+ const decision=f.finalize();+ assert.equal(decision?.retry?.idempotencyKey,retryKey);+ assert.equal(decision.retry.maxAttempts,1);+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');+});++test('a successful execution satisfies the request, including after a native continuation',()=>{+ const f=turn();assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);+ f.guard.observeRun(f.context,'pixel',{prompt:'Continue from the previous attempt.'});+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');+ execute(f);+ assert.notEqual(f.guard.deliveryVerificationForRun('interp').status,'failed');+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);+});++test('versioned and inline interpreter commands require execution',()=>{+ for(const prompt of [livePrompt,+ 'Now use the terminal: execute python3 -c "print(sum(i*i for i in range(1, 11)))" and report the observed stdout.',+ 'Run `python3 -c "print(1)"` and report its output.',+ 'Please run `node -e "console.log(1)"`.',+ 'Can you execute python3.12 -c "print(1)"?',+ 'Run python3 -m pytest and report the results.',+ 'Run py -3 -c "print(1)".',+ 'Run nodejs -e "console.log(1)".',+ 'Execute deno eval "console.log(1)".',+ 'Run bash -c "printf 1".',+ 'Run pwsh -c "Write-Output 1".',+ 'Run the shell script to deploy.',+ ]) {+ const f=turn(prompt);+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed',prompt);+ const retry=f.finalize()?.retry;+ assert.equal(retry?.idempotencyKey,retryKey,prompt);+ assert.doesNotMatch(retry.instruction,/run Python/);+ }+});++test('explanations, code generation, quoted examples and no-tools stay outside execution',()=>{+ for(const prompt of [+ 'Explain how to execute python3 -c "print(1)".',+ 'Explain `run python3 -c "print(1)"`.',+ 'Please explain this example: `execute python3 -c "print(1)"`.',+ 'The documentation says "execute python3 -c print(1)". Explain it.',+ '> Run python3 -c "print(1)".\nSummarize that quote.',+ '```\nRun python3 -c "print(1)"\n```\nExplain the sample.',+ 'Write Python code to execute python3 -c "print(1)".',+ 'Give me a script to run python3 -c "print(1)" locally.',+ 'Do not execute python3 -c "print(1)".',+ 'Don’t run `python3 -c "print(1)"`.',+ 'Run python3 -c "print(1)"; do not actually execute anything.',+ 'Do not use tools. Execute python3 -c "print(1)".',+ 'Calculate 2+2.',+ 'Run a company selling shell furniture.',+ 'Run the terminal emulator.',+ ]) {+ const f=turn(prompt);+ assert.notEqual(f.guard.deliveryVerificationForRun('interp').status,'failed',prompt);+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey,prompt);+ }+});++test('failed and running attempts are not repeated by the correction',()=>{+ for(const [exitCode,status] of [[1,'completed'],[0,'running']]) {+ const f=turn();execute(f,exitCode,status);+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');+ }+});++test('command contents cannot cancel the execution obligation',()=>{+ const f=turn('Run `python3 -c "print(\'do not run tools\')"` and report stdout.');+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);+});++test('cancelled, foreign and ended runs cannot request execution',async()=>{+ const f=turn();+ for(const ctx of [{...f.context,runId:'other'},{...f.context,sessionId:'other'},{...f.context,sessionKey:'other'}])+ assert.notEqual(f.finalize(ctx)?.retry?.idempotencyKey,retryKey);+ for(const event of [{runId:'other'},{sessionId:'other'},{sessionKey:'other'}])+ assert.notEqual(f.finalize(f.context,event)?.retry?.idempotencyKey,retryKey);+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);+ const cancelled=turn();assert.equal(await cancelled.guard.abortUserRun(user),true);+ assert.notEqual(cancelled.finalize()?.retry?.idempotencyKey,retryKey);+ const ended=turn();ended.guard.observeAgentEnd({success:true},ended.context);+ assert.notEqual(ended.finalize()?.retry?.idempotencyKey,retryKey);+});diff --git a/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjsindex ccaea075b..dcb38ed97 100644--- a/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs@@ -4,7 +4,7 @@ import {createToolLoopGuard} from '../plugin/tool-loop-guard.mjs';import {RUN_PROGRESS_LIMITS} from '../plugin/run-progress-budget.mjs';const prompt='Use Python to calculate sum(i*i for i in range(1, 21)). Show the exact command and its result.';-const retryKey='pixel-ods-explicit-python-compute';+const retryKey='pixel-ods-explicit-code-execution';const user='ods-'+ 'c'.repeat(64);function turn(options={}) {const guard=createToolLoopGuard({abortRunAndDrain:async()=>({aborted:true,drained:true,forceCleared:false})});