Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(pixel): require receipts for explicit interpreter commands (#8272)

Mike Bradleyd95c47e78faf
diff --git a/.github/workflows/portal-runtime.yml b/.github/workflows/portal-runtime.yml
index f25325fbc..a5dc8cf2d 100644
--- a/.github/workflows/portal-runtime.yml
+++ b/.github/workflows/portal-runtime.yml
@@ -71,8 +71,8 @@ jobs:
run: node --test ods/extensions/services/pixel-agent/tests/artifact_publication_intent.test.mjs ods/extensions/services/pixel-agent/tests/repeat_document_delivery.test.mjs
- name: Verify per-turn no-tools policy and installed ingress layout
run: node --test ods/extensions/services/pixel-agent/tests/ingress_owner_no_tools.test.mjs
- - name: Verify explicit Python execution requests and bounded recovery
- run: node --test ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
+ - name: Verify explicit execution requests and bounded recovery
+ run: node --test ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs
- name: Verify same-chat source receipts and follow-up execution
run: node --test ods/extensions/services/pixel-agent/tests/current_source_receipts.test.mjs ods/extensions/services/pixel-agent/tests/prior_source_reuse.test.mjs
- name: Verify bounded preview delivery across platforms
diff --git a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
index d7327f1e2..9e7f23e79 100644
--- a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
+++ b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
@@ -5088,13 +5088,20 @@ function ownerRequestsPythonCompute(text) {
function ownerRequestsCodeExecution(text) {
if (!text || ownerForbidsTools(text)) return false;
- const lane = ownerLaneText(text);
+ // An inline command is an operand only when owner prose asks to execute it.
+ // Replace its contents with a fixed noun; code cannot supply intent or a ban.
+ const lane = ownerLaneText(text, {preserveCommands:true}).replace(/`[^`\n]*`/g,
+ (value, offset, prose) => /\b(?:execute|run|interpret)\s+$/i.test(prose.slice(0, offset))
+ ? 'command' : ' ');
// A later instruction can withdraw an earlier execution request. Keep such
// turns outside this delivery obligation; tool authority is handled elsewhere.
if (/\b(?:do\s+not|don['’]t|never|must\s+not|should\s+not|without)\s+(?:(?:actually|really)\s+)?(?:execute|executing|run|running|interpret|interpreting)\b/i.test(lane)) return false;
const clauses = lane.split(/[!?;\n]+|\.(?=\s|$)/);
const executionVerb = /\b(?:execute|run|interpret)\b/i;
const codeObject = /\b(?:code|script|program|python|javascript|typescript|node|command|interpreter|execution\s+tool|exec\s+tool)\b/i;
+ // Require an invocation, rather than adding ordinary nouns like shell or
+ // terminal to codeObject. Version dots are part of the executable token.
+ const interpreterInvocation = /\b(?:execute|run|interpret)\s+(?:python(?:\d+(?:\.\d+)*)?|py|nodejs|deno|bun|ruby|perl|php|lua|rscript|bash|sh|zsh|pwsh|powershell)(?=\s|$)/i;
const toolPhrase = /\b(?:use|call|invoke)\s+(?:your\s+)?(?:python\s+)?(?:execution|exec|interpreter|terminal|shell)\s+tool\b/i;
// Meta/instructional framings that mention execution without requesting it.
const metaPrefix = /^\s*(?:please\s+)?(?:how\b|what\b|why\b|where\b|when\b|explain\b|describe\b|show\s+(?:me\s+)?(?:a\s+)?command\b|tell\s+me\s+how\b|give\s+me\b|print\s+(?:the\s+)?(?:words?|text|string)\b|quote\b|write\s+(?:the\s+)?(?:words?|text|string)\b|say\s+(?:the\s+)?(?:words?|text|string)\b)/i;
@@ -5108,7 +5115,7 @@ function ownerRequestsCodeExecution(text) {
if (metaPrefix.test(clause)) return false;
if (codeToRun.test(clause)) return false;
if (toolPhrase.test(clause)) return true;
- return executionVerb.test(clause) && codeObject.test(clause);
+ return (executionVerb.test(clause) && codeObject.test(clause)) || interpreterInvocation.test(clause);
});
}
@@ -8416,8 +8423,7 @@ export function createToolLoopGuard({
// owner request to execute code.
executionReceiptObserved: false,
executionRequested: false,
- explicitPythonComputeRequested: false,
- explicitPythonComputeRetried: false,
+ executionRequestRetried: false,
// Phantom-process bookkeeping: allowed exec calls whose receipt has
// not been observed yet, and whether any exec went to the background.
execCallsInFlight: new Set(),
@@ -10950,9 +10956,10 @@ export function createToolLoopGuard({
state.workspacePreviewRequired ||
userMessageRequestsWorkspaceMutation(event?.messages, event?.prompt);
// Later native attempts carry a harness instruction, not a new owner
- // request. Keep the computation obligation bound to the first request.
- state.explicitPythonComputeRequested = ownerRequestsPythonCompute(state.ownerRequestText);
- state.executionRequested = ownerRequestsCodeExecution(laneText) || state.explicitPythonComputeRequested;
+ // request. Keep the execution obligation bound to the first request,
+ // including inline command operands stripped from other intent lanes.
+ state.executionRequested = ownerRequestsCodeExecution(state.ownerRequestText) ||
+ ownerRequestsPythonCompute(state.ownerRequestText);
state.workspaceTaskPath = userMessageWorkspaceContinuationPath(
event?.messages,
event?.prompt
@@ -13855,11 +13862,11 @@ export function createToolLoopGuard({
'Do not substitute a raw path, Markdown link or image for publication.',
idempotencyKey:'pixel-ods-workspace-document-delivery', maxAttempts:1}};
}
- // A prose-only answer can omit the Python execution the owner requested.
+ // A prose-only answer can omit the code execution the owner requested.
// Let the model correct that once before any effectful tool attempt.
// Existing continuations take precedence, and the normal tool admission
// and terminal execution-receipt checks still decide what may run/pass.
- if (state?.explicitPythonComputeRequested && !state.explicitPythonComputeRetried &&
+ if (state?.executionRequested && !state.executionRequestRetried &&
!state.executionReceiptObserved && !state.toolExecutionAttempted &&
!state.ownerNoTools && !state.extensionCompletionGate?.active &&
!state.odsAborted && !state.codingExhausted && !state.runDeadlineReached &&
@@ -13869,11 +13876,11 @@ export function createToolLoopGuard({
(!event?.sessionId || event.sessionId === state.currentSessionId) &&
(!event?.sessionKey || event.sessionKey === state.currentSessionKey) &&
!workspaceArtifactUnavailableReason({...context, runId})) {
- state.explicitPythonComputeRetried = true;
- return {action:'revise', reason:'The requested Python computation has no execution receipt.', retry:{
- instruction:'The owner asked you to use Python for a computation. Use the exec tool to run Python, ' +
+ state.executionRequestRetried = true;
+ return {action:'revise', reason:'The requested code run has no execution receipt.', retry:{
+ instruction:'The owner requested actual code execution. Use the exec tool to run the requested code, ' +
'then report the command and its observed output. Do not substitute an estimated or inferred answer.',
- idempotencyKey:'pixel-ods-explicit-python-compute', maxAttempts:1}};
+ idempotencyKey:'pixel-ods-explicit-code-execution', maxAttempts:1}};
}
const decision = state?.extensionCompletionGate?.active && !extensionStopped
? state.extensionCompletionGate.finalize()
diff --git a/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs b/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs
new file mode 100644
index 000000000..4b02e9efe
--- /dev/null
+++ b/ods/extensions/services/pixel-agent/tests/explicit_interpreter_execution.test.mjs
@@ -0,0 +1,118 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {createToolLoopGuard} from '../plugin/tool-loop-guard.mjs';
+
+const retryKey='pixel-ods-explicit-code-execution';
+const user='ods-'+ 'd'.repeat(64);
+const livePrompt='Execute python3 -c "print(sum(i*i for i in range(1, 14)))" in the terminal and report the observed stdout.';
+function turn(prompt=livePrompt, overrides={}) {
+ const guard=createToolLoopGuard({abortRunAndDrain:async()=>({aborted:true,drained:true,forceCleared:false})});
+ const context={agentId:'pixel',runId:'interp',sessionId:'session',sessionKey:'agent:pixel:openai-user:'+user,trigger:'user',...overrides};
+ guard.observeRun(context,'pixel',{prompt});
+ const finalize=(ctx=context,event={})=>guard.beforeAgentFinalize({runId:context.runId,
+ lastAssistantMessage:'The observed stdout is 91.',
+ messages:[{role:'assistant',content:[{type:'text',text:'The observed stdout is 91.'}]}],...event},ctx);
+ return {guard,context,finalize};
+}
+function execute(f, exitCode=0, status='completed', command='python3 -c "print(sum(i*i for i in range(1, 14)))"') {
+ const context={...f.context,toolCallId:'interp-exec',toolName:'exec'};
+ const params={command,workdir:'/workspace'};
+ const prepared=f.guard.beforeToolCall({toolName:'exec',params},context);
+ assert.notEqual(prepared?.block,true,prepared?.blockReason);
+ f.guard.afterToolCall({toolName:'exec',params:prepared?.params??params,
+ result:{details:{status,exitCode,aggregated:exitCode===0?'819':'Interpreter failed'}}},context);
+}
+
+test('the exact live command cannot deliver a fabricated stdout',()=>{
+ const f=turn(), result=f.guard.deliveryVerificationForRun('interp');
+ assert.equal(result.status,'failed');
+ assert.match(result.text,/did not receive a successful terminal execution receipt/);
+ assert.doesNotMatch(result.text,/91/);
+ const decision=f.finalize();
+ assert.equal(decision?.retry?.idempotencyKey,retryKey);
+ assert.equal(decision.retry.maxAttempts,1);
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');
+});
+
+test('a successful execution satisfies the request, including after a native continuation',()=>{
+ const f=turn();assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);
+ f.guard.observeRun(f.context,'pixel',{prompt:'Continue from the previous attempt.'});
+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');
+ execute(f);
+ assert.notEqual(f.guard.deliveryVerificationForRun('interp').status,'failed');
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+});
+
+test('versioned and inline interpreter commands require execution',()=>{
+ for(const prompt of [livePrompt,
+ 'Now use the terminal: execute python3 -c "print(sum(i*i for i in range(1, 11)))" and report the observed stdout.',
+ 'Run `python3 -c "print(1)"` and report its output.',
+ 'Please run `node -e "console.log(1)"`.',
+ 'Can you execute python3.12 -c "print(1)"?',
+ 'Run python3 -m pytest and report the results.',
+ 'Run py -3 -c "print(1)".',
+ 'Run nodejs -e "console.log(1)".',
+ 'Execute deno eval "console.log(1)".',
+ 'Run bash -c "printf 1".',
+ 'Run pwsh -c "Write-Output 1".',
+ 'Run the shell script to deploy.',
+ ]) {
+ const f=turn(prompt);
+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed',prompt);
+ const retry=f.finalize()?.retry;
+ assert.equal(retry?.idempotencyKey,retryKey,prompt);
+ assert.doesNotMatch(retry.instruction,/run Python/);
+ }
+});
+
+test('explanations, code generation, quoted examples and no-tools stay outside execution',()=>{
+ for(const prompt of [
+ 'Explain how to execute python3 -c "print(1)".',
+ 'Explain `run python3 -c "print(1)"`.',
+ 'Please explain this example: `execute python3 -c "print(1)"`.',
+ 'The documentation says "execute python3 -c print(1)". Explain it.',
+ '> Run python3 -c "print(1)".\nSummarize that quote.',
+ '```\nRun python3 -c "print(1)"\n```\nExplain the sample.',
+ 'Write Python code to execute python3 -c "print(1)".',
+ 'Give me a script to run python3 -c "print(1)" locally.',
+ 'Do not execute python3 -c "print(1)".',
+ 'Don’t run `python3 -c "print(1)"`.',
+ 'Run python3 -c "print(1)"; do not actually execute anything.',
+ 'Do not use tools. Execute python3 -c "print(1)".',
+ 'Calculate 2+2.',
+ 'Run a company selling shell furniture.',
+ 'Run the terminal emulator.',
+ ]) {
+ const f=turn(prompt);
+ assert.notEqual(f.guard.deliveryVerificationForRun('interp').status,'failed',prompt);
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey,prompt);
+ }
+});
+
+test('failed and running attempts are not repeated by the correction',()=>{
+ for(const [exitCode,status] of [[1,'completed'],[0,'running']]) {
+ const f=turn();execute(f,exitCode,status);
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');
+ }
+});
+
+test('command contents cannot cancel the execution obligation',()=>{
+ const f=turn('Run `python3 -c "print(\'do not run tools\')"` and report stdout.');
+ assert.equal(f.guard.deliveryVerificationForRun('interp').status,'failed');
+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);
+});
+
+test('cancelled, foreign and ended runs cannot request execution',async()=>{
+ const f=turn();
+ for(const ctx of [{...f.context,runId:'other'},{...f.context,sessionId:'other'},{...f.context,sessionKey:'other'}])
+ assert.notEqual(f.finalize(ctx)?.retry?.idempotencyKey,retryKey);
+ for(const event of [{runId:'other'},{sessionId:'other'},{sessionKey:'other'}])
+ assert.notEqual(f.finalize(f.context,event)?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);
+ const cancelled=turn();assert.equal(await cancelled.guard.abortUserRun(user),true);
+ assert.notEqual(cancelled.finalize()?.retry?.idempotencyKey,retryKey);
+ const ended=turn();ended.guard.observeAgentEnd({success:true},ended.context);
+ assert.notEqual(ended.finalize()?.retry?.idempotencyKey,retryKey);
+});
diff --git a/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
index ccaea075b..dcb38ed97 100644
--- a/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
+++ b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
@@ -4,7 +4,7 @@ import {createToolLoopGuard} from '../plugin/tool-loop-guard.mjs';
import {RUN_PROGRESS_LIMITS} from '../plugin/run-progress-budget.mjs';
const prompt='Use Python to calculate sum(i*i for i in range(1, 21)). Show the exact command and its result.';
-const retryKey='pixel-ods-explicit-python-compute';
+const retryKey='pixel-ods-explicit-code-execution';
const user='ods-'+ 'c'.repeat(64);
function turn(options={}) {
const guard=createToolLoopGuard({abortRunAndDrain:async()=>({aborted:true,drained:true,forceCleared:false})});