Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(pixel): require execution for explicit Python computations (#8259)

Mike Bradley1f66492cff1b
diff --git a/.github/workflows/portal-runtime.yml b/.github/workflows/portal-runtime.yml
index eec32c5ea..0759a580b 100644
--- a/.github/workflows/portal-runtime.yml
+++ b/.github/workflows/portal-runtime.yml
@@ -71,6 +71,8 @@ jobs:
run: node --test ods/extensions/services/pixel-agent/tests/artifact_publication_intent.test.mjs ods/extensions/services/pixel-agent/tests/repeat_document_delivery.test.mjs
- name: Verify per-turn no-tools policy and installed ingress layout
run: node --test ods/extensions/services/pixel-agent/tests/ingress_owner_no_tools.test.mjs
+ - name: Verify explicit Python execution requests and bounded recovery
+ run: node --test ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
- name: Verify bounded preview delivery across platforms
run: |
node --test ods/extensions/services/pixel-agent/tests/preview_delivery_recovery.test.mjs
diff --git a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
index f697bcbef..cc623fc1e 100644
--- a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
+++ b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs
@@ -5072,6 +5072,20 @@ function namesAttachedPicture(params, ids) {
// boundary only; it grants no tool or execution authority. Explanations,
// quoted commands, code fences, ordinary math, no-tools directives and
// negations are excluded. A request to write AND execute code is preserved.
+// An explicit choice of Python for a computation also requests execution,
+// even when the owner never says "run". Examples and explanations are not
+// requests to execute. This records an obligation, not additional authority.
+function ownerRequestsPythonCompute(text) {
+ if (!text || ownerForbidsTools(text)) return false;
+ const lane = ownerLaneText(text);
+ if (/\b(?:do\s+not|don['’]t|never|must\s+not|should\s+not|without)\s+(?:(?:actually|really)\s+)?(?:execute|executing|run|running|interpret|interpreting|use|using|call|calling|invoke|invoking|calculate|compute|evaluate|check|verify)\b/i.test(lane)) return false;
+ return lane.split(/[!?;\n]+|\.(?=\s|$)/).some(raw => {
+ const clause = raw.trim().replace(/^(?:please|can\s+you|could\s+you|would\s+you)\s+/i, '');
+ return /^use\s+(?:(?:the|your)\s+)?python(?:\s+(?:tool|interpreter|execution\s+tool|exec\s+tool))?\s+to\s+(?:calculate|compute|evaluate|check|verify)\b/i.test(clause) ||
+ /^(?:calculate|compute|evaluate|check|verify)\b[^.!?;\n]{0,200}\b(?:using|with)\s+(?:(?:the|your)\s+)?python\b/i.test(clause);
+ });
+}
+
function ownerRequestsCodeExecution(text) {
if (!text || ownerForbidsTools(text)) return false;
const lane = ownerLaneText(text);
@@ -8402,6 +8416,8 @@ export function createToolLoopGuard({
// owner request to execute code.
executionReceiptObserved: false,
executionRequested: false,
+ explicitPythonComputeRequested: false,
+ explicitPythonComputeRetried: false,
// Phantom-process bookkeeping: allowed exec calls whose receipt has
// not been observed yet, and whether any exec went to the background.
execCallsInFlight: new Set(),
@@ -10911,7 +10927,10 @@ export function createToolLoopGuard({
state.workspaceMutationRequested =
state.workspacePreviewRequired ||
userMessageRequestsWorkspaceMutation(event?.messages, event?.prompt);
- state.executionRequested = ownerRequestsCodeExecution(laneText);
+ // Later native attempts carry a harness instruction, not a new owner
+ // request. Keep the computation obligation bound to the first request.
+ state.explicitPythonComputeRequested = ownerRequestsPythonCompute(state.ownerRequestText);
+ state.executionRequested = ownerRequestsCodeExecution(laneText) || state.explicitPythonComputeRequested;
state.workspaceTaskPath = userMessageWorkspaceContinuationPath(
event?.messages,
event?.prompt
@@ -13807,6 +13826,26 @@ export function createToolLoopGuard({
'Do not substitute a raw path, Markdown link or image for publication.',
idempotencyKey:'pixel-ods-workspace-document-delivery', maxAttempts:1}};
}
+ // A prose-only answer can omit the Python execution the owner requested.
+ // Let the model correct that once before any effectful tool attempt.
+ // Existing continuations take precedence, and the normal tool admission
+ // and terminal execution-receipt checks still decide what may run/pass.
+ if (state?.explicitPythonComputeRequested && !state.explicitPythonComputeRetried &&
+ !state.executionReceiptObserved && !state.toolExecutionAttempted &&
+ !state.ownerNoTools && !state.extensionCompletionGate?.active &&
+ !state.odsAborted && !state.codingExhausted && !state.runDeadlineReached &&
+ !state.progressBudget.exhaustedLanes.length &&
+ !state.pendingExecSessions.size && !state.execCallsInFlight.size && !state.backgroundExecStarted &&
+ (!event?.runId || event.runId === runId) &&
+ (!event?.sessionId || event.sessionId === state.currentSessionId) &&
+ (!event?.sessionKey || event.sessionKey === state.currentSessionKey) &&
+ !workspaceArtifactUnavailableReason({...context, runId})) {
+ state.explicitPythonComputeRetried = true;
+ return {action:'revise', reason:'The requested Python computation has no execution receipt.', retry:{
+ instruction:'The owner asked you to use Python for a computation. Use the exec tool to run Python, ' +
+ 'then report the command and its observed output. Do not substitute an estimated or inferred answer.',
+ idempotencyKey:'pixel-ods-explicit-python-compute', maxAttempts:1}};
+ }
const decision = state?.extensionCompletionGate?.active && !extensionStopped
? state.extensionCompletionGate.finalize()
// Generic promise recovery cannot distinguish the suspended portion
diff --git a/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
new file mode 100644
index 000000000..ccaea075b
--- /dev/null
+++ b/ods/extensions/services/pixel-agent/tests/explicit_python_compute.test.mjs
@@ -0,0 +1,115 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {createToolLoopGuard} from '../plugin/tool-loop-guard.mjs';
+import {RUN_PROGRESS_LIMITS} from '../plugin/run-progress-budget.mjs';
+
+const prompt='Use Python to calculate sum(i*i for i in range(1, 21)). Show the exact command and its result.';
+const retryKey='pixel-ods-explicit-python-compute';
+const user='ods-'+ 'c'.repeat(64);
+function turn(options={}) {
+ const guard=createToolLoopGuard({abortRunAndDrain:async()=>({aborted:true,drained:true,forceCleared:false})});
+ const context={agentId:'pixel',runId:'compute',sessionId:'session',sessionKey:'agent:pixel:openai-user:'+user,trigger:'user',...options.context};
+ guard.observeRun(context,'pixel',{prompt:options.prompt??prompt});
+ const finalize=(ctx=context,event={})=>guard.beforeAgentFinalize({runId:context.runId,
+ lastAssistantMessage:'The sum is 2870.',messages:[{role:'assistant',content:[{type:'text',text:'The sum is 2870.'}]}],...event},ctx);
+ return {guard,context,finalize};
+}
+function execute(f,exitCode=0,status='completed') {
+ const context={...f.context,toolCallId:'compute-exec',toolName:'exec'};
+ const params={command:"python3 -c 'print(sum(i*i for i in range(1,21)))'",workdir:'/workspace'};
+ const prepared=f.guard.beforeToolCall({toolName:'exec',params},context);
+ assert.notEqual(prepared?.block,true,prepared?.blockReason);
+ f.guard.afterToolCall({toolName:'exec',params:prepared?.params??params,
+ result:{details:{status,exitCode,aggregated:exitCode===0?'2870':'Python failed'}}},context);
+}
+
+test('the exact live Python request requires an execution receipt before finalization',()=>{
+ const f=turn(),result=f.guard.deliveryVerificationForRun('compute');
+ assert.equal(result.status,'failed');
+ assert.match(result.text,/did not receive a successful terminal execution receipt/);
+ assert.doesNotMatch(result.text,/2870/);
+});
+
+test('one revision asks for real execution and never manufactures its receipt',()=>{
+ const f=turn(),decision=f.finalize();
+ assert.equal(decision?.action,'revise');
+ assert.equal(decision.retry.idempotencyKey,retryKey);
+ assert.equal(decision.retry.maxAttempts,1);
+ assert.match(decision.retry.instruction,/exec/);
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.guard.deliveryVerificationForRun('compute').status,'failed');
+});
+
+test('revision followed by a bound successful execution satisfies the request',()=>{
+ const f=turn();assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);
+ // A native continuation cannot erase the original owner's obligation.
+ f.guard.observeRun(f.context,'pixel',{prompt:'Continue from the previous attempt.'});
+ assert.equal(f.guard.deliveryVerificationForRun('compute').status,'failed');
+ execute(f);
+ assert.notEqual(f.guard.deliveryVerificationForRun('compute').status,'failed');
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+});
+
+test('failed and pending execution attempts are not automatically repeated',()=>{
+ for(const [exitCode,status] of [[1,'completed'],[undefined,'running']]) {
+ const f=turn();execute(f,exitCode,status);
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.guard.deliveryVerificationForRun('compute').status,'failed');
+ }
+});
+
+test('imperative Python calculations are recognized without classifying examples as commands',()=>{
+ for(const text of [prompt,'Please use the Python tool to compute 2+2.',
+ 'Calculate the mean using Python.','Evaluate this expression with Python.',
+ 'Can you use Python to verify this sum?']) {
+ const f=turn({prompt:text});
+ assert.equal(f.guard.deliveryVerificationForRun('compute').status,'failed',text);
+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey,text);
+ }
+ for(const text of ['What is 2+2?','Explain how to use Python to calculate a sum.',
+ 'Write Python code to calculate a sum.','Give me Python code to run locally.',
+ 'The documentation says "Use Python to calculate the sum." Explain it.',
+ '> Use Python to calculate 2+2.\nSummarize that quote.',
+ '```\nUse Python to calculate 2+2.\n```\nExplain the sample.',
+ 'Use Python to calculate 2+2; do not actually execute anything.',
+ 'Don’t use Python to calculate it.','Do not calculate this using Python.',
+ 'Without using tools, use Python to calculate 2+2.',
+ 'Evaluate whether Python is a good teaching language.']) {
+ const f=turn({prompt:text});
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey,text);
+ assert.notEqual(f.guard.deliveryVerificationForRun('compute').status,'failed',text);
+ }
+});
+
+test('foreign event or context identities cannot consume the owner revision',()=>{
+ const f=turn();
+ for(const context of [{...f.context,runId:'other'},{...f.context,sessionId:'other'},
+ {...f.context,sessionKey:'other'}]) assert.notEqual(f.finalize(context)?.retry?.idempotencyKey,retryKey);
+ for(const event of [{runId:'other'},{sessionId:'other'},{sessionKey:'other'}])
+ assert.notEqual(f.finalize(f.context,event)?.retry?.idempotencyKey,retryKey);
+ assert.equal(f.finalize()?.retry?.idempotencyKey,retryKey);
+});
+
+test('superseded and ended runs cannot request more execution',()=>{
+ const f=turn();f.guard.observeRun({...f.context,runId:'next'},'pixel',{prompt:'Hello'});
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+ const ended=turn();ended.guard.observeAgentEnd({success:true},ended.context);
+ assert.notEqual(ended.finalize()?.retry?.idempotencyKey,retryKey);
+});
+
+test('cancelled, noninteractive, managed-team and no-tools turns cannot retry',async()=>{
+ const cancelled=turn();assert.equal(await cancelled.guard.abortUserRun(user),true);
+ assert.notEqual(cancelled.finalize()?.retry?.idempotencyKey,retryKey);
+ for(const options of [{context:{trigger:'cron'}},{context:{sessionKey:'agent:pixel:main'}},
+ {prompt:"You are the Builder in the owner's Portal team. Use Python to calculate 2+2."},
+ {prompt:'Do not use tools. Use Python to calculate 2+2.'}]) {
+ assert.notEqual(turn(options).finalize()?.retry?.idempotencyKey,retryKey,JSON.stringify(options));
+ }
+});
+
+test('exhausted progress does not obtain an additional execution revision',()=>{
+ const f=turn();
+ for(let i=0;i<RUN_PROGRESS_LIMITS.consecutiveFailures;i++) f.guard.toolResultPersist({toolCallId:'failed-'+i,
+ message:{role:'toolResult',toolName:'read',isError:true,content:[]}}, {...f.context,toolName:'read',toolCallId:'failed-'+i});
+ assert.notEqual(f.finalize()?.retry?.idempotencyKey,retryKey);
+});