Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): stop the GFS output publish retry on cancel and retry the resolve too
Alfredo Lopez5f4eff6485e4
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 09b8aeca9..3da8ec4dd 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 5d5a68519..76e8ba257 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.tsindex 3344eeba9..61c8ff670 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts@@ -1,5 +1,5 @@import { describe, expect, it, vi } from 'vitest'-import { publishWorkflowOutputsToGfs } from './gfsOutputPublisher'+import { GfsPublishCancelledError, publishWorkflowOutputsToGfs } from './gfsOutputPublisher'const PARENT_ID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'@@ -124,27 +124,86 @@ describe('publishWorkflowOutputsToGfs', () => {})}- async function publishWith(responses: Response[]) {+ async function publishWith(+ responses: Response[],+ opts: { target?: string; cancelDuringSleep?: boolean } = {}+ ) {const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => {const next = responses.shift()if (!next) throw new Error('unexpected extra fetch')return next})- const sleepFn = vi.fn(async (_ms: number) => {})+ let cancelled = false+ const sleepFn = vi.fn(async (_ms: number) => {+ if (opts.cancelDuringSleep) cancelled = true+ })const published = publishWorkflowOutputsToGfs(- { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },+ { gfs: { publishTargets: [{ drive: 'main', target: opts.target ?? PARENT_ID }] } },{ workflowName: 'daily-report' },{ summarize: { ok: true } },{- env,+ env: { ...env, CLERUM_GFSC_BASE_URL: 'http://reader.local' },fetchFn: fetchFn as unknown as typeof fetch,readFileFn: vi.fn(async () => 'runtime-access') as never,sleepFn,+ isCancelled: () => cancelled,})return { published, fetchFn, sleepFn }}+ it('R1-L3: stops without the retry when the run is cancelled during the wait', async () => {+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied('7')], {+ cancelDuringSleep: true,+ })++ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)+ // Witness: the wait was entered, so the check after it is what stopped the retry.+ expect(sleepFn.mock.calls).toEqual([[7_000]])+ expect(fetchFn).toHaveBeenCalledTimes(1)+ })++ it('N5: retries a gfs:// resolve once on an agent_reads 429, then publishes', async () => {+ const { published, fetchFn, sleepFn } = await publishWith(+ [+ agentDenied('2', 'agent_reads'),+ jsonResponse({ data: { resourceId: PARENT_ID } }),+ jsonResponse({ data: { resourceId: 'created-file' } }, 201),+ ],+ { target: 'gfs://main/some-folder' }+ )++ await expect(published).resolves.toBeUndefined()+ expect(sleepFn.mock.calls).toEqual([[2_000]])+ expect(fetchFn.mock.calls.map(([url]) => String(url))).toEqual([+ 'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',+ 'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',+ `http://writer.local/v1/resources/${PARENT_ID}/children`,+ ])+ })++ it('N5: fails the resolve with its status after a second agent_reads 429', async () => {+ const { published, fetchFn } = await publishWith(+ [agentDenied('2', 'agent_reads'), agentDenied('2', 'agent_reads')],+ { target: 'gfs://main/some-folder' }+ )++ await expect(published).rejects.toThrow(/GFS output target resolve failed: HTTP 429/)+ expect(fetchFn).toHaveBeenCalledTimes(2)+ })++ it('N5: stops without the resolve retry when the run is cancelled during the wait', async () => {+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied('2', 'agent_reads')], {+ target: 'gfs://main/some-folder',+ cancelDuringSleep: true,+ })++ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)+ // Witness: the resolve's wait was entered.+ expect(sleepFn.mock.calls).toEqual([[2_000]])+ expect(fetchFn).toHaveBeenCalledTimes(1)+ })+it('waits Retry-After and publishes on the one retry, sending the same request twice', async () => {const { published, fetchFn, sleepFn } = await publishWith([agentDenied('7'),diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.tsindex 2d3b6114c..2d335e2c8 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts@@ -18,6 +18,25 @@ export interface GfsOutputPublisherDeps {fetchFn?: typeof fetchreadFileFn?: typeof readFilesleepFn?: (ms: number) => Promise<void>+ /** True once the run has been cancelled; read after each retry wait. */+ isCancelled?: () => boolean+}++/**+ * The run was cancelled while the publisher waited out a 429's Retry-After, so+ * the retry was not sent. The caller reports the run as cancelled, not failed.+ */+export class GfsPublishCancelledError extends Error {+ constructor() {+ super('GFS output publish stopped: the workflow run was cancelled')+ this.name = 'GfsPublishCancelledError'+ }+}++interface AgentRetry {+ fetchFn: typeof fetch+ sleepFn: (ms: number) => Promise<void>+ isCancelled: () => boolean}const DEFAULT_GFSC_READER_BASE_URL = 'http://gfsc.gfs.svc.cluster.local:8087'@@ -46,6 +65,7 @@ export async function publishWorkflowOutputsToGfs(const fetchFn = deps.fetchFn ?? fetchconst readFileFn = deps.readFileFn ?? readFileconst sleepFn = deps.sleepFn ?? ((ms: number) => delay(ms))+ const retry: AgentRetry = { fetchFn, sleepFn, isCancelled: deps.isCancelled ?? (() => false) }const accessFile = env.GFS_ACCESS_FILE?.trim()if (!accessFile) {throw new Error('GFS_ACCESS_FILE is required when spec.gfs.publishTargets is configured')@@ -68,25 +88,20 @@ export async function publishWorkflowOutputsToGfs()for (const target of targets) {- const parentId = await resolvePublishParent(target, accessValue, fetchFn, env)+ const parentId = await resolvePublishParent(target, accessValue, retry, env)const url = `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`- const init: RequestInit = {- method: 'POST',- headers: {- [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,- 'content-type': 'application/json',+ const response = await fetchWithAgentRetry(+ url,+ {+ method: 'POST',+ headers: {+ [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,+ 'content-type': 'application/json',+ },+ body: JSON.stringify({ name, kind: 'file', content }),},- body: JSON.stringify({ name, kind: 'file', content }),- }- let response = await fetchFn(url, init)- // One retry, only for an agent-limiter 429 that states a delay of at most- // MAX_RETRY_AFTER_SECONDS; a second denial fails below with its status.- const retryAfterSeconds = agentRetryAfterSeconds(response)- if (retryAfterSeconds !== undefined) {- await response.body?.cancel()- await sleepFn(retryAfterSeconds * 1000)- response = await fetchFn(url, init)- }+ retry+ )if (!response.ok) {throw new Error(`GFS output publish failed: HTTP ${response.status} ${await responseText(response)}`@@ -98,7 +113,7 @@ export async function publishWorkflowOutputsToGfs(async function resolvePublishParent(target: GfsPublishTarget,accessValue: string,- fetchFn: typeof fetch,+ retry: AgentRetry,env: NodeJS.ProcessEnv): Promise<string> {const rawTarget = target.target.trim()@@ -107,9 +122,10 @@ async function resolvePublishParent(throw new Error(`unsupported GFS publish target: ${rawTarget}`)}- const response = await fetchFn(+ const response = await fetchWithAgentRetry(`${baseUrl(env.CLERUM_GFSC_BASE_URL, DEFAULT_GFSC_READER_BASE_URL)}/v1/resolve?uri=${encodeURIComponent(rawTarget)}`,- { headers: { [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}` } }+ { headers: { [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}` } },+ retry)if (!response.ok) {throw new Error(@@ -122,6 +138,26 @@ async function resolvePublishParent(return resourceId}+/**+ * Sends the request, and once more after an agent-limiter 429 that states a+ * delay of at most MAX_RETRY_AFTER_SECONDS. A second denial is returned for the+ * caller to fail with its status. A cancel that arrives during the wait stops+ * the retry: the run is over, and the wait can last up to a minute.+ */+async function fetchWithAgentRetry(+ url: string,+ init: RequestInit,+ retry: AgentRetry+): Promise<Response> {+ const response = await retry.fetchFn(url, init)+ const retryAfterSeconds = agentRetryAfterSeconds(response)+ if (retryAfterSeconds === undefined) return response+ await response.body?.cancel()+ await retry.sleepFn(retryAfterSeconds * 1000)+ if (retry.isCancelled()) throw new GfsPublishCancelledError()+ return retry.fetchFn(url, init)+}+/** The Retry-After of a retryable agent-limiter 429, in seconds; undefined otherwise. */function agentRetryAfterSeconds(response: Response): number | undefined {if (response.status !== 429) return undefineddiff --git a/workflow-recipes/src/workflow/sdkRuntime.test.ts b/workflow-recipes/src/workflow/sdkRuntime.test.tsindex b0111d86a..7dfa48150 100644--- a/workflow-recipes/src/workflow/sdkRuntime.test.ts+++ b/workflow-recipes/src/workflow/sdkRuntime.test.ts@@ -101,6 +101,52 @@ describe('runWorkflowRuntime GFS publishing', () => {}})+ it('R1-L3: reports the run cancelled when the cancel arrives while the publish waits out a 429', async () => {+ const tmp = await mkdtemp(join(tmpdir(), 'wrc-gfs-'))+ const accessFile = join(tmp, 'access')+ await writeFile(accessFile, 'runtime-access')+ const fetchFn = vi.fn(+ async () =>+ new Response('{}', {+ status: 429,+ headers: { 'retry-after': '1', 'x-gfs-ratelimit-scope': 'agent_writes' },+ })+ )+ vi.stubGlobal('fetch', fetchFn)+ const previousFile = process.env.GFS_ACCESS_FILE+ process.env.GFS_ACCESS_FILE = accessFile+ try {+ const deps = buildRuntimeDeps({+ spec: {+ namespace: 'sandbox-recipes',+ name: 'publish-recipe',+ steps: [{ id: 'publish' }],+ gfs: { publishTargets: [{ drive: 'main', target: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }] },+ },+ })+ vi.mocked(deps.coordinator.runWorkflow).mockResolvedValue({ publish: { ok: true } })+ // Not cancelled when the steps finish; cancelled by the end of the wait.+ vi.mocked(deps.signals.hasSignal).mockReturnValueOnce(false).mockReturnValue(true)++ const result = await runWorkflowRuntime(deps)++ expect(result).toMatchObject({ exitCode: 1, workflowPhase: 'cancelled' })+ expect(deps.status.reportWorkflowStatus).toHaveBeenCalledWith('cancelled')+ // Witness: the publish was sent and denied, and the retry was not.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(deps.status.reportWorkflowStatus).not.toHaveBeenCalledWith('failed', expect.anything())+ expect(deps.status.reportWorkflowStatus).not.toHaveBeenCalledWith(+ 'completed',+ expect.anything()+ )+ } finally {+ vi.unstubAllGlobals()+ if (previousFile === undefined) delete process.env.GFS_ACCESS_FILE+ else process.env.GFS_ACCESS_FILE = previousFile+ await rm(tmp, { recursive: true, force: true })+ }+ })+it('fails the workflow when GFSC denies a configured publish target', async () => {const tmp = await mkdtemp(join(tmpdir(), 'wrc-gfs-'))const accessFile = join(tmp, 'access')diff --git a/workflow-recipes/src/workflow/sdkRuntime.ts b/workflow-recipes/src/workflow/sdkRuntime.tsindex 4c5b26875..bef81dbab 100644--- a/workflow-recipes/src/workflow/sdkRuntime.ts+++ b/workflow-recipes/src/workflow/sdkRuntime.ts@@ -20,7 +20,7 @@ import {withRetry,} from '@clerum/workflow-runtime-core'import { workflowStepDurationSeconds, workflowStepTotal } from '../metrics'-import { publishWorkflowOutputsToGfs } from './gfsOutputPublisher'+import { GfsPublishCancelledError, publishWorkflowOutputsToGfs } from './gfsOutputPublisher'import { workflowStepNeedsMcpHost, workflowStepUsesSnippetRunner } from './runtimePlan'import { SNIPPET_RUN_KEYS } from './snippetRunSchema'import { SnippetRunnerClient } from './snippetRunnerClient'@@ -324,10 +324,16 @@ export async function runWorkflowRuntime(deps: RuntimeDependencies): Promise<Runreturn { exitCode: 1, workflowPhase: 'cancelled' }}- await publishWorkflowOutputsToGfs(spec, config, finalOutputs)+ await publishWorkflowOutputsToGfs(spec, config, finalOutputs, {+ isCancelled: () => signals.hasSignal('cancel'),+ })await status.reportWorkflowStatus('completed', { completedAt: new Date().toISOString() })return { exitCode: 0, workflowPhase: 'completed' }} catch (err) {+ if (err instanceof GfsPublishCancelledError) {+ await status.reportWorkflowStatus('cancelled')+ return { exitCode: 1, workflowPhase: 'cancelled' }+ }const failureReason = err instanceof Error ? err.message : String(err)const failedStepId = err instanceof StepRuntimeError ? err.stepId : undefinedif (failedStepId) {