Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): stop the GFS output publish retry on cancel and retry the resolve too

Alfredo Lopez5f4eff6485e4
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 09b8aeca9..3da8ec4dd 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 5d5a68519..76e8ba257 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
index 3344eeba9..61c8ff670 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
-import { publishWorkflowOutputsToGfs } from './gfsOutputPublisher'
+import { GfsPublishCancelledError, publishWorkflowOutputsToGfs } from './gfsOutputPublisher'
const PARENT_ID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
@@ -124,27 +124,86 @@ describe('publishWorkflowOutputsToGfs', () => {
})
}
- async function publishWith(responses: Response[]) {
+ async function publishWith(
+ responses: Response[],
+ opts: { target?: string; cancelDuringSleep?: boolean } = {}
+ ) {
const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => {
const next = responses.shift()
if (!next) throw new Error('unexpected extra fetch')
return next
})
- const sleepFn = vi.fn(async (_ms: number) => {})
+ let cancelled = false
+ const sleepFn = vi.fn(async (_ms: number) => {
+ if (opts.cancelDuringSleep) cancelled = true
+ })
const published = publishWorkflowOutputsToGfs(
- { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },
+ { gfs: { publishTargets: [{ drive: 'main', target: opts.target ?? PARENT_ID }] } },
{ workflowName: 'daily-report' },
{ summarize: { ok: true } },
{
- env,
+ env: { ...env, CLERUM_GFSC_BASE_URL: 'http://reader.local' },
fetchFn: fetchFn as unknown as typeof fetch,
readFileFn: vi.fn(async () => 'runtime-access') as never,
sleepFn,
+ isCancelled: () => cancelled,
}
)
return { published, fetchFn, sleepFn }
}
+ it('R1-L3: stops without the retry when the run is cancelled during the wait', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied('7')], {
+ cancelDuringSleep: true,
+ })
+
+ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)
+ // Witness: the wait was entered, so the check after it is what stopped the retry.
+ expect(sleepFn.mock.calls).toEqual([[7_000]])
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ })
+
+ it('N5: retries a gfs:// resolve once on an agent_reads 429, then publishes', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith(
+ [
+ agentDenied('2', 'agent_reads'),
+ jsonResponse({ data: { resourceId: PARENT_ID } }),
+ jsonResponse({ data: { resourceId: 'created-file' } }, 201),
+ ],
+ { target: 'gfs://main/some-folder' }
+ )
+
+ await expect(published).resolves.toBeUndefined()
+ expect(sleepFn.mock.calls).toEqual([[2_000]])
+ expect(fetchFn.mock.calls.map(([url]) => String(url))).toEqual([
+ 'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',
+ 'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',
+ `http://writer.local/v1/resources/${PARENT_ID}/children`,
+ ])
+ })
+
+ it('N5: fails the resolve with its status after a second agent_reads 429', async () => {
+ const { published, fetchFn } = await publishWith(
+ [agentDenied('2', 'agent_reads'), agentDenied('2', 'agent_reads')],
+ { target: 'gfs://main/some-folder' }
+ )
+
+ await expect(published).rejects.toThrow(/GFS output target resolve failed: HTTP 429/)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ })
+
+ it('N5: stops without the resolve retry when the run is cancelled during the wait', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied('2', 'agent_reads')], {
+ target: 'gfs://main/some-folder',
+ cancelDuringSleep: true,
+ })
+
+ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)
+ // Witness: the resolve's wait was entered.
+ expect(sleepFn.mock.calls).toEqual([[2_000]])
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ })
+
it('waits Retry-After and publishes on the one retry, sending the same request twice', async () => {
const { published, fetchFn, sleepFn } = await publishWith([
agentDenied('7'),
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
index 2d3b6114c..2d335e2c8 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
@@ -18,6 +18,25 @@ export interface GfsOutputPublisherDeps {
fetchFn?: typeof fetch
readFileFn?: typeof readFile
sleepFn?: (ms: number) => Promise<void>
+ /** True once the run has been cancelled; read after each retry wait. */
+ isCancelled?: () => boolean
+}
+
+/**
+ * The run was cancelled while the publisher waited out a 429's Retry-After, so
+ * the retry was not sent. The caller reports the run as cancelled, not failed.
+ */
+export class GfsPublishCancelledError extends Error {
+ constructor() {
+ super('GFS output publish stopped: the workflow run was cancelled')
+ this.name = 'GfsPublishCancelledError'
+ }
+}
+
+interface AgentRetry {
+ fetchFn: typeof fetch
+ sleepFn: (ms: number) => Promise<void>
+ isCancelled: () => boolean
}
const DEFAULT_GFSC_READER_BASE_URL = 'http://gfsc.gfs.svc.cluster.local:8087'
@@ -46,6 +65,7 @@ export async function publishWorkflowOutputsToGfs(
const fetchFn = deps.fetchFn ?? fetch
const readFileFn = deps.readFileFn ?? readFile
const sleepFn = deps.sleepFn ?? ((ms: number) => delay(ms))
+ const retry: AgentRetry = { fetchFn, sleepFn, isCancelled: deps.isCancelled ?? (() => false) }
const accessFile = env.GFS_ACCESS_FILE?.trim()
if (!accessFile) {
throw new Error('GFS_ACCESS_FILE is required when spec.gfs.publishTargets is configured')
@@ -68,25 +88,20 @@ export async function publishWorkflowOutputsToGfs(
)
for (const target of targets) {
- const parentId = await resolvePublishParent(target, accessValue, fetchFn, env)
+ const parentId = await resolvePublishParent(target, accessValue, retry, env)
const url = `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`
- const init: RequestInit = {
- method: 'POST',
- headers: {
- [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,
- 'content-type': 'application/json',
+ const response = await fetchWithAgentRetry(
+ url,
+ {
+ method: 'POST',
+ headers: {
+ [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,
+ 'content-type': 'application/json',
+ },
+ body: JSON.stringify({ name, kind: 'file', content }),
},
- body: JSON.stringify({ name, kind: 'file', content }),
- }
- let response = await fetchFn(url, init)
- // One retry, only for an agent-limiter 429 that states a delay of at most
- // MAX_RETRY_AFTER_SECONDS; a second denial fails below with its status.
- const retryAfterSeconds = agentRetryAfterSeconds(response)
- if (retryAfterSeconds !== undefined) {
- await response.body?.cancel()
- await sleepFn(retryAfterSeconds * 1000)
- response = await fetchFn(url, init)
- }
+ retry
+ )
if (!response.ok) {
throw new Error(
`GFS output publish failed: HTTP ${response.status} ${await responseText(response)}`
@@ -98,7 +113,7 @@ export async function publishWorkflowOutputsToGfs(
async function resolvePublishParent(
target: GfsPublishTarget,
accessValue: string,
- fetchFn: typeof fetch,
+ retry: AgentRetry,
env: NodeJS.ProcessEnv
): Promise<string> {
const rawTarget = target.target.trim()
@@ -107,9 +122,10 @@ async function resolvePublishParent(
throw new Error(`unsupported GFS publish target: ${rawTarget}`)
}
- const response = await fetchFn(
+ const response = await fetchWithAgentRetry(
`${baseUrl(env.CLERUM_GFSC_BASE_URL, DEFAULT_GFSC_READER_BASE_URL)}/v1/resolve?uri=${encodeURIComponent(rawTarget)}`,
- { headers: { [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}` } }
+ { headers: { [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}` } },
+ retry
)
if (!response.ok) {
throw new Error(
@@ -122,6 +138,26 @@ async function resolvePublishParent(
return resourceId
}
+/**
+ * Sends the request, and once more after an agent-limiter 429 that states a
+ * delay of at most MAX_RETRY_AFTER_SECONDS. A second denial is returned for the
+ * caller to fail with its status. A cancel that arrives during the wait stops
+ * the retry: the run is over, and the wait can last up to a minute.
+ */
+async function fetchWithAgentRetry(
+ url: string,
+ init: RequestInit,
+ retry: AgentRetry
+): Promise<Response> {
+ const response = await retry.fetchFn(url, init)
+ const retryAfterSeconds = agentRetryAfterSeconds(response)
+ if (retryAfterSeconds === undefined) return response
+ await response.body?.cancel()
+ await retry.sleepFn(retryAfterSeconds * 1000)
+ if (retry.isCancelled()) throw new GfsPublishCancelledError()
+ return retry.fetchFn(url, init)
+}
+
/** The Retry-After of a retryable agent-limiter 429, in seconds; undefined otherwise. */
function agentRetryAfterSeconds(response: Response): number | undefined {
if (response.status !== 429) return undefined
diff --git a/workflow-recipes/src/workflow/sdkRuntime.test.ts b/workflow-recipes/src/workflow/sdkRuntime.test.ts
index b0111d86a..7dfa48150 100644
--- a/workflow-recipes/src/workflow/sdkRuntime.test.ts
+++ b/workflow-recipes/src/workflow/sdkRuntime.test.ts
@@ -101,6 +101,52 @@ describe('runWorkflowRuntime GFS publishing', () => {
}
})
+ it('R1-L3: reports the run cancelled when the cancel arrives while the publish waits out a 429', async () => {
+ const tmp = await mkdtemp(join(tmpdir(), 'wrc-gfs-'))
+ const accessFile = join(tmp, 'access')
+ await writeFile(accessFile, 'runtime-access')
+ const fetchFn = vi.fn(
+ async () =>
+ new Response('{}', {
+ status: 429,
+ headers: { 'retry-after': '1', 'x-gfs-ratelimit-scope': 'agent_writes' },
+ })
+ )
+ vi.stubGlobal('fetch', fetchFn)
+ const previousFile = process.env.GFS_ACCESS_FILE
+ process.env.GFS_ACCESS_FILE = accessFile
+ try {
+ const deps = buildRuntimeDeps({
+ spec: {
+ namespace: 'sandbox-recipes',
+ name: 'publish-recipe',
+ steps: [{ id: 'publish' }],
+ gfs: { publishTargets: [{ drive: 'main', target: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }] },
+ },
+ })
+ vi.mocked(deps.coordinator.runWorkflow).mockResolvedValue({ publish: { ok: true } })
+ // Not cancelled when the steps finish; cancelled by the end of the wait.
+ vi.mocked(deps.signals.hasSignal).mockReturnValueOnce(false).mockReturnValue(true)
+
+ const result = await runWorkflowRuntime(deps)
+
+ expect(result).toMatchObject({ exitCode: 1, workflowPhase: 'cancelled' })
+ expect(deps.status.reportWorkflowStatus).toHaveBeenCalledWith('cancelled')
+ // Witness: the publish was sent and denied, and the retry was not.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(deps.status.reportWorkflowStatus).not.toHaveBeenCalledWith('failed', expect.anything())
+ expect(deps.status.reportWorkflowStatus).not.toHaveBeenCalledWith(
+ 'completed',
+ expect.anything()
+ )
+ } finally {
+ vi.unstubAllGlobals()
+ if (previousFile === undefined) delete process.env.GFS_ACCESS_FILE
+ else process.env.GFS_ACCESS_FILE = previousFile
+ await rm(tmp, { recursive: true, force: true })
+ }
+ })
+
it('fails the workflow when GFSC denies a configured publish target', async () => {
const tmp = await mkdtemp(join(tmpdir(), 'wrc-gfs-'))
const accessFile = join(tmp, 'access')
diff --git a/workflow-recipes/src/workflow/sdkRuntime.ts b/workflow-recipes/src/workflow/sdkRuntime.ts
index 4c5b26875..bef81dbab 100644
--- a/workflow-recipes/src/workflow/sdkRuntime.ts
+++ b/workflow-recipes/src/workflow/sdkRuntime.ts
@@ -20,7 +20,7 @@ import {
withRetry,
} from '@clerum/workflow-runtime-core'
import { workflowStepDurationSeconds, workflowStepTotal } from '../metrics'
-import { publishWorkflowOutputsToGfs } from './gfsOutputPublisher'
+import { GfsPublishCancelledError, publishWorkflowOutputsToGfs } from './gfsOutputPublisher'
import { workflowStepNeedsMcpHost, workflowStepUsesSnippetRunner } from './runtimePlan'
import { SNIPPET_RUN_KEYS } from './snippetRunSchema'
import { SnippetRunnerClient } from './snippetRunnerClient'
@@ -324,10 +324,16 @@ export async function runWorkflowRuntime(deps: RuntimeDependencies): Promise<Run
return { exitCode: 1, workflowPhase: 'cancelled' }
}
- await publishWorkflowOutputsToGfs(spec, config, finalOutputs)
+ await publishWorkflowOutputsToGfs(spec, config, finalOutputs, {
+ isCancelled: () => signals.hasSignal('cancel'),
+ })
await status.reportWorkflowStatus('completed', { completedAt: new Date().toISOString() })
return { exitCode: 0, workflowPhase: 'completed' }
} catch (err) {
+ if (err instanceof GfsPublishCancelledError) {
+ await status.reportWorkflowStatus('cancelled')
+ return { exitCode: 1, workflowPhase: 'cancelled' }
+ }
const failureReason = err instanceof Error ? err.message : String(err)
const failedStepId = err instanceof StepRuntimeError ? err.stepId : undefined
if (failedStepId) {