Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): retry the output publish once on an agent 429

Alfredo Lopezfa6c838388c6
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 9dfd0c21a..68eac6fd9 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 774903918..cbb672a79 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
index f77a2103c..f82f2c884 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
@@ -105,4 +105,94 @@ describe('publishWorkflowOutputsToGfs', () => {
)
).rejects.toThrow(/HTTP 403/)
})
+
+ describe('a 429 from the gfsc agent limiter', () => {
+ const env = {
+ GFS_ACCESS_FILE: '/var/run/workflow/gfs',
+ CLERUM_WORKFLOW_RUN_ID: 'run-123',
+ CLERUM_GFSC_WRITER_BASE_URL: 'http://writer.local',
+ }
+
+ function agentDenied(retryAfter: string, scope = 'agent_writes'): Response {
+ return new Response(JSON.stringify({ error: { code: 'rate_limited', limit: scope } }), {
+ status: 429,
+ headers: {
+ 'content-type': 'application/json',
+ 'retry-after': retryAfter,
+ 'x-gfs-ratelimit-scope': scope,
+ },
+ })
+ }
+
+ async function publishWith(responses: Response[]) {
+ const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => {
+ const next = responses.shift()
+ if (!next) throw new Error('unexpected extra fetch')
+ return next
+ })
+ const sleepFn = vi.fn(async (_ms: number) => {})
+ const published = publishWorkflowOutputsToGfs(
+ { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },
+ { workflowName: 'daily-report' },
+ { summarize: { ok: true } },
+ {
+ env,
+ fetchFn: fetchFn as unknown as typeof fetch,
+ readFileFn: vi.fn(async () => 'runtime-access') as never,
+ sleepFn,
+ }
+ )
+ return { published, fetchFn, sleepFn }
+ }
+
+ it('waits Retry-After and publishes on the one retry, sending the same request twice', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith([
+ agentDenied('7'),
+ jsonResponse({ data: { resourceId: 'created-file' } }, 201),
+ ])
+
+ await expect(published).resolves.toBeUndefined()
+ expect(sleepFn.mock.calls).toEqual([[7_000]])
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ const [first, second] = fetchFn.mock.calls
+ expect(String(second![0])).toBe(String(first![0]))
+ expect(String(first![0])).toBe(`http://writer.local/v1/resources/${PARENT_ID}/children`)
+ expect(second![1]?.method).toBe('POST')
+ expect(second![1]?.body).toBe(first![1]?.body)
+ expect(second![1]?.headers).toEqual(first![1]?.headers)
+ })
+
+ it('fails with the status after a second 429, making exactly two requests', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith([
+ agentDenied('3'),
+ agentDenied('3'),
+ ])
+
+ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ expect(sleepFn.mock.calls).toEqual([[3_000]])
+ })
+
+ it('does not retry an upload-quota 429', async () => {
+ const { published, fetchFn, sleepFn } = await publishWith([
+ agentDenied('3', 'active_sessions_subject'),
+ ])
+
+ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)
+ // Witness: the one request was made and its 429 surfaced.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(sleepFn).not.toHaveBeenCalled()
+ })
+
+ it.each(['61', '0', '', '1.5', '3600'])(
+ 'does not retry when Retry-After is %j (outside 1..60 s)',
+ async retryAfter => {
+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied(retryAfter)])
+
+ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(sleepFn).not.toHaveBeenCalled()
+ }
+ )
+ })
})
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
index 938588217..2d3b6114c 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
@@ -1,4 +1,5 @@
import { readFile } from 'node:fs/promises'
+import { setTimeout as delay } from 'node:timers/promises'
import type { WorkflowConfig } from '@clerum/workflow-runtime-core'
export interface GfsPublishTarget {
@@ -16,6 +17,7 @@ export interface GfsOutputPublisherDeps {
env?: NodeJS.ProcessEnv
fetchFn?: typeof fetch
readFileFn?: typeof readFile
+ sleepFn?: (ms: number) => Promise<void>
}
const DEFAULT_GFSC_READER_BASE_URL = 'http://gfsc.gfs.svc.cluster.local:8087'
@@ -23,6 +25,11 @@ const DEFAULT_GFSC_WRITER_BASE_URL = 'http://gfsc-writer.gfs.svc.cluster.local:8
const RESOURCE_ID_RE = /^[a-fA-F0-9][a-fA-F0-9-]{30,40}[a-fA-F0-9]$/
const HEADER_NAME = 'authorization'
const HEADER_SCHEME = 'Bearer'
+// gfsc's per-replica agent limiter answers before the permission store and
+// the executor run, so a write it denied had no effect and sending it again
+// is safe. Upload-quota 429s carry other scopes and are not retried.
+const RETRYABLE_SCOPES = new Set(['agent_reads', 'agent_writes'])
+const MAX_RETRY_AFTER_SECONDS = 60
export async function publishWorkflowOutputsToGfs(
spec: GfsPublishWorkflowSpec,
@@ -38,6 +45,7 @@ export async function publishWorkflowOutputsToGfs(
const env = deps.env ?? process.env
const fetchFn = deps.fetchFn ?? fetch
const readFileFn = deps.readFileFn ?? readFile
+ const sleepFn = deps.sleepFn ?? ((ms: number) => delay(ms))
const accessFile = env.GFS_ACCESS_FILE?.trim()
if (!accessFile) {
throw new Error('GFS_ACCESS_FILE is required when spec.gfs.publishTargets is configured')
@@ -61,17 +69,24 @@ export async function publishWorkflowOutputsToGfs(
for (const target of targets) {
const parentId = await resolvePublishParent(target, accessValue, fetchFn, env)
- const response = await fetchFn(
- `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`,
- {
- method: 'POST',
- headers: {
- [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,
- 'content-type': 'application/json',
- },
- body: JSON.stringify({ name, kind: 'file', content }),
- }
- )
+ const url = `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`
+ const init: RequestInit = {
+ method: 'POST',
+ headers: {
+ [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,
+ 'content-type': 'application/json',
+ },
+ body: JSON.stringify({ name, kind: 'file', content }),
+ }
+ let response = await fetchFn(url, init)
+ // One retry, only for an agent-limiter 429 that states a delay of at most
+ // MAX_RETRY_AFTER_SECONDS; a second denial fails below with its status.
+ const retryAfterSeconds = agentRetryAfterSeconds(response)
+ if (retryAfterSeconds !== undefined) {
+ await response.body?.cancel()
+ await sleepFn(retryAfterSeconds * 1000)
+ response = await fetchFn(url, init)
+ }
if (!response.ok) {
throw new Error(
`GFS output publish failed: HTTP ${response.status} ${await responseText(response)}`
@@ -107,6 +122,16 @@ async function resolvePublishParent(
return resourceId
}
+/** The Retry-After of a retryable agent-limiter 429, in seconds; undefined otherwise. */
+function agentRetryAfterSeconds(response: Response): number | undefined {
+ if (response.status !== 429) return undefined
+ if (!RETRYABLE_SCOPES.has(response.headers.get('x-gfs-ratelimit-scope') ?? '')) return undefined
+ const retryAfter = response.headers.get('retry-after') ?? ''
+ if (!/^[1-9][0-9]?$/.test(retryAfter)) return undefined
+ const seconds = Number(retryAfter)
+ return seconds <= MAX_RETRY_AFTER_SECONDS ? seconds : undefined
+}
+
function outputFileName(workflowName: string, workflowRunId: string | null): string {
const suffix = workflowRunId || workflowName || 'workflow'
return `workflow-output-${suffix}.json`