Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): retry the output publish once on an agent 429
Alfredo Lopezfa6c838388c6
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 9dfd0c21a..68eac6fd9 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 774903918..cbb672a79 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.tsindex f77a2103c..f82f2c884 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts@@ -105,4 +105,94 @@ describe('publishWorkflowOutputsToGfs', () => {)).rejects.toThrow(/HTTP 403/)})++ describe('a 429 from the gfsc agent limiter', () => {+ const env = {+ GFS_ACCESS_FILE: '/var/run/workflow/gfs',+ CLERUM_WORKFLOW_RUN_ID: 'run-123',+ CLERUM_GFSC_WRITER_BASE_URL: 'http://writer.local',+ }++ function agentDenied(retryAfter: string, scope = 'agent_writes'): Response {+ return new Response(JSON.stringify({ error: { code: 'rate_limited', limit: scope } }), {+ status: 429,+ headers: {+ 'content-type': 'application/json',+ 'retry-after': retryAfter,+ 'x-gfs-ratelimit-scope': scope,+ },+ })+ }++ async function publishWith(responses: Response[]) {+ const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => {+ const next = responses.shift()+ if (!next) throw new Error('unexpected extra fetch')+ return next+ })+ const sleepFn = vi.fn(async (_ms: number) => {})+ const published = publishWorkflowOutputsToGfs(+ { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },+ { workflowName: 'daily-report' },+ { summarize: { ok: true } },+ {+ env,+ fetchFn: fetchFn as unknown as typeof fetch,+ readFileFn: vi.fn(async () => 'runtime-access') as never,+ sleepFn,+ }+ )+ return { published, fetchFn, sleepFn }+ }++ it('waits Retry-After and publishes on the one retry, sending the same request twice', async () => {+ const { published, fetchFn, sleepFn } = await publishWith([+ agentDenied('7'),+ jsonResponse({ data: { resourceId: 'created-file' } }, 201),+ ])++ await expect(published).resolves.toBeUndefined()+ expect(sleepFn.mock.calls).toEqual([[7_000]])+ expect(fetchFn).toHaveBeenCalledTimes(2)+ const [first, second] = fetchFn.mock.calls+ expect(String(second![0])).toBe(String(first![0]))+ expect(String(first![0])).toBe(`http://writer.local/v1/resources/${PARENT_ID}/children`)+ expect(second![1]?.method).toBe('POST')+ expect(second![1]?.body).toBe(first![1]?.body)+ expect(second![1]?.headers).toEqual(first![1]?.headers)+ })++ it('fails with the status after a second 429, making exactly two requests', async () => {+ const { published, fetchFn, sleepFn } = await publishWith([+ agentDenied('3'),+ agentDenied('3'),+ ])++ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)+ expect(fetchFn).toHaveBeenCalledTimes(2)+ expect(sleepFn.mock.calls).toEqual([[3_000]])+ })++ it('does not retry an upload-quota 429', async () => {+ const { published, fetchFn, sleepFn } = await publishWith([+ agentDenied('3', 'active_sessions_subject'),+ ])++ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)+ // Witness: the one request was made and its 429 surfaced.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(sleepFn).not.toHaveBeenCalled()+ })++ it.each(['61', '0', '', '1.5', '3600'])(+ 'does not retry when Retry-After is %j (outside 1..60 s)',+ async retryAfter => {+ const { published, fetchFn, sleepFn } = await publishWith([agentDenied(retryAfter)])++ await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(sleepFn).not.toHaveBeenCalled()+ }+ )+ })})diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.tsindex 938588217..2d3b6114c 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts@@ -1,4 +1,5 @@import { readFile } from 'node:fs/promises'+import { setTimeout as delay } from 'node:timers/promises'import type { WorkflowConfig } from '@clerum/workflow-runtime-core'export interface GfsPublishTarget {@@ -16,6 +17,7 @@ export interface GfsOutputPublisherDeps {env?: NodeJS.ProcessEnvfetchFn?: typeof fetchreadFileFn?: typeof readFile+ sleepFn?: (ms: number) => Promise<void>}const DEFAULT_GFSC_READER_BASE_URL = 'http://gfsc.gfs.svc.cluster.local:8087'@@ -23,6 +25,11 @@ const DEFAULT_GFSC_WRITER_BASE_URL = 'http://gfsc-writer.gfs.svc.cluster.local:8const RESOURCE_ID_RE = /^[a-fA-F0-9][a-fA-F0-9-]{30,40}[a-fA-F0-9]$/const HEADER_NAME = 'authorization'const HEADER_SCHEME = 'Bearer'+// gfsc's per-replica agent limiter answers before the permission store and+// the executor run, so a write it denied had no effect and sending it again+// is safe. Upload-quota 429s carry other scopes and are not retried.+const RETRYABLE_SCOPES = new Set(['agent_reads', 'agent_writes'])+const MAX_RETRY_AFTER_SECONDS = 60export async function publishWorkflowOutputsToGfs(spec: GfsPublishWorkflowSpec,@@ -38,6 +45,7 @@ export async function publishWorkflowOutputsToGfs(const env = deps.env ?? process.envconst fetchFn = deps.fetchFn ?? fetchconst readFileFn = deps.readFileFn ?? readFile+ const sleepFn = deps.sleepFn ?? ((ms: number) => delay(ms))const accessFile = env.GFS_ACCESS_FILE?.trim()if (!accessFile) {throw new Error('GFS_ACCESS_FILE is required when spec.gfs.publishTargets is configured')@@ -61,17 +69,24 @@ export async function publishWorkflowOutputsToGfs(for (const target of targets) {const parentId = await resolvePublishParent(target, accessValue, fetchFn, env)- const response = await fetchFn(- `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`,- {- method: 'POST',- headers: {- [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,- 'content-type': 'application/json',- },- body: JSON.stringify({ name, kind: 'file', content }),- }- )+ const url = `${baseUrl(env.CLERUM_GFSC_WRITER_BASE_URL, DEFAULT_GFSC_WRITER_BASE_URL)}/v1/resources/${encodeURIComponent(parentId)}/children`+ const init: RequestInit = {+ method: 'POST',+ headers: {+ [HEADER_NAME]: `${HEADER_SCHEME} ${accessValue}`,+ 'content-type': 'application/json',+ },+ body: JSON.stringify({ name, kind: 'file', content }),+ }+ let response = await fetchFn(url, init)+ // One retry, only for an agent-limiter 429 that states a delay of at most+ // MAX_RETRY_AFTER_SECONDS; a second denial fails below with its status.+ const retryAfterSeconds = agentRetryAfterSeconds(response)+ if (retryAfterSeconds !== undefined) {+ await response.body?.cancel()+ await sleepFn(retryAfterSeconds * 1000)+ response = await fetchFn(url, init)+ }if (!response.ok) {throw new Error(`GFS output publish failed: HTTP ${response.status} ${await responseText(response)}`@@ -107,6 +122,16 @@ async function resolvePublishParent(return resourceId}+/** The Retry-After of a retryable agent-limiter 429, in seconds; undefined otherwise. */+function agentRetryAfterSeconds(response: Response): number | undefined {+ if (response.status !== 429) return undefined+ if (!RETRYABLE_SCOPES.has(response.headers.get('x-gfs-ratelimit-scope') ?? '')) return undefined+ const retryAfter = response.headers.get('retry-after') ?? ''+ if (!/^[1-9][0-9]?$/.test(retryAfter)) return undefined+ const seconds = Number(retryAfter)+ return seconds <= MAX_RETRY_AFTER_SECONDS ? seconds : undefined+}+function outputFileName(workflowName: string, workflowRunId: string | null): string {const suffix = workflowRunId || workflowName || 'workflow'return `workflow-output-${suffix}.json`