Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): end the GFS publish retry wait within a second of a cancel
Alfredo Lopez4e48d345a2a3
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 3da8ec4dd..f2f146703 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.375",+ "version": "0.1.376","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.375",+ "version": "0.1.376","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 76e8ba257..8ebe94a41 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.375",+ "version": "0.1.376","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.tsindex 61c8ff670..dca7ceb06 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts@@ -152,17 +152,43 @@ describe('publishWorkflowOutputsToGfs', () => {return { published, fetchFn, sleepFn }}- it('R1-L3: stops without the retry when the run is cancelled during the wait', async () => {+ it('R1-L3: stops the wait and the retry within one second of a cancel', async () => {const { published, fetchFn, sleepFn } = await publishWith([agentDenied('7')], {cancelDuringSleep: true,})await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)- // Witness: the wait was entered, so the check after it is what stopped the retry.- expect(sleepFn.mock.calls).toEqual([[7_000]])+ // Witness: the wait was entered. The cancel landed in its first second,+ // and the remaining six seconds were not waited.+ expect(sleepFn.mock.calls).toEqual([[1_000]])expect(fetchFn).toHaveBeenCalledTimes(1)})+ it('R1-L3: does not wait at all when the run is already cancelled at the 429', async () => {+ const denied = agentDenied('7')+ const cancel = vi.spyOn(denied.body!, 'cancel')+ const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => denied)+ const sleepFn = vi.fn(async (_ms: number) => {})+ const published = publishWorkflowOutputsToGfs(+ { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },+ { workflowName: 'daily-report' },+ { summarize: { ok: true } },+ {+ env: { ...env, CLERUM_GFSC_BASE_URL: 'http://reader.local' },+ fetchFn: fetchFn as unknown as typeof fetch,+ readFileFn: vi.fn(async () => 'runtime-access') as never,+ sleepFn,+ isCancelled: () => true,+ }+ )++ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)+ // Witness: the 429 was received and its body released.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(cancel).toHaveBeenCalledTimes(1)+ expect(sleepFn).not.toHaveBeenCalled()+ })+it('N5: retries a gfs:// resolve once on an agent_reads 429, then publishes', async () => {const { published, fetchFn, sleepFn } = await publishWith([@@ -174,7 +200,7 @@ describe('publishWorkflowOutputsToGfs', () => {)await expect(published).resolves.toBeUndefined()- expect(sleepFn.mock.calls).toEqual([[2_000]])+ expect(sleepFn.mock.calls).toEqual([[1_000], [1_000]])expect(fetchFn.mock.calls.map(([url]) => String(url))).toEqual(['http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder','http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',@@ -199,8 +225,8 @@ describe('publishWorkflowOutputsToGfs', () => {})await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)- // Witness: the resolve's wait was entered.- expect(sleepFn.mock.calls).toEqual([[2_000]])+ // Witness: the resolve's wait was entered; the cancel ended it after one second.+ expect(sleepFn.mock.calls).toEqual([[1_000]])expect(fetchFn).toHaveBeenCalledTimes(1)})@@ -211,7 +237,8 @@ describe('publishWorkflowOutputsToGfs', () => {])await expect(published).resolves.toBeUndefined()- expect(sleepFn.mock.calls).toEqual([[7_000]])+ // The 7 s Retry-After is waited in full, one second at a time.+ expect(sleepFn.mock.calls).toEqual(Array.from({ length: 7 }, () => [1_000]))expect(fetchFn).toHaveBeenCalledTimes(2)const [first, second] = fetchFn.mock.callsexpect(String(second![0])).toBe(String(first![0]))@@ -244,7 +271,7 @@ describe('publishWorkflowOutputsToGfs', () => {await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)expect(fetchFn).toHaveBeenCalledTimes(2)- expect(sleepFn.mock.calls).toEqual([[3_000]])+ expect(sleepFn.mock.calls).toEqual([[1_000], [1_000], [1_000]])})it('does not retry an upload-quota 429', async () => {diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.tsindex 2d335e2c8..0559ec8c3 100644--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts@@ -18,12 +18,12 @@ export interface GfsOutputPublisherDeps {fetchFn?: typeof fetchreadFileFn?: typeof readFilesleepFn?: (ms: number) => Promise<void>- /** True once the run has been cancelled; read after each retry wait. */+ /** True once the run has been cancelled; read at least once a second during a retry wait. */isCancelled?: () => boolean}/**- * The run was cancelled while the publisher waited out a 429's Retry-After, so+ * The run was cancelled after a 429, before or during the Retry-After wait, so* the retry was not sent. The caller reports the run as cancelled, not failed.*/export class GfsPublishCancelledError extends Error {@@ -49,6 +49,8 @@ const HEADER_SCHEME = 'Bearer'// is safe. Upload-quota 429s carry other scopes and are not retried.const RETRYABLE_SCOPES = new Set(['agent_reads', 'agent_writes'])const MAX_RETRY_AFTER_SECONDS = 60+// How often a retry wait checks for a cancel of the run.+const CANCEL_CHECK_INTERVAL_MS = 1000export async function publishWorkflowOutputsToGfs(spec: GfsPublishWorkflowSpec,@@ -141,8 +143,9 @@ async function resolvePublishParent(/*** Sends the request, and once more after an agent-limiter 429 that states a* delay of at most MAX_RETRY_AFTER_SECONDS. A second denial is returned for the- * caller to fail with its status. A cancel that arrives during the wait stops- * the retry: the run is over, and the wait can last up to a minute.+ * caller to fail with its status. The wait can last up to a minute, so it is+ * taken in CANCEL_CHECK_INTERVAL_MS steps and a cancel ends it, and the retry,+ * within one step.*/async function fetchWithAgentRetry(url: string,@@ -153,7 +156,13 @@ async function fetchWithAgentRetry(const retryAfterSeconds = agentRetryAfterSeconds(response)if (retryAfterSeconds === undefined) return responseawait response.body?.cancel()- await retry.sleepFn(retryAfterSeconds * 1000)+ let remainingMs = retryAfterSeconds * 1000+ while (remainingMs > 0) {+ if (retry.isCancelled()) throw new GfsPublishCancelledError()+ const stepMs = Math.min(CANCEL_CHECK_INTERVAL_MS, remainingMs)+ await retry.sleepFn(stepMs)+ remainingMs -= stepMs+ }if (retry.isCancelled()) throw new GfsPublishCancelledError()return retry.fetchFn(url, init)}