Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): end the GFS publish retry wait within a second of a cancel

Alfredo Lopez4e48d345a2a3
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 3da8ec4dd..f2f146703 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.375",
+ "version": "0.1.376",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.375",
+ "version": "0.1.376",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 76e8ba257..8ebe94a41 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.375",
+ "version": "0.1.376",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
index 61c8ff670..dca7ceb06 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.test.ts
@@ -152,17 +152,43 @@ describe('publishWorkflowOutputsToGfs', () => {
return { published, fetchFn, sleepFn }
}
- it('R1-L3: stops without the retry when the run is cancelled during the wait', async () => {
+ it('R1-L3: stops the wait and the retry within one second of a cancel', async () => {
const { published, fetchFn, sleepFn } = await publishWith([agentDenied('7')], {
cancelDuringSleep: true,
})
await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)
- // Witness: the wait was entered, so the check after it is what stopped the retry.
- expect(sleepFn.mock.calls).toEqual([[7_000]])
+ // Witness: the wait was entered. The cancel landed in its first second,
+ // and the remaining six seconds were not waited.
+ expect(sleepFn.mock.calls).toEqual([[1_000]])
expect(fetchFn).toHaveBeenCalledTimes(1)
})
+ it('R1-L3: does not wait at all when the run is already cancelled at the 429', async () => {
+ const denied = agentDenied('7')
+ const cancel = vi.spyOn(denied.body!, 'cancel')
+ const fetchFn = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => denied)
+ const sleepFn = vi.fn(async (_ms: number) => {})
+ const published = publishWorkflowOutputsToGfs(
+ { gfs: { publishTargets: [{ drive: 'main', target: PARENT_ID }] } },
+ { workflowName: 'daily-report' },
+ { summarize: { ok: true } },
+ {
+ env: { ...env, CLERUM_GFSC_BASE_URL: 'http://reader.local' },
+ fetchFn: fetchFn as unknown as typeof fetch,
+ readFileFn: vi.fn(async () => 'runtime-access') as never,
+ sleepFn,
+ isCancelled: () => true,
+ }
+ )
+
+ await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)
+ // Witness: the 429 was received and its body released.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(cancel).toHaveBeenCalledTimes(1)
+ expect(sleepFn).not.toHaveBeenCalled()
+ })
+
it('N5: retries a gfs:// resolve once on an agent_reads 429, then publishes', async () => {
const { published, fetchFn, sleepFn } = await publishWith(
[
@@ -174,7 +200,7 @@ describe('publishWorkflowOutputsToGfs', () => {
)
await expect(published).resolves.toBeUndefined()
- expect(sleepFn.mock.calls).toEqual([[2_000]])
+ expect(sleepFn.mock.calls).toEqual([[1_000], [1_000]])
expect(fetchFn.mock.calls.map(([url]) => String(url))).toEqual([
'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',
'http://reader.local/v1/resolve?uri=gfs%3A%2F%2Fmain%2Fsome-folder',
@@ -199,8 +225,8 @@ describe('publishWorkflowOutputsToGfs', () => {
})
await expect(published).rejects.toBeInstanceOf(GfsPublishCancelledError)
- // Witness: the resolve's wait was entered.
- expect(sleepFn.mock.calls).toEqual([[2_000]])
+ // Witness: the resolve's wait was entered; the cancel ended it after one second.
+ expect(sleepFn.mock.calls).toEqual([[1_000]])
expect(fetchFn).toHaveBeenCalledTimes(1)
})
@@ -211,7 +237,8 @@ describe('publishWorkflowOutputsToGfs', () => {
])
await expect(published).resolves.toBeUndefined()
- expect(sleepFn.mock.calls).toEqual([[7_000]])
+ // The 7 s Retry-After is waited in full, one second at a time.
+ expect(sleepFn.mock.calls).toEqual(Array.from({ length: 7 }, () => [1_000]))
expect(fetchFn).toHaveBeenCalledTimes(2)
const [first, second] = fetchFn.mock.calls
expect(String(second![0])).toBe(String(first![0]))
@@ -244,7 +271,7 @@ describe('publishWorkflowOutputsToGfs', () => {
await expect(published).rejects.toThrow(/GFS output publish failed: HTTP 429/)
expect(fetchFn).toHaveBeenCalledTimes(2)
- expect(sleepFn.mock.calls).toEqual([[3_000]])
+ expect(sleepFn.mock.calls).toEqual([[1_000], [1_000], [1_000]])
})
it('does not retry an upload-quota 429', async () => {
diff --git a/workflow-recipes/src/workflow/gfsOutputPublisher.ts b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
index 2d335e2c8..0559ec8c3 100644
--- a/workflow-recipes/src/workflow/gfsOutputPublisher.ts
+++ b/workflow-recipes/src/workflow/gfsOutputPublisher.ts
@@ -18,12 +18,12 @@ export interface GfsOutputPublisherDeps {
fetchFn?: typeof fetch
readFileFn?: typeof readFile
sleepFn?: (ms: number) => Promise<void>
- /** True once the run has been cancelled; read after each retry wait. */
+ /** True once the run has been cancelled; read at least once a second during a retry wait. */
isCancelled?: () => boolean
}
/**
- * The run was cancelled while the publisher waited out a 429's Retry-After, so
+ * The run was cancelled after a 429, before or during the Retry-After wait, so
* the retry was not sent. The caller reports the run as cancelled, not failed.
*/
export class GfsPublishCancelledError extends Error {
@@ -49,6 +49,8 @@ const HEADER_SCHEME = 'Bearer'
// is safe. Upload-quota 429s carry other scopes and are not retried.
const RETRYABLE_SCOPES = new Set(['agent_reads', 'agent_writes'])
const MAX_RETRY_AFTER_SECONDS = 60
+// How often a retry wait checks for a cancel of the run.
+const CANCEL_CHECK_INTERVAL_MS = 1000
export async function publishWorkflowOutputsToGfs(
spec: GfsPublishWorkflowSpec,
@@ -141,8 +143,9 @@ async function resolvePublishParent(
/**
* Sends the request, and once more after an agent-limiter 429 that states a
* delay of at most MAX_RETRY_AFTER_SECONDS. A second denial is returned for the
- * caller to fail with its status. A cancel that arrives during the wait stops
- * the retry: the run is over, and the wait can last up to a minute.
+ * caller to fail with its status. The wait can last up to a minute, so it is
+ * taken in CANCEL_CHECK_INTERVAL_MS steps and a cancel ends it, and the retry,
+ * within one step.
*/
async function fetchWithAgentRetry(
url: string,
@@ -153,7 +156,13 @@ async function fetchWithAgentRetry(
const retryAfterSeconds = agentRetryAfterSeconds(response)
if (retryAfterSeconds === undefined) return response
await response.body?.cancel()
- await retry.sleepFn(retryAfterSeconds * 1000)
+ let remainingMs = retryAfterSeconds * 1000
+ while (remainingMs > 0) {
+ if (retry.isCancelled()) throw new GfsPublishCancelledError()
+ const stepMs = Math.min(CANCEL_CHECK_INTERVAL_MS, remainingMs)
+ await retry.sleepFn(stepMs)
+ remainingMs -= stepMs
+ }
if (retry.isCancelled()) throw new GfsPublishCancelledError()
return retry.fetchFn(url, init)
}