Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(self-hosted): function worker retries (#50618)

Luiz Felipe Machadoff80bb14991e
diff --git a/docker/volumes/functions/main/index.ts b/docker/volumes/functions/main/index.ts
index ee55cf0f887..7a2b16f8482 100644
--- a/docker/volumes/functions/main/index.ts
+++ b/docker/volumes/functions/main/index.ts
@@ -2,6 +2,8 @@ import * as jose from 'jsr:@panva/jose@6'
console.log('main function started')
+const MAX_WORKER_RETRIES = 3
+
const JWT_SECRET = Deno.env.get('JWT_SECRET')
const SUPABASE_JWKS = parseJwks(Deno.env.get('SUPABASE_JWKS'))
const LOCAL_JWKS = SUPABASE_JWKS ? jose.createLocalJWKSet(SUPABASE_JWKS) : null
@@ -354,23 +356,41 @@ Deno.serve(async (req: Request) => {
const envVarsObj = { ...Deno.env.toObject(), SUPABASE_FUNCTION_SLUG: service_name }
const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]])
- try {
- const worker = await EdgeRuntime.userWorkers.create({
- servicePath,
- memoryLimitMb,
- workerTimeoutMs,
- context: { supervisor: { requestAbsentTimeoutMs } },
- noModuleCache,
- importMapPath,
- envVars,
- })
- // Gateway-minted internal JWT is for this router only; never expose it to user functions.
- const userReq = new Request(req)
- userReq.headers.delete('sb-api-key')
- EdgeRuntime.applySupabaseTag(req, userReq)
- return handleWorkerResponse(await worker.fetch(userReq))
- } catch (e) {
- console.error(e)
- return getFunctionErrorResponse(resolveRuntimeError(e))
+ const callWorker = async (req: Request, retriesLeft = MAX_WORKER_RETRIES): Promise<Response> => {
+ // Preserve the body before fetch() can consume it, even on a failed attempt.
+ // Must run before `new Request(req)` below, which takes over the body.
+ // The unread retry branch can buffer the entire body in main-worker memory,
+ // even when the first attempt succeeds.
+ const retryReq = retriesLeft > 0 ? req.clone() : null
+
+ try {
+ const worker = await EdgeRuntime.userWorkers.create({
+ servicePath,
+ memoryLimitMb,
+ workerTimeoutMs,
+ context: { supervisor: { requestAbsentTimeoutMs } },
+ noModuleCache,
+ importMapPath,
+ envVars,
+ })
+ // Gateway-minted internal JWT is for this router only; never expose it to user functions.
+ const userReq = new Request(req)
+ userReq.headers.delete('sb-api-key')
+ EdgeRuntime.applySupabaseTag(req, userReq)
+ return handleWorkerResponse(await worker.fetch(userReq))
+ } catch (e) {
+ // Retirement rejects before dispatch, so user code has not run yet.
+ if (e instanceof Deno.errors.WorkerAlreadyRetired && retryReq) {
+ console.warn(`${service_name}: worker retired before dispatch; retrying (${retriesLeft} left)`)
+ // Request.clone() does not copy the tag that connects streaming to the client.
+ EdgeRuntime.applySupabaseTag(req, retryReq)
+ return await callWorker(retryReq, retriesLeft - 1)
+ }
+
+ console.error(e)
+ return getFunctionErrorResponse(resolveRuntimeError(e))
+ }
}
+
+ return await callWorker(req)
})
diff --git a/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl b/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl
index f7dd77b5dad..91419f269f6 100644
--- a/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl
+++ b/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl
@@ -83,6 +83,9 @@ server {
location /functions {
proxy_pass http://api_gw_upstream;
+
+ # Outlast the runtime's 150s request idle timeout, matching the API gateway.
+ proxy_read_timeout 160s;
}
location /mcp {