Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(deploy): answer a gateway connect-timeout 504 as control_plane_unavailable (#820)

Alfredo Lopez7bef054d71a5
diff --git a/deploy/base/control-plane/configmaps.yaml b/deploy/base/control-plane/configmaps.yaml
index c3740fe09..4ef42b1ec 100644
--- a/deploy/base/control-plane/configmaps.yaml
+++ b/deploy/base/control-plane/configmaps.yaml
@@ -156,8 +156,7 @@ data:
# processed the request, so the code means "no response", not "never
# received". A 502 control-api returns passes through, because
# `proxy_intercept_errors` is off. Only the redeem locations, whose
- # failure reaches the user, map it. A 504 is not mapped: control-api
- # may be alive and slow.
+ # failure reaches the user, map it.
location @control_plane_unavailable {
default_type application/json;
add_header Cache-Control "no-store, private" always;
@@ -166,6 +165,24 @@ data:
return 503 '{"error":"control_plane_unavailable"}';
}
+ # G1-12 (#820): nginx generates a 504 for a connect timeout and for a
+ # read or send timeout. During a control-api restart the Service has
+ # no endpoint and the SYN is dropped, so the connect times out. nginx
+ # sets $upstream_connect_time only once the connection is made
+ # (ngx_http_upstream_send_request), so "-" here means control-api
+ # never received the request; only then is it mapped. A read or send
+ # timeout keeps nginx's own 504: control-api may be alive and slow.
+ location @control_plane_connect_timeout {
+ default_type application/json;
+ if ($upstream_connect_time = "-") {
+ add_header Cache-Control "no-store, private" always;
+ add_header Pragma "no-cache" always;
+ add_header X-Content-Type-Options "nosniff" always;
+ return 503 '{"error":"control_plane_unavailable"}';
+ }
+ return 504;
+ }
+
location ~ ^/api/v1/rpc/access/users/[^/]+/mcp-servers$ {
limit_except GET {
deny all;
@@ -303,6 +320,10 @@ data:
deny all;
}
error_page 502 = @control_plane_unavailable;
+ error_page 504 = @control_plane_connect_timeout;
+ # Shorter than the proxy's 15 s redeem abort
+ # (CONTROL_API_REQUEST_TIMEOUT_MS), so the proxy sees this answer.
+ proxy_connect_timeout 5s;
proxy_pass http://control_api_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
@@ -340,6 +361,10 @@ data:
deny all;
}
error_page 502 = @control_plane_unavailable;
+ error_page 504 = @control_plane_connect_timeout;
+ # Shorter than the proxy's 15 s redeem abort
+ # (CONTROL_API_REQUEST_TIMEOUT_MS), so the proxy sees this answer.
+ proxy_connect_timeout 5s;
proxy_pass http://control_api_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
@@ -412,8 +437,7 @@ data:
# processed the request, so the code means "no response", not "never
# received". A 502 control-api returns passes through, because
# `proxy_intercept_errors` is off. Only the authorize location, whose
- # failure reaches the user, maps it. A 504 is not mapped: control-api
- # may be alive and slow.
+ # failure reaches the user, maps it.
location @control_plane_unavailable {
default_type application/json;
add_header Cache-Control "no-store, private" always;
@@ -422,6 +446,25 @@ data:
return 503 '{"error":"control_plane_unavailable"}';
}
+ # G1-12 (#820): nginx generates a 504 for a connect timeout (this
+ # server's proxy_connect_timeout) and for a read or send timeout.
+ # During a control-api restart the Service has no endpoint and the SYN
+ # is dropped, so the connect times out. nginx sets
+ # $upstream_connect_time only once the connection is made
+ # (ngx_http_upstream_send_request), so "-" here means control-api
+ # never received the request; only then is it mapped. A read or send
+ # timeout keeps nginx's own 504: control-api may be alive and slow.
+ location @control_plane_connect_timeout {
+ default_type application/json;
+ if ($upstream_connect_time = "-") {
+ add_header Cache-Control "no-store, private" always;
+ add_header Pragma "no-cache" always;
+ add_header X-Content-Type-Options "nosniff" always;
+ return 503 '{"error":"control_plane_unavailable"}';
+ }
+ return 504;
+ }
+
# Runtime gateway only; provisioner issuance bypasses this route.
location = /api/v1/workflow-approval-mediums/resolve {
@@ -844,6 +887,7 @@ data:
deny all;
}
error_page 502 = @control_plane_unavailable;
+ error_page 504 = @control_plane_connect_timeout;
proxy_pass http://control_api_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;