Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(bedrock-runtime): return errors when upstream error bodies stall (#5527)

david lee4204ee2ea5b1
diff --git a/docs/services/bedrock-runtime.md b/docs/services/bedrock-runtime.md
index 4e6a7ff6b0..5e7990cc80 100644
--- a/docs/services/bedrock-runtime.md
+++ b/docs/services/bedrock-runtime.md
@@ -38,6 +38,7 @@ InvokeModel bodies are passed through as opaque bytes; neither backend parses re
| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_DEFAULT_MODEL` | _(none)_ | Fallback OpenAI-side model id when no mapping matches and passthrough is disabled |
| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_MODEL_MAPPING` | _(none)_ | Comma-separated `bedrockModelId=openaiModelId` pairs, e.g. `anthropic.claude-3-sonnet-20240229-v1:0=claude-3-sonnet` |
| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_PASSTHROUGH` | `false` | When true, forward the raw Bedrock model id as-is if no mapping matches |
+| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_ERROR_BODY_TIMEOUT_MILLIS` | `1000` | How long to collect an upstream error body before returning the error without its details. Must be positive. |
## Proxy Backend
@@ -60,7 +61,10 @@ that stall between tokens. The model server controls how it reacts to that signa
Operational HTTP-error warnings retain the upstream status code and omit the
backend response body. A connected caller receives `ModelErrorException` with a
-bounded backend error description. Streaming error-body reads are limited to 8 KiB.
+bounded backend error description. Error-body reads, for both `Converse` and
+`ConverseStream`, are limited to 8 KiB with a one-second collection deadline
+(`FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_ERROR_BODY_TIMEOUT_MILLIS`). If the body
+stalls, the proxy closes it and returns the known model error without body details.
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.java b/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.java
new file mode 100644
index 0000000000..cd0f24e7dd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.java
@@ -0,0 +1,28 @@
+package io.github.hectorvent.floci.config;
+
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+
+@ApplicationScoped
+public class BedrockRuntimeStartupValidator {
+
+ private final EmulatorConfig config;
+
+ @Inject
+ public BedrockRuntimeStartupValidator(EmulatorConfig config) {
+ this.config = config;
+ }
+
+ void onStart(@Observes StartupEvent ignored) {
+ validateErrorBodyTimeoutMillis(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis());
+ }
+
+ public static void validateErrorBodyTimeoutMillis(int value) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(
+ "floci.services.bedrock-runtime.proxy.error-body-timeout-millis must be positive");
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index a256153c47..49d1fc9982 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -2375,6 +2375,14 @@ public interface EmulatorConfig {
*/
@WithDefault("60")
int requestTimeoutSeconds();
+
+ /**
+ * How long to collect an upstream error response body before returning the
+ * known model error without its details. Bounds a backend that sends an error
+ * status and then stalls, since the request timeout stops at the headers.
+ */
+ @WithDefault("1000")
+ int errorBodyTimeoutMillis();
}
interface TextractServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
index 131e683290..c82dc142c7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
@@ -51,6 +52,7 @@ public class ProxyBackend implements BedrockBackend {
// Config is immutable for the process's lifetime, so the mapping string is parsed
// once here rather than on every Converse request.
private final Map<String, String> modelMapping;
+ private final Duration errorBodyReadTimeout;
private final HttpClient httpClient = HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
@@ -63,6 +65,7 @@ public class ProxyBackend implements BedrockBackend {
this.config = config;
this.currentRequest = currentRequest;
this.modelMapping = parseModelMapping(config.services().bedrockRuntime().proxy().modelMapping().orElse(""));
+ this.errorBodyReadTimeout = Duration.ofMillis(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis());
}
@Override
@@ -73,16 +76,21 @@ public class ProxyBackend implements BedrockBackend {
HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);
long start = System.nanoTime();
- HttpResponse<String> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofString());
- long latencyMs = (System.nanoTime() - start) / 1_000_000;
-
- if (response.statusCode() >= 300) {
- throw failedRequest(response.statusCode(), response.body());
+ UpstreamCall call = send(modelId, request, "backend", "request");
+ String responseBody;
+ try (InputStream body = call.response().body()) {
+ responseBody = new String(body.readAllBytes(), StandardCharsets.UTF_8);
+ } catch (IOException failure) {
+ LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);
+ throw new AwsException("ModelErrorException", "Proxy backend unavailable or disconnected.", 424);
+ } finally {
+ call.finish();
}
+ long latencyMs = (System.nanoTime() - start) / 1_000_000;
JsonNode openAiResponse;
try {
- openAiResponse = objectMapper.readTree(response.body());
+ openAiResponse = objectMapper.readTree(responseBody);
} catch (Exception e) {
throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);
}
@@ -110,62 +118,15 @@ public class ProxyBackend implements BedrockBackend {
// BedrockOpenAiTranslator.streamBedrockEvents).
HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);
long startNanos = System.nanoTime();
- // A stalled upstream must be closed on caller disconnect without waiting for another token.
- RoutingContext context = currentRequest.getCurrent();
- CompletableFuture<HttpResponse<InputStream>> pending = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream());
- AtomicBoolean complete = new AtomicBoolean();
- Thread monitor = monitorCancellation(context.response()::closed, complete, () -> {
- pending.thenAccept(response -> closeBody(response.body()));
- pending.cancel(true);
- });
- HttpResponse<InputStream> response;
- try {
- response = pending.get();
- if (response.statusCode() >= 300) {
- String errorBody;
- try {
- errorBody = new String(response.body().readNBytes(8192), StandardCharsets.UTF_8);
- } catch (IOException unavailable) {
- LOG.warnv("Failed to read Bedrock proxy error response: modelId={0}, status={1}",
- modelId, response.statusCode());
- errorBody = "";
- } finally {
- closeBody(response.body());
- }
- throw failedRequest(response.statusCode(), errorBody);
- }
- } catch (InterruptedException interrupted) {
- Thread.currentThread().interrupt();
- pending.thenAccept(completed -> closeBody(completed.body()));
- pending.cancel(true);
- complete.set(true);
- monitor.interrupt();
- throw new AwsException("ModelErrorException", "Proxy stream interrupted.", 424);
- } catch (ExecutionException | CancellationException failure) {
- complete.set(true);
- monitor.interrupt();
- if (failure.getCause() instanceof HttpTimeoutException) {
- LOG.warnv("Bedrock proxy backend timed out: modelId={0}", modelId);
- throw new AwsException("ModelTimeoutException", "Proxy stream timed out.", 408);
- }
- if (failure instanceof ExecutionException) {
- LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);
- }
- throw new AwsException("ModelErrorException", "Proxy stream unavailable or disconnected.", 424);
- } catch (RuntimeException failure) {
- complete.set(true);
- monitor.interrupt();
- throw failure;
- }
+ UpstreamCall call = send(modelId, request, "stream", "stream");
return output -> {
- try (InputStream body = response.body();
+ try (InputStream body = call.response().body();
Stream<String> lines = new BufferedReader(new InputStreamReader(body, StandardCharsets.UTF_8)).lines()) {
BedrockOpenAiTranslator.streamBedrockEvents(objectMapper, lines, output, startNanos);
} catch (IOException failure) {
throw new UncheckedIOException(failure);
} finally {
- complete.set(true);
- monitor.interrupt();
+ call.finish();
}
};
}
@@ -184,27 +145,71 @@ public class ProxyBackend implements BedrockBackend {
"Proxy backend returned HTTP %d: %s".formatted(response, truncate(errorBody, 512)), 424);
}
- private <T> HttpResponse<T> invoke(String modelId, HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler) {
+ /**
+ * Sends the request and returns once a successful status arrives, leaving the body to the
+ * caller. The cancellation monitor keeps running until {@link UpstreamCall#finish()}, so a
+ * caller disconnect still closes the upstream while its body is read. An error status is
+ * thrown here, after a read of its body that a timer cuts off at the deadline: the request
+ * timeout stops at the headers, so nothing else bounds a backend that stalls mid-error-body.
+ */
+ private UpstreamCall send(String modelId, HttpRequest request, String subject, String interruptedSubject) {
+ // A stalled upstream must be closed on caller disconnect without waiting for another token.
RoutingContext context = currentRequest.getCurrent();
- CompletableFuture<HttpResponse<T>> pending = httpClient.sendAsync(request, bodyHandler);
+ CompletableFuture<HttpResponse<InputStream>> pending = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream());
AtomicBoolean complete = new AtomicBoolean();
- Thread monitor = monitorCancellation(context.response()::closed, complete, () -> pending.cancel(true));
+ Thread monitor = monitorCancellation(context.response()::closed, complete, () -> {
+ pending.thenAccept(response -> closeBody(response.body()));
+ pending.cancel(true);
+ });
+ UpstreamCall call;
try {
- return pending.get();
+ call = new UpstreamCall(pending.get(), complete, monitor);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
+ pending.thenAccept(completed -> closeBody(completed.body()));
pending.cancel(true);
- throw new AwsException("ModelErrorException", "Proxy request interrupted.", 424);
+ complete.set(true);
+ monitor.interrupt();
+ throw new AwsException("ModelErrorException", "Proxy " + interruptedSubject + " interrupted.", 424);
} catch (ExecutionException | CancellationException failure) {
+ complete.set(true);
+ monitor.interrupt();
if (failure.getCause() instanceof HttpTimeoutException) {
LOG.warnv("Bedrock proxy backend timed out: modelId={0}", modelId);
- throw new AwsException("ModelTimeoutException", "Proxy backend timed out.", 408);
+ throw new AwsException("ModelTimeoutException", "Proxy " + subject + " timed out.", 408);
}
if (failure instanceof ExecutionException) {
LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);
}
- throw new AwsException("ModelErrorException", "Proxy backend unavailable or disconnected.", 424);
+ throw new AwsException("ModelErrorException", "Proxy " + subject + " unavailable or disconnected.", 424);
+ }
+ HttpResponse<InputStream> response = call.response();
+ if (response.statusCode() < 300) {
+ return call;
+ }
+ // Collecting error details must not delay a known upstream failure indefinitely.
+ CompletableFuture.delayedExecutor(errorBodyReadTimeout.toMillis(), TimeUnit.MILLISECONDS)
+ .execute(() -> {
+ if (!complete.get()) {
+ closeBody(response.body());
+ }
+ });
+ String errorBody;
+ try {
+ errorBody = new String(response.body().readNBytes(8192), StandardCharsets.UTF_8);
+ } catch (IOException unavailable) {
+ LOG.warnv("Failed to read Bedrock proxy error response: modelId={0}, status={1}",
+ modelId, response.statusCode());
+ errorBody = "";
} finally {
+ closeBody(response.body());
+ call.finish();
+ }
+ throw failedRequest(response.statusCode(), errorBody);
+ }
+
+ private record UpstreamCall(HttpResponse<InputStream> response, AtomicBoolean complete, Thread monitor) {
+ void finish() {
complete.set(true);
monitor.interrupt();
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 740c75237d..933ace86b5 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -743,6 +743,7 @@ floci:
backend: stub
proxy:
passthrough: false
+ error-body-timeout-millis: 1000
# url: http://localhost:11434/v1
# api-key: sk-...
# default-model: llama3
diff --git a/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.java b/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.java
new file mode 100644
index 0000000000..a53ffa055a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.java
@@ -0,0 +1,38 @@
+package io.github.hectorvent.floci.config;
+
+import io.quarkus.runtime.StartupEvent;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class BedrockRuntimeStartupValidatorTest {
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, -1, Integer.MIN_VALUE})
+ void nonPositiveErrorBodyTimeoutIsRejectedByStartupObserver(int value) {
+ BedrockRuntimeStartupValidator validator = validator(value);
+ IllegalArgumentException error = assertThrows(IllegalArgumentException.class,
+ () -> validator.onStart(new StartupEvent()));
+ assertEquals("floci.services.bedrock-runtime.proxy.error-body-timeout-millis must be positive",
+ error.getMessage());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 10, 1000, Integer.MAX_VALUE})
+ void positiveErrorBodyTimeoutIsAcceptedByStartupObserver(int value) {
+ BedrockRuntimeStartupValidator validator = validator(value);
+ assertDoesNotThrow(() -> validator.onStart(new StartupEvent()));
+ }
+
+ private static BedrockRuntimeStartupValidator validator(int value) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis()).thenReturn(value);
+ return new BedrockRuntimeStartupValidator(config);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java
index 9c1bebd434..89325deaef 100644
--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java
@@ -7,17 +7,11 @@ import io.quarkus.test.junit.TestProfile;
import org.junit.jupiter.api.Test;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.ServerSocket;
import java.net.Socket;
-import java.net.SocketException;
import java.net.URI;
import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
-import java.time.Duration;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static io.github.hectorvent.floci.testing.BedrockProxyBackendProfile.PORT;
@@ -42,63 +36,18 @@ class BedrockProxyCancellationIntegrationTest {
@Test
void disconnectClosesUpstreamWithoutAnotherToken() throws Exception {
- CountDownLatch closed = new CountDownLatch(1);
- try (ServerSocket server = new ServerSocket(PORT)) {
- server.setSoTimeout(10000);
- Thread upstream = Thread.ofVirtual().start(() -> {
- try (Socket socket = server.accept()) {
- socket.setSoTimeout(10000);
- InputStream input = socket.getInputStream();
- StringBuilder header = new StringBuilder();
- while (!header.toString().endsWith("\r\n\r\n")) {
- int next = input.read();
- if (next < 0 || header.length() > 65536) {
- return;
- }
- header.append((char) next);
- }
- int bytes = 0;
- for (String line : header.toString().split("\r\n")) {
- if (line.toLowerCase().startsWith("content-length:")) {
- bytes = Integer.parseInt(line.substring(line.indexOf(':') + 1).trim());
- }
- }
- assertEquals(bytes, input.readNBytes(bytes).length);
- OutputStream output = socket.getOutputStream();
- output.write("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n".getBytes(StandardCharsets.US_ASCII));
- byte[] chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n".getBytes(StandardCharsets.UTF_8);
- output.write((Integer.toHexString(chunk.length) + "\r\n").getBytes(StandardCharsets.US_ASCII));
- output.write(chunk);
- output.write("\r\n".getBytes(StandardCharsets.US_ASCII));
- output.flush();
- // No more output: disconnect must close the transport independently of token delivery.
- try {
- if (input.read() == -1) {
- closed.countDown();
- }
- } catch (SocketException disconnected) {
- closed.countDown();
- }
- } catch (Exception expected) {
- // The latch remains unset so a backend setup failure cannot pass the assertion.
- }
- });
- try (HttpClient client = HttpClient.newHttpClient()) {
- HttpRequest request = HttpRequest.newBuilder(base.resolve("/model/fixture/converse-stream"))
- .timeout(Duration.ofSeconds(10)).header("Content-Type", "application/json")
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")
- .POST(HttpRequest.BodyPublishers.ofString("{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}"))
- .build();
- HttpResponse<InputStream> response = client.send(request, HttpResponse.BodyHandlers.ofInputStream());
- assertEquals(200, response.statusCode());
- try (InputStream body = response.body()) {
- assertTrue(body.read() >= 0);
- }
- assertTrue(closed.await(5, TimeUnit.SECONDS), "Client disconnect must close the stalled upstream");
- } finally {
- upstream.interrupt();
- upstream.join(1000);
+ String chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n";
+ String firstToken = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n"
+ + Integer.toHexString(chunk.getBytes(StandardCharsets.UTF_8).length) + "\r\n" + chunk + "\r\n";
+ try (StallingUpstream upstream = new StallingUpstream(PORT, firstToken);
+ HttpClient client = HttpClient.newHttpClient()) {
+ HttpResponse<InputStream> response = client.send(ConverseRequests.to(base, "converse-stream"),
+ HttpResponse.BodyHandlers.ofInputStream());
+ assertEquals(200, response.statusCode());
+ try (InputStream body = response.body()) {
+ assertTrue(body.read() >= 0);
}
+ assertTrue(upstream.awaitClosed(5, TimeUnit.SECONDS), "Client disconnect must close the stalled upstream");
}
}
@@ -118,64 +67,21 @@ class BedrockProxyCancellationIntegrationTest {
}
private void disconnectWhileUpstreamStalls(String route, boolean partialResponse) throws Exception {
- CountDownLatch accepted = new CountDownLatch(1);
- CountDownLatch closed = new CountDownLatch(1);
- try (ServerSocket server = new ServerSocket(PORT)) {
- server.setSoTimeout(10000);
- Thread upstream = Thread.ofVirtual().start(() -> {
- try (Socket socket = server.accept()) {
- socket.setSoTimeout(10000);
- InputStream input = socket.getInputStream();
- StringBuilder header = new StringBuilder();
- while (!header.toString().endsWith("\r\n\r\n")) {
- int next = input.read();
- if (next < 0 || header.length() > 65536) {
- return;
- }
- header.append((char) next);
- }
- int bytes = 0;
- for (String line : header.toString().split("\r\n")) {
- if (line.toLowerCase().startsWith("content-length:")) {
- bytes = Integer.parseInt(line.substring(line.indexOf(':') + 1).trim());
- }
- }
- if (input.readNBytes(bytes).length != bytes) {
- return;
- }
- if (partialResponse) {
- socket.getOutputStream().write(("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
- + "Content-Length: 1000\r\n\r\n{").getBytes(StandardCharsets.US_ASCII));
- socket.getOutputStream().flush();
- }
- accepted.countDown();
- // The upstream stalls before completing its response, so cancellation must close it.
- try {
- if (input.read() == -1) {
- closed.countDown();
- }
- } catch (SocketException disconnected) {
- closed.countDown();
- }
- } catch (Exception expected) {
- // An unset latch exposes setup and transport failures to the test thread.
- }
- });
- try (Socket frontend = new Socket(base.getHost(), base.getPort())) {
- String body = "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}";
- String request = "POST /model/fixture/" + route + " HTTP/1.1\r\nHost: localhost\r\n"
- + "Content-Type: application/json\r\n"
- + "Authorization: AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request\r\n"
- + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;
- frontend.getOutputStream().write(request.getBytes(StandardCharsets.UTF_8));
- frontend.getOutputStream().flush();
- assertTrue(accepted.await(5, TimeUnit.SECONDS), "Upstream must receive the request before disconnect");
- frontend.close();
- assertTrue(closed.await(5, TimeUnit.SECONDS), "Disconnect must close the stalled upstream request");
- } finally {
- upstream.interrupt();
- upstream.join(1000);
- }
+ String prefix = partialResponse
+ ? "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n{"
+ : "";
+ try (StallingUpstream upstream = new StallingUpstream(PORT, prefix);
+ Socket frontend = new Socket(base.getHost(), base.getPort())) {
+ String body = ConverseRequests.BODY;
+ String request = "POST /model/fixture/" + route + " HTTP/1.1\r\nHost: localhost\r\n"
+ + "Content-Type: application/json\r\n"
+ + "Authorization: " + ConverseRequests.AUTHORIZATION + "\r\n"
+ + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;
+ frontend.getOutputStream().write(request.getBytes(StandardCharsets.UTF_8));
+ frontend.getOutputStream().flush();
+ assertTrue(upstream.awaitResponded(5, TimeUnit.SECONDS), "Upstream must receive the request before disconnect");
+ frontend.close();
+ assertTrue(upstream.awaitClosed(5, TimeUnit.SECONDS), "Disconnect must close the stalled upstream request");
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java
index 20cab6932d..54fcdac013 100644
--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java
@@ -1,26 +1,21 @@
package io.github.hectorvent.floci.services.bedrockruntime;
-import com.sun.net.httpserver.HttpServer;
import io.github.hectorvent.floci.services.bedrockruntime.backend.ProxyBackend;
import io.github.hectorvent.floci.testing.BedrockProxyBackendProfile;
import io.github.hectorvent.floci.testutil.LogCapture;
import io.quarkus.test.common.http.TestHTTPResource;
import io.quarkus.test.junit.QuarkusTest;
import io.quarkus.test.junit.TestProfile;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
-import java.io.IOException;
-import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
-import java.time.Duration;
import java.util.List;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.LogRecord;
@@ -30,6 +25,7 @@ import static io.github.hectorvent.floci.testing.BedrockProxyBackendProfile.PORT
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -51,25 +47,17 @@ class BedrockProxyLoggingIntegrationTest {
@ParameterizedTest
@ValueSource(strings = {"converse", "converse-stream"})
- void upstreamErrorIsLoggedWithoutResponseBody(String operation) throws IOException {
+ void upstreamErrorIsLoggedWithoutResponseBody(String operation) throws Exception {
String marker = "synthetic-private-upstream-content";
- HttpServer backend = HttpServer.create(new InetSocketAddress("127.0.0.1", PORT), 0);
- backend.createContext("/v1/chat/completions", exchange -> {
- exchange.getRequestBody().readAllBytes();
- byte[] body = ("{\"error\":\"" + marker + "\"}").getBytes(StandardCharsets.UTF_8);
- exchange.getResponseHeaders().add("Content-Type", "application/json");
- exchange.sendResponseHeaders(503, body.length);
- try (exchange) {
- exchange.getResponseBody().write(body);
- }
- });
- backend.start();
- try {
+ String body = "{\"error\":\"" + marker + "\"}";
+ String completeError = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
+ + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;
+ try (StallingUpstream upstream = new StallingUpstream(PORT, completeError)) {
List<LogRecord> records = LogCapture.capture(ProxyBackend.class, () ->
given()
.contentType("application/json")
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")
- .body("{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}")
+ .header("Authorization", ConverseRequests.AUTHORIZATION)
+ .body(ConverseRequests.BODY)
.when()
.post("/model/fixture/" + operation)
.then()
@@ -82,45 +70,50 @@ class BedrockProxyLoggingIntegrationTest {
&& formatter.format(record).contains("HTTP 503")), "The upstream status must remain diagnosable");
assertFalse(records.stream().anyMatch(record -> formatter.format(record).contains(marker)),
"Operational logs must not include upstream response bodies");
- } finally {
- backend.stop(0);
}
}
- @Test
- void streamingErrorBodyReadIsBoundedWithoutWaitingForEof() throws Exception {
- CountDownLatch release = new CountDownLatch(1);
- HttpServer backend = HttpServer.create(new InetSocketAddress("127.0.0.1", PORT), 0);
- backend.createContext("/v1/chat/completions", exchange -> {
- exchange.getRequestBody().readAllBytes();
- exchange.sendResponseHeaders(503, 32768);
- try (exchange) {
- exchange.getResponseBody().write("x".repeat(8192).getBytes(StandardCharsets.UTF_8));
- exchange.getResponseBody().flush();
- try {
- release.await(10, TimeUnit.SECONDS);
- } catch (InterruptedException interrupted) {
- Thread.currentThread().interrupt();
- }
- }
- });
- backend.start();
- try (HttpClient client = HttpClient.newHttpClient()) {
- HttpRequest request = HttpRequest.newBuilder(base.resolve("/model/fixture/converse-stream"))
- .timeout(Duration.ofSeconds(5))
- .header("Content-Type", "application/json")
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")
- .POST(HttpRequest.BodyPublishers.ofString(
- "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}"))
- .build();
- HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+ @ParameterizedTest
+ @ValueSource(strings = {"converse", "converse-stream"})
+ void errorBodyReadStopsAtTheCapWithoutWaitingForEof(String operation) throws Exception {
+ String cappedError = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: 32768\r\n\r\n"
+ + "x".repeat(8192);
+ try (StallingUpstream upstream = new StallingUpstream(PORT, cappedError);
+ HttpClient client = HttpClient.newHttpClient()) {
+ HttpResponse<String> response = client.send(ConverseRequests.to(base, operation),
+ HttpResponse.BodyHandlers.ofString());
assertEquals(424, response.statusCode());
assertTrue(response.body().contains("ModelErrorException"));
assertTrue(response.body().contains("HTTP 503"));
+ // Only a read that ends at the cap carries the body; the deadline cut would return none of it.
+ assertTrue(response.body().contains("x".repeat(512)), "The error read must end at the 8 KiB cap");
assertTrue(response.body().length() < 1024, "Caller error descriptions must remain bounded");
- } finally {
- release.countDown();
- backend.stop(0);
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"converse, 0", "converse, 1", "converse, 50",
+ "converse-stream, 0", "converse-stream, 1", "converse-stream, 50"})
+ void stalledErrorBodyHasDeadlineAndClosesUpstream(String operation, int lines) throws Exception {
+ String marker = "synthetic-private-upstream-content";
+ String errorPrefix = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: 32768\r\n\r\n"
+ + (marker + "\n").repeat(lines);
+ try (StallingUpstream upstream = new StallingUpstream(PORT, errorPrefix);
+ HttpClient client = HttpClient.newHttpClient()) {
+ HttpRequest request = ConverseRequests.to(base, operation);
+ List<LogRecord> records = LogCapture.capture(ProxyBackend.class, () -> {
+ HttpResponse<String> response = assertDoesNotThrow(
+ () -> client.send(request, HttpResponse.BodyHandlers.ofString()));
+ assertEquals(424, response.statusCode());
+ assertTrue(response.body().contains("ModelErrorException"));
+ assertTrue(response.body().contains("HTTP 503"));
+ assertTrue(response.body().length() < 1024);
+ assertTrue(assertDoesNotThrow(() -> upstream.awaitClosed(1, TimeUnit.SECONDS)),
+ "The proxy must close the stalled upstream body");
+ });
+ SimpleFormatter formatter = new SimpleFormatter();
+ assertFalse(records.stream().anyMatch(record -> formatter.format(record).contains(marker)));
+ assertTrue(records.stream().anyMatch(record -> formatter.format(record).contains("HTTP 503")));
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.java
new file mode 100644
index 0000000000..6d49647623
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.java
@@ -0,0 +1,27 @@
+package io.github.hectorvent.floci.services.bedrockruntime;
+
+import java.net.URI;
+import java.net.http.HttpRequest;
+import java.time.Duration;
+
+/**
+ * The Converse request the proxy tests send: a SigV4-shaped Authorization header and one user
+ * message, to the {@code fixture} model id, which reaches the stand-in through passthrough.
+ */
+final class ConverseRequests {
+
+ static final String AUTHORIZATION = "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request";
+ static final String BODY = "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}";
+
+ private ConverseRequests() {
+ }
+
+ static HttpRequest to(URI base, String operation) {
+ return HttpRequest.newBuilder(base.resolve("/model/fixture/" + operation))
+ .timeout(Duration.ofSeconds(10))
+ .header("Content-Type", "application/json")
+ .header("Authorization", AUTHORIZATION)
+ .POST(HttpRequest.BodyPublishers.ofString(BODY))
+ .build();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.java
new file mode 100644
index 0000000000..f33a348729
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.java
@@ -0,0 +1,86 @@
+package io.github.hectorvent.floci.services.bedrockruntime;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * A raw-socket stand-in for an OpenAI-compatible backend. It accepts one request, writes a
+ * fixed response prefix, then stalls, so a test can observe whether Floci closes the upstream.
+ */
+final class StallingUpstream implements AutoCloseable {
+
+ private final ServerSocket server;
+ private final Thread thread;
+ private final CountDownLatch responded = new CountDownLatch(1);
+ private final CompletableFuture<Boolean> closed = new CompletableFuture<>();
+
+ StallingUpstream(int port, String responsePrefix) throws IOException {
+ server = new ServerSocket(port);
+ server.setSoTimeout(10000);
+ thread = Thread.ofVirtual().start(() -> serve(responsePrefix));
+ }
+
+ private void serve(String responsePrefix) {
+ try (Socket socket = server.accept()) {
+ socket.setSoTimeout(10000);
+ InputStream input = socket.getInputStream();
+ StringBuilder headers = new StringBuilder();
+ while (!headers.toString().endsWith("\r\n\r\n")) {
+ int next = input.read();
+ if (next < 0 || headers.length() > 65536) {
+ throw new IOException("Incomplete fixture request headers");
+ }
+ headers.append((char) next);
+ }
+ int length = 0;
+ for (String header : headers.toString().split("\r\n")) {
+ if (header.toLowerCase().startsWith("content-length:")) {
+ length = Integer.parseInt(header.substring(header.indexOf(':') + 1).trim());
+ }
+ }
+ if (input.readNBytes(length).length != length) {
+ throw new IOException("Incomplete fixture request body");
+ }
+ OutputStream output = socket.getOutputStream();
+ output.write(responsePrefix.getBytes(StandardCharsets.UTF_8));
+ output.flush();
+ responded.countDown();
+ // Nothing more is sent, so only Floci closing its side ends this read.
+ try {
+ closed.complete(input.read() == -1);
+ } catch (SocketException disconnected) {
+ closed.complete(true);
+ }
+ } catch (Exception failure) {
+ closed.completeExceptionally(failure);
+ }
+ }
+
+ boolean awaitResponded(long timeout, TimeUnit unit) throws InterruptedException {
+ return responded.await(timeout, unit);
+ }
+
+ boolean awaitClosed(long timeout, TimeUnit unit) throws Exception {
+ try {
+ return closed.get(timeout, unit);
+ } catch (TimeoutException stillOpen) {
+ return false;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ thread.interrupt();
+ thread.join(1000);
+ server.close();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java b/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java
index b94d84133b..a8f4f5518f 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java
@@ -21,12 +21,16 @@ import java.util.Map;
* <li>{@code proxy.passthrough=true} forwards a model id with no mapping as it is.
* {@code BedrockProxyPassthroughIntegrationTest}, {@code BedrockProxyCancellationIntegrationTest}
* and {@code BedrockProxyLoggingIntegrationTest} rely on it.</li>
+ * <li>{@code proxy.error-body-timeout-millis=10} shortens the deadline for collecting an
+ * upstream error body, so the stalled-error-body tests do not each wait the one-second default.
+ * {@code BedrockProxyLoggingIntegrationTest} relies on it.</li>
* </ul>
*
* <p>Each override is inert for the classes that do not rely on it. Only the stand-in in
* {@code BedrockProxyIntegrationTest} reads the {@code Authorization} header. The other classes
* call model ids the mapping does not name, and every call in {@code BedrockProxyIntegrationTest}
- * names the mapped id, which the mapping resolves before passthrough is consulted.
+ * names the mapped id, which the mapping resolves before passthrough is consulted. Their stand-ins
+ * write each error body whole, so the shorter error-body deadline never fires for them.
*
* <p>{@code BedrockProxyTimeoutIntegrationTest} stays apart. Its one-second request timeout would
* also close an upstream that stalls before its headers, so the cancellation tests would pass
@@ -44,6 +48,7 @@ public class BedrockProxyBackendProfile implements QuarkusTestProfile {
"floci.services.bedrock-runtime.proxy.url", "http://127.0.0.1:" + PORT + "/v1",
"floci.services.bedrock-runtime.proxy.api-key", "test-key",
"floci.services.bedrock-runtime.proxy.model-mapping", MAPPED_MODEL_ID + "=claude-3-haiku",
- "floci.services.bedrock-runtime.proxy.passthrough", "true");
+ "floci.services.bedrock-runtime.proxy.passthrough", "true",
+ "floci.services.bedrock-runtime.proxy.error-body-timeout-millis", "10");
}
}