Light, fluffy, and always free - The AWS Local Emulator alternative
fix(bedrock-runtime): return errors when upstream error bodies stall (#5527)
david lee4204ee2ea5b1
diff --git a/docs/services/bedrock-runtime.md b/docs/services/bedrock-runtime.mdindex 4e6a7ff6b0..5e7990cc80 100644--- a/docs/services/bedrock-runtime.md+++ b/docs/services/bedrock-runtime.md@@ -38,6 +38,7 @@ InvokeModel bodies are passed through as opaque bytes; neither backend parses re| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_DEFAULT_MODEL` | _(none)_ | Fallback OpenAI-side model id when no mapping matches and passthrough is disabled || `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_MODEL_MAPPING` | _(none)_ | Comma-separated `bedrockModelId=openaiModelId` pairs, e.g. `anthropic.claude-3-sonnet-20240229-v1:0=claude-3-sonnet` || `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_PASSTHROUGH` | `false` | When true, forward the raw Bedrock model id as-is if no mapping matches |+| `FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_ERROR_BODY_TIMEOUT_MILLIS` | `1000` | How long to collect an upstream error body before returning the error without its details. Must be positive. |## Proxy Backend@@ -60,7 +61,10 @@ that stall between tokens. The model server controls how it reacts to that signaOperational HTTP-error warnings retain the upstream status code and omit thebackend response body. A connected caller receives `ModelErrorException` with a-bounded backend error description. Streaming error-body reads are limited to 8 KiB.+bounded backend error description. Error-body reads, for both `Converse` and+`ConverseStream`, are limited to 8 KiB with a one-second collection deadline+(`FLOCI_SERVICES_BEDROCK_RUNTIME_PROXY_ERROR_BODY_TIMEOUT_MILLIS`). If the body+stalls, the proxy closes it and returns the known model error without body details.## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.java b/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.javanew file mode 100644index 0000000000..cd0f24e7dd--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidator.java@@ -0,0 +1,28 @@+package io.github.hectorvent.floci.config;++import io.quarkus.runtime.StartupEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;++@ApplicationScoped+public class BedrockRuntimeStartupValidator {++ private final EmulatorConfig config;++ @Inject+ public BedrockRuntimeStartupValidator(EmulatorConfig config) {+ this.config = config;+ }++ void onStart(@Observes StartupEvent ignored) {+ validateErrorBodyTimeoutMillis(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis());+ }++ public static void validateErrorBodyTimeoutMillis(int value) {+ if (value <= 0) {+ throw new IllegalArgumentException(+ "floci.services.bedrock-runtime.proxy.error-body-timeout-millis must be positive");+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex a256153c47..49d1fc9982 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -2375,6 +2375,14 @@ public interface EmulatorConfig {*/@WithDefault("60")int requestTimeoutSeconds();++ /**+ * How long to collect an upstream error response body before returning the+ * known model error without its details. Bounds a backend that sends an error+ * status and then stalls, since the request timeout stops at the headers.+ */+ @WithDefault("1000")+ int errorBodyTimeoutMillis();}interface TextractServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.javaindex 131e683290..c82dc142c7 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java@@ -29,6 +29,7 @@ import java.util.Map;import java.util.concurrent.CancellationException;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ExecutionException;+import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import java.util.function.BooleanSupplier;import java.util.function.Consumer;@@ -51,6 +52,7 @@ public class ProxyBackend implements BedrockBackend {// Config is immutable for the process's lifetime, so the mapping string is parsed// once here rather than on every Converse request.private final Map<String, String> modelMapping;+ private final Duration errorBodyReadTimeout;private final HttpClient httpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1)@@ -63,6 +65,7 @@ public class ProxyBackend implements BedrockBackend {this.config = config;this.currentRequest = currentRequest;this.modelMapping = parseModelMapping(config.services().bedrockRuntime().proxy().modelMapping().orElse(""));+ this.errorBodyReadTimeout = Duration.ofMillis(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis());}@Override@@ -73,16 +76,21 @@ public class ProxyBackend implements BedrockBackend {HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);long start = System.nanoTime();- HttpResponse<String> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofString());- long latencyMs = (System.nanoTime() - start) / 1_000_000;-- if (response.statusCode() >= 300) {- throw failedRequest(response.statusCode(), response.body());+ UpstreamCall call = send(modelId, request, "backend", "request");+ String responseBody;+ try (InputStream body = call.response().body()) {+ responseBody = new String(body.readAllBytes(), StandardCharsets.UTF_8);+ } catch (IOException failure) {+ LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);+ throw new AwsException("ModelErrorException", "Proxy backend unavailable or disconnected.", 424);+ } finally {+ call.finish();}+ long latencyMs = (System.nanoTime() - start) / 1_000_000;JsonNode openAiResponse;try {- openAiResponse = objectMapper.readTree(response.body());+ openAiResponse = objectMapper.readTree(responseBody);} catch (Exception e) {throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);}@@ -110,62 +118,15 @@ public class ProxyBackend implements BedrockBackend {// BedrockOpenAiTranslator.streamBedrockEvents).HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);long startNanos = System.nanoTime();- // A stalled upstream must be closed on caller disconnect without waiting for another token.- RoutingContext context = currentRequest.getCurrent();- CompletableFuture<HttpResponse<InputStream>> pending = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream());- AtomicBoolean complete = new AtomicBoolean();- Thread monitor = monitorCancellation(context.response()::closed, complete, () -> {- pending.thenAccept(response -> closeBody(response.body()));- pending.cancel(true);- });- HttpResponse<InputStream> response;- try {- response = pending.get();- if (response.statusCode() >= 300) {- String errorBody;- try {- errorBody = new String(response.body().readNBytes(8192), StandardCharsets.UTF_8);- } catch (IOException unavailable) {- LOG.warnv("Failed to read Bedrock proxy error response: modelId={0}, status={1}",- modelId, response.statusCode());- errorBody = "";- } finally {- closeBody(response.body());- }- throw failedRequest(response.statusCode(), errorBody);- }- } catch (InterruptedException interrupted) {- Thread.currentThread().interrupt();- pending.thenAccept(completed -> closeBody(completed.body()));- pending.cancel(true);- complete.set(true);- monitor.interrupt();- throw new AwsException("ModelErrorException", "Proxy stream interrupted.", 424);- } catch (ExecutionException | CancellationException failure) {- complete.set(true);- monitor.interrupt();- if (failure.getCause() instanceof HttpTimeoutException) {- LOG.warnv("Bedrock proxy backend timed out: modelId={0}", modelId);- throw new AwsException("ModelTimeoutException", "Proxy stream timed out.", 408);- }- if (failure instanceof ExecutionException) {- LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);- }- throw new AwsException("ModelErrorException", "Proxy stream unavailable or disconnected.", 424);- } catch (RuntimeException failure) {- complete.set(true);- monitor.interrupt();- throw failure;- }+ UpstreamCall call = send(modelId, request, "stream", "stream");return output -> {- try (InputStream body = response.body();+ try (InputStream body = call.response().body();Stream<String> lines = new BufferedReader(new InputStreamReader(body, StandardCharsets.UTF_8)).lines()) {BedrockOpenAiTranslator.streamBedrockEvents(objectMapper, lines, output, startNanos);} catch (IOException failure) {throw new UncheckedIOException(failure);} finally {- complete.set(true);- monitor.interrupt();+ call.finish();}};}@@ -184,27 +145,71 @@ public class ProxyBackend implements BedrockBackend {"Proxy backend returned HTTP %d: %s".formatted(response, truncate(errorBody, 512)), 424);}- private <T> HttpResponse<T> invoke(String modelId, HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler) {+ /**+ * Sends the request and returns once a successful status arrives, leaving the body to the+ * caller. The cancellation monitor keeps running until {@link UpstreamCall#finish()}, so a+ * caller disconnect still closes the upstream while its body is read. An error status is+ * thrown here, after a read of its body that a timer cuts off at the deadline: the request+ * timeout stops at the headers, so nothing else bounds a backend that stalls mid-error-body.+ */+ private UpstreamCall send(String modelId, HttpRequest request, String subject, String interruptedSubject) {+ // A stalled upstream must be closed on caller disconnect without waiting for another token.RoutingContext context = currentRequest.getCurrent();- CompletableFuture<HttpResponse<T>> pending = httpClient.sendAsync(request, bodyHandler);+ CompletableFuture<HttpResponse<InputStream>> pending = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream());AtomicBoolean complete = new AtomicBoolean();- Thread monitor = monitorCancellation(context.response()::closed, complete, () -> pending.cancel(true));+ Thread monitor = monitorCancellation(context.response()::closed, complete, () -> {+ pending.thenAccept(response -> closeBody(response.body()));+ pending.cancel(true);+ });+ UpstreamCall call;try {- return pending.get();+ call = new UpstreamCall(pending.get(), complete, monitor);} catch (InterruptedException interrupted) {Thread.currentThread().interrupt();+ pending.thenAccept(completed -> closeBody(completed.body()));pending.cancel(true);- throw new AwsException("ModelErrorException", "Proxy request interrupted.", 424);+ complete.set(true);+ monitor.interrupt();+ throw new AwsException("ModelErrorException", "Proxy " + interruptedSubject + " interrupted.", 424);} catch (ExecutionException | CancellationException failure) {+ complete.set(true);+ monitor.interrupt();if (failure.getCause() instanceof HttpTimeoutException) {LOG.warnv("Bedrock proxy backend timed out: modelId={0}", modelId);- throw new AwsException("ModelTimeoutException", "Proxy backend timed out.", 408);+ throw new AwsException("ModelTimeoutException", "Proxy " + subject + " timed out.", 408);}if (failure instanceof ExecutionException) {LOG.warnv("Bedrock proxy backend call failed: modelId={0}", modelId);}- throw new AwsException("ModelErrorException", "Proxy backend unavailable or disconnected.", 424);+ throw new AwsException("ModelErrorException", "Proxy " + subject + " unavailable or disconnected.", 424);+ }+ HttpResponse<InputStream> response = call.response();+ if (response.statusCode() < 300) {+ return call;+ }+ // Collecting error details must not delay a known upstream failure indefinitely.+ CompletableFuture.delayedExecutor(errorBodyReadTimeout.toMillis(), TimeUnit.MILLISECONDS)+ .execute(() -> {+ if (!complete.get()) {+ closeBody(response.body());+ }+ });+ String errorBody;+ try {+ errorBody = new String(response.body().readNBytes(8192), StandardCharsets.UTF_8);+ } catch (IOException unavailable) {+ LOG.warnv("Failed to read Bedrock proxy error response: modelId={0}, status={1}",+ modelId, response.statusCode());+ errorBody = "";} finally {+ closeBody(response.body());+ call.finish();+ }+ throw failedRequest(response.statusCode(), errorBody);+ }++ private record UpstreamCall(HttpResponse<InputStream> response, AtomicBoolean complete, Thread monitor) {+ void finish() {complete.set(true);monitor.interrupt();}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 740c75237d..933ace86b5 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -743,6 +743,7 @@ floci:backend: stubproxy:passthrough: false+ error-body-timeout-millis: 1000# url: http://localhost:11434/v1# api-key: sk-...# default-model: llama3diff --git a/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.java b/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.javanew file mode 100644index 0000000000..a53ffa055a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/config/BedrockRuntimeStartupValidatorTest.java@@ -0,0 +1,38 @@+package io.github.hectorvent.floci.config;++import io.quarkus.runtime.StartupEvent;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class BedrockRuntimeStartupValidatorTest {++ @ParameterizedTest+ @ValueSource(ints = {0, -1, Integer.MIN_VALUE})+ void nonPositiveErrorBodyTimeoutIsRejectedByStartupObserver(int value) {+ BedrockRuntimeStartupValidator validator = validator(value);+ IllegalArgumentException error = assertThrows(IllegalArgumentException.class,+ () -> validator.onStart(new StartupEvent()));+ assertEquals("floci.services.bedrock-runtime.proxy.error-body-timeout-millis must be positive",+ error.getMessage());+ }++ @ParameterizedTest+ @ValueSource(ints = {1, 10, 1000, Integer.MAX_VALUE})+ void positiveErrorBodyTimeoutIsAcceptedByStartupObserver(int value) {+ BedrockRuntimeStartupValidator validator = validator(value);+ assertDoesNotThrow(() -> validator.onStart(new StartupEvent()));+ }++ private static BedrockRuntimeStartupValidator validator(int value) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().bedrockRuntime().proxy().errorBodyTimeoutMillis()).thenReturn(value);+ return new BedrockRuntimeStartupValidator(config);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.javaindex 9c1bebd434..89325deaef 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyCancellationIntegrationTest.java@@ -7,17 +7,11 @@ import io.quarkus.test.junit.TestProfile;import org.junit.jupiter.api.Test;import java.io.InputStream;-import java.io.OutputStream;-import java.net.ServerSocket;import java.net.Socket;-import java.net.SocketException;import java.net.URI;import java.net.http.HttpClient;-import java.net.http.HttpRequest;import java.net.http.HttpResponse;import java.nio.charset.StandardCharsets;-import java.time.Duration;-import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import static io.github.hectorvent.floci.testing.BedrockProxyBackendProfile.PORT;@@ -42,63 +36,18 @@ class BedrockProxyCancellationIntegrationTest {@Testvoid disconnectClosesUpstreamWithoutAnotherToken() throws Exception {- CountDownLatch closed = new CountDownLatch(1);- try (ServerSocket server = new ServerSocket(PORT)) {- server.setSoTimeout(10000);- Thread upstream = Thread.ofVirtual().start(() -> {- try (Socket socket = server.accept()) {- socket.setSoTimeout(10000);- InputStream input = socket.getInputStream();- StringBuilder header = new StringBuilder();- while (!header.toString().endsWith("\r\n\r\n")) {- int next = input.read();- if (next < 0 || header.length() > 65536) {- return;- }- header.append((char) next);- }- int bytes = 0;- for (String line : header.toString().split("\r\n")) {- if (line.toLowerCase().startsWith("content-length:")) {- bytes = Integer.parseInt(line.substring(line.indexOf(':') + 1).trim());- }- }- assertEquals(bytes, input.readNBytes(bytes).length);- OutputStream output = socket.getOutputStream();- output.write("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n".getBytes(StandardCharsets.US_ASCII));- byte[] chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n".getBytes(StandardCharsets.UTF_8);- output.write((Integer.toHexString(chunk.length) + "\r\n").getBytes(StandardCharsets.US_ASCII));- output.write(chunk);- output.write("\r\n".getBytes(StandardCharsets.US_ASCII));- output.flush();- // No more output: disconnect must close the transport independently of token delivery.- try {- if (input.read() == -1) {- closed.countDown();- }- } catch (SocketException disconnected) {- closed.countDown();- }- } catch (Exception expected) {- // The latch remains unset so a backend setup failure cannot pass the assertion.- }- });- try (HttpClient client = HttpClient.newHttpClient()) {- HttpRequest request = HttpRequest.newBuilder(base.resolve("/model/fixture/converse-stream"))- .timeout(Duration.ofSeconds(10)).header("Content-Type", "application/json")- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")- .POST(HttpRequest.BodyPublishers.ofString("{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}"))- .build();- HttpResponse<InputStream> response = client.send(request, HttpResponse.BodyHandlers.ofInputStream());- assertEquals(200, response.statusCode());- try (InputStream body = response.body()) {- assertTrue(body.read() >= 0);- }- assertTrue(closed.await(5, TimeUnit.SECONDS), "Client disconnect must close the stalled upstream");- } finally {- upstream.interrupt();- upstream.join(1000);+ String chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n";+ String firstToken = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n"+ + Integer.toHexString(chunk.getBytes(StandardCharsets.UTF_8).length) + "\r\n" + chunk + "\r\n";+ try (StallingUpstream upstream = new StallingUpstream(PORT, firstToken);+ HttpClient client = HttpClient.newHttpClient()) {+ HttpResponse<InputStream> response = client.send(ConverseRequests.to(base, "converse-stream"),+ HttpResponse.BodyHandlers.ofInputStream());+ assertEquals(200, response.statusCode());+ try (InputStream body = response.body()) {+ assertTrue(body.read() >= 0);}+ assertTrue(upstream.awaitClosed(5, TimeUnit.SECONDS), "Client disconnect must close the stalled upstream");}}@@ -118,64 +67,21 @@ class BedrockProxyCancellationIntegrationTest {}private void disconnectWhileUpstreamStalls(String route, boolean partialResponse) throws Exception {- CountDownLatch accepted = new CountDownLatch(1);- CountDownLatch closed = new CountDownLatch(1);- try (ServerSocket server = new ServerSocket(PORT)) {- server.setSoTimeout(10000);- Thread upstream = Thread.ofVirtual().start(() -> {- try (Socket socket = server.accept()) {- socket.setSoTimeout(10000);- InputStream input = socket.getInputStream();- StringBuilder header = new StringBuilder();- while (!header.toString().endsWith("\r\n\r\n")) {- int next = input.read();- if (next < 0 || header.length() > 65536) {- return;- }- header.append((char) next);- }- int bytes = 0;- for (String line : header.toString().split("\r\n")) {- if (line.toLowerCase().startsWith("content-length:")) {- bytes = Integer.parseInt(line.substring(line.indexOf(':') + 1).trim());- }- }- if (input.readNBytes(bytes).length != bytes) {- return;- }- if (partialResponse) {- socket.getOutputStream().write(("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"- + "Content-Length: 1000\r\n\r\n{").getBytes(StandardCharsets.US_ASCII));- socket.getOutputStream().flush();- }- accepted.countDown();- // The upstream stalls before completing its response, so cancellation must close it.- try {- if (input.read() == -1) {- closed.countDown();- }- } catch (SocketException disconnected) {- closed.countDown();- }- } catch (Exception expected) {- // An unset latch exposes setup and transport failures to the test thread.- }- });- try (Socket frontend = new Socket(base.getHost(), base.getPort())) {- String body = "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}";- String request = "POST /model/fixture/" + route + " HTTP/1.1\r\nHost: localhost\r\n"- + "Content-Type: application/json\r\n"- + "Authorization: AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request\r\n"- + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;- frontend.getOutputStream().write(request.getBytes(StandardCharsets.UTF_8));- frontend.getOutputStream().flush();- assertTrue(accepted.await(5, TimeUnit.SECONDS), "Upstream must receive the request before disconnect");- frontend.close();- assertTrue(closed.await(5, TimeUnit.SECONDS), "Disconnect must close the stalled upstream request");- } finally {- upstream.interrupt();- upstream.join(1000);- }+ String prefix = partialResponse+ ? "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n{"+ : "";+ try (StallingUpstream upstream = new StallingUpstream(PORT, prefix);+ Socket frontend = new Socket(base.getHost(), base.getPort())) {+ String body = ConverseRequests.BODY;+ String request = "POST /model/fixture/" + route + " HTTP/1.1\r\nHost: localhost\r\n"+ + "Content-Type: application/json\r\n"+ + "Authorization: " + ConverseRequests.AUTHORIZATION + "\r\n"+ + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;+ frontend.getOutputStream().write(request.getBytes(StandardCharsets.UTF_8));+ frontend.getOutputStream().flush();+ assertTrue(upstream.awaitResponded(5, TimeUnit.SECONDS), "Upstream must receive the request before disconnect");+ frontend.close();+ assertTrue(upstream.awaitClosed(5, TimeUnit.SECONDS), "Disconnect must close the stalled upstream request");}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.javaindex 20cab6932d..54fcdac013 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyLoggingIntegrationTest.java@@ -1,26 +1,21 @@package io.github.hectorvent.floci.services.bedrockruntime;-import com.sun.net.httpserver.HttpServer;import io.github.hectorvent.floci.services.bedrockruntime.backend.ProxyBackend;import io.github.hectorvent.floci.testing.BedrockProxyBackendProfile;import io.github.hectorvent.floci.testutil.LogCapture;import io.quarkus.test.common.http.TestHTTPResource;import io.quarkus.test.junit.QuarkusTest;import io.quarkus.test.junit.TestProfile;-import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;import org.junit.jupiter.params.provider.ValueSource;-import java.io.IOException;-import java.net.InetSocketAddress;import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;import java.nio.charset.StandardCharsets;-import java.time.Duration;import java.util.List;-import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.LogRecord;@@ -30,6 +25,7 @@ import static io.github.hectorvent.floci.testing.BedrockProxyBackendProfile.PORTimport static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -51,25 +47,17 @@ class BedrockProxyLoggingIntegrationTest {@ParameterizedTest@ValueSource(strings = {"converse", "converse-stream"})- void upstreamErrorIsLoggedWithoutResponseBody(String operation) throws IOException {+ void upstreamErrorIsLoggedWithoutResponseBody(String operation) throws Exception {String marker = "synthetic-private-upstream-content";- HttpServer backend = HttpServer.create(new InetSocketAddress("127.0.0.1", PORT), 0);- backend.createContext("/v1/chat/completions", exchange -> {- exchange.getRequestBody().readAllBytes();- byte[] body = ("{\"error\":\"" + marker + "\"}").getBytes(StandardCharsets.UTF_8);- exchange.getResponseHeaders().add("Content-Type", "application/json");- exchange.sendResponseHeaders(503, body.length);- try (exchange) {- exchange.getResponseBody().write(body);- }- });- backend.start();- try {+ String body = "{\"error\":\"" + marker + "\"}";+ String completeError = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"+ + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n\r\n" + body;+ try (StallingUpstream upstream = new StallingUpstream(PORT, completeError)) {List<LogRecord> records = LogCapture.capture(ProxyBackend.class, () ->given().contentType("application/json")- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")- .body("{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}")+ .header("Authorization", ConverseRequests.AUTHORIZATION)+ .body(ConverseRequests.BODY).when().post("/model/fixture/" + operation).then()@@ -82,45 +70,50 @@ class BedrockProxyLoggingIntegrationTest {&& formatter.format(record).contains("HTTP 503")), "The upstream status must remain diagnosable");assertFalse(records.stream().anyMatch(record -> formatter.format(record).contains(marker)),"Operational logs must not include upstream response bodies");- } finally {- backend.stop(0);}}- @Test- void streamingErrorBodyReadIsBoundedWithoutWaitingForEof() throws Exception {- CountDownLatch release = new CountDownLatch(1);- HttpServer backend = HttpServer.create(new InetSocketAddress("127.0.0.1", PORT), 0);- backend.createContext("/v1/chat/completions", exchange -> {- exchange.getRequestBody().readAllBytes();- exchange.sendResponseHeaders(503, 32768);- try (exchange) {- exchange.getResponseBody().write("x".repeat(8192).getBytes(StandardCharsets.UTF_8));- exchange.getResponseBody().flush();- try {- release.await(10, TimeUnit.SECONDS);- } catch (InterruptedException interrupted) {- Thread.currentThread().interrupt();- }- }- });- backend.start();- try (HttpClient client = HttpClient.newHttpClient()) {- HttpRequest request = HttpRequest.newBuilder(base.resolve("/model/fixture/converse-stream"))- .timeout(Duration.ofSeconds(5))- .header("Content-Type", "application/json")- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request")- .POST(HttpRequest.BodyPublishers.ofString(- "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}"))- .build();- HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());+ @ParameterizedTest+ @ValueSource(strings = {"converse", "converse-stream"})+ void errorBodyReadStopsAtTheCapWithoutWaitingForEof(String operation) throws Exception {+ String cappedError = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: 32768\r\n\r\n"+ + "x".repeat(8192);+ try (StallingUpstream upstream = new StallingUpstream(PORT, cappedError);+ HttpClient client = HttpClient.newHttpClient()) {+ HttpResponse<String> response = client.send(ConverseRequests.to(base, operation),+ HttpResponse.BodyHandlers.ofString());assertEquals(424, response.statusCode());assertTrue(response.body().contains("ModelErrorException"));assertTrue(response.body().contains("HTTP 503"));+ // Only a read that ends at the cap carries the body; the deadline cut would return none of it.+ assertTrue(response.body().contains("x".repeat(512)), "The error read must end at the 8 KiB cap");assertTrue(response.body().length() < 1024, "Caller error descriptions must remain bounded");- } finally {- release.countDown();- backend.stop(0);+ }+ }++ @ParameterizedTest+ @CsvSource({"converse, 0", "converse, 1", "converse, 50",+ "converse-stream, 0", "converse-stream, 1", "converse-stream, 50"})+ void stalledErrorBodyHasDeadlineAndClosesUpstream(String operation, int lines) throws Exception {+ String marker = "synthetic-private-upstream-content";+ String errorPrefix = "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: 32768\r\n\r\n"+ + (marker + "\n").repeat(lines);+ try (StallingUpstream upstream = new StallingUpstream(PORT, errorPrefix);+ HttpClient client = HttpClient.newHttpClient()) {+ HttpRequest request = ConverseRequests.to(base, operation);+ List<LogRecord> records = LogCapture.capture(ProxyBackend.class, () -> {+ HttpResponse<String> response = assertDoesNotThrow(+ () -> client.send(request, HttpResponse.BodyHandlers.ofString()));+ assertEquals(424, response.statusCode());+ assertTrue(response.body().contains("ModelErrorException"));+ assertTrue(response.body().contains("HTTP 503"));+ assertTrue(response.body().length() < 1024);+ assertTrue(assertDoesNotThrow(() -> upstream.awaitClosed(1, TimeUnit.SECONDS)),+ "The proxy must close the stalled upstream body");+ });+ SimpleFormatter formatter = new SimpleFormatter();+ assertFalse(records.stream().anyMatch(record -> formatter.format(record).contains(marker)));+ assertTrue(records.stream().anyMatch(record -> formatter.format(record).contains("HTTP 503")));}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.javanew file mode 100644index 0000000000..6d49647623--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/ConverseRequests.java@@ -0,0 +1,27 @@+package io.github.hectorvent.floci.services.bedrockruntime;++import java.net.URI;+import java.net.http.HttpRequest;+import java.time.Duration;++/**+ * The Converse request the proxy tests send: a SigV4-shaped Authorization header and one user+ * message, to the {@code fixture} model id, which reaches the stand-in through passthrough.+ */+final class ConverseRequests {++ static final String AUTHORIZATION = "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/bedrock/aws4_request";+ static final String BODY = "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"synthetic\"}]}]}";++ private ConverseRequests() {+ }++ static HttpRequest to(URI base, String operation) {+ return HttpRequest.newBuilder(base.resolve("/model/fixture/" + operation))+ .timeout(Duration.ofSeconds(10))+ .header("Content-Type", "application/json")+ .header("Authorization", AUTHORIZATION)+ .POST(HttpRequest.BodyPublishers.ofString(BODY))+ .build();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.javanew file mode 100644index 0000000000..f33a348729--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/StallingUpstream.java@@ -0,0 +1,86 @@+package io.github.hectorvent.floci.services.bedrockruntime;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.ServerSocket;+import java.net.Socket;+import java.net.SocketException;+import java.nio.charset.StandardCharsets;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;++/**+ * A raw-socket stand-in for an OpenAI-compatible backend. It accepts one request, writes a+ * fixed response prefix, then stalls, so a test can observe whether Floci closes the upstream.+ */+final class StallingUpstream implements AutoCloseable {++ private final ServerSocket server;+ private final Thread thread;+ private final CountDownLatch responded = new CountDownLatch(1);+ private final CompletableFuture<Boolean> closed = new CompletableFuture<>();++ StallingUpstream(int port, String responsePrefix) throws IOException {+ server = new ServerSocket(port);+ server.setSoTimeout(10000);+ thread = Thread.ofVirtual().start(() -> serve(responsePrefix));+ }++ private void serve(String responsePrefix) {+ try (Socket socket = server.accept()) {+ socket.setSoTimeout(10000);+ InputStream input = socket.getInputStream();+ StringBuilder headers = new StringBuilder();+ while (!headers.toString().endsWith("\r\n\r\n")) {+ int next = input.read();+ if (next < 0 || headers.length() > 65536) {+ throw new IOException("Incomplete fixture request headers");+ }+ headers.append((char) next);+ }+ int length = 0;+ for (String header : headers.toString().split("\r\n")) {+ if (header.toLowerCase().startsWith("content-length:")) {+ length = Integer.parseInt(header.substring(header.indexOf(':') + 1).trim());+ }+ }+ if (input.readNBytes(length).length != length) {+ throw new IOException("Incomplete fixture request body");+ }+ OutputStream output = socket.getOutputStream();+ output.write(responsePrefix.getBytes(StandardCharsets.UTF_8));+ output.flush();+ responded.countDown();+ // Nothing more is sent, so only Floci closing its side ends this read.+ try {+ closed.complete(input.read() == -1);+ } catch (SocketException disconnected) {+ closed.complete(true);+ }+ } catch (Exception failure) {+ closed.completeExceptionally(failure);+ }+ }++ boolean awaitResponded(long timeout, TimeUnit unit) throws InterruptedException {+ return responded.await(timeout, unit);+ }++ boolean awaitClosed(long timeout, TimeUnit unit) throws Exception {+ try {+ return closed.get(timeout, unit);+ } catch (TimeoutException stillOpen) {+ return false;+ }+ }++ @Override+ public void close() throws Exception {+ thread.interrupt();+ thread.join(1000);+ server.close();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java b/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.javaindex b94d84133b..a8f4f5518f 100644--- a/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java+++ b/src/test/java/io/github/hectorvent/floci/testing/BedrockProxyBackendProfile.java@@ -21,12 +21,16 @@ import java.util.Map;* <li>{@code proxy.passthrough=true} forwards a model id with no mapping as it is.* {@code BedrockProxyPassthroughIntegrationTest}, {@code BedrockProxyCancellationIntegrationTest}* and {@code BedrockProxyLoggingIntegrationTest} rely on it.</li>+ * <li>{@code proxy.error-body-timeout-millis=10} shortens the deadline for collecting an+ * upstream error body, so the stalled-error-body tests do not each wait the one-second default.+ * {@code BedrockProxyLoggingIntegrationTest} relies on it.</li>* </ul>** <p>Each override is inert for the classes that do not rely on it. Only the stand-in in* {@code BedrockProxyIntegrationTest} reads the {@code Authorization} header. The other classes* call model ids the mapping does not name, and every call in {@code BedrockProxyIntegrationTest}- * names the mapped id, which the mapping resolves before passthrough is consulted.+ * names the mapped id, which the mapping resolves before passthrough is consulted. Their stand-ins+ * write each error body whole, so the shorter error-body deadline never fires for them.** <p>{@code BedrockProxyTimeoutIntegrationTest} stays apart. Its one-second request timeout would* also close an upstream that stalls before its headers, so the cancellation tests would pass@@ -44,6 +48,7 @@ public class BedrockProxyBackendProfile implements QuarkusTestProfile {"floci.services.bedrock-runtime.proxy.url", "http://127.0.0.1:" + PORT + "/v1","floci.services.bedrock-runtime.proxy.api-key", "test-key","floci.services.bedrock-runtime.proxy.model-mapping", MAPPED_MODEL_ID + "=claude-3-haiku",- "floci.services.bedrock-runtime.proxy.passthrough", "true");+ "floci.services.bedrock-runtime.proxy.passthrough", "true",+ "floci.services.bedrock-runtime.proxy.error-body-timeout-millis", "10");}}