Light, fluffy, and always free - The AWS Local Emulator alternative
feat(bedrock-runtime): stream ConverseStream incrementally (#3043)
Scott Steen0471d6f488f4
diff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xmlindex 14afc79ae0..188257bc69 100644--- a/compatibility-tests/sdk-test-java/pom.xml+++ b/compatibility-tests/sdk-test-java/pom.xml@@ -337,6 +337,11 @@<groupId>software.amazon.awssdk</groupId><artifactId>bedrockagentcore</artifactId></dependency>+ <!-- Bedrock Runtime client (Converse/ConverseStream) -->+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>bedrockruntime</artifactId>+ </dependency><!-- AWS IoT client --><dependency><groupId>software.amazon.awssdk</groupId>diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex 7c09e3a095..fb85a938b6 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -68,6 +68,7 @@ import software.amazon.awssdk.services.ec2.Ec2Client;import software.amazon.awssdk.services.ecr.EcrClient;import software.amazon.awssdk.services.bedrockagentcore.BedrockAgentCoreClient;import software.amazon.awssdk.services.bedrockagentcorecontrol.BedrockAgentCoreControlClient;+import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient;import software.amazon.awssdk.services.pipes.PipesClient;import software.amazon.awssdk.services.codebuild.CodeBuildClient;import software.amazon.awssdk.services.codedeploy.CodeDeployClient;@@ -925,6 +926,16 @@ public final class TestFixtures {.build();}+ public static BedrockRuntimeAsyncClient bedrockRuntimeAsyncClient() {+ return BedrockRuntimeAsyncClient.builder()+ .endpointOverride(ENDPOINT)+ .region(REGION)+ .credentialsProvider(CREDENTIALS)+ .httpClientBuilder(NettyNioAsyncHttpClient.builder()+ .protocol(Protocol.HTTP1_1))+ .build();+ }+public static IotClient iotClient() {return IotClient.builder().endpointOverride(ENDPOINT)diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.javanew file mode 100644index 0000000000..fb9351d31d--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.java@@ -0,0 +1,70 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient;+import software.amazon.awssdk.services.bedrockruntime.model.*;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;+import java.util.stream.Collectors;++import static org.assertj.core.api.Assertions.assertThat;++/**+ * Verifies Bedrock Runtime's ConverseStream against the real AWS SDK's event-stream decoder,+ * not just Floci's own raw-byte assertions - proves the {@code application/vnd.amazon.eventstream}+ * frames Floci emits actually round-trip through {@link BedrockRuntimeAsyncClient}'s generated+ * unmarshaller end to end.+ */+@DisplayName("Bedrock Runtime ConverseStream")+class BedrockRuntimeConverseStreamTest {++ private static final String MODEL_ID = "anthropic.claude-3-haiku-20240307-v1:0";++ @Test+ void converseStream_deliversTextDeltasAndCompletes() throws Exception {+ try (BedrockRuntimeAsyncClient client = TestFixtures.bedrockRuntimeAsyncClient()) {+ List<ConverseStreamOutput> received = new ArrayList<>();+ AtomicReference<Throwable> error = new AtomicReference<>();+ CountDownLatch complete = new CountDownLatch(1);++ ConverseStreamResponseHandler handler = ConverseStreamResponseHandler.builder()+ .subscriber(received::add)+ .onError(error::set)+ .onComplete(complete::countDown)+ .build();++ ConverseStreamRequest request = ConverseStreamRequest.builder()+ .modelId(MODEL_ID)+ .messages(Message.builder()+ .role(ConversationRole.USER)+ .content(List.of(ContentBlock.fromText("hi")))+ .build())+ .build();++ CompletableFuture<Void> future = client.converseStream(request, handler);+ future.get(15, TimeUnit.SECONDS);++ assertThat(complete.await(15, TimeUnit.SECONDS)).as("onComplete should fire").isTrue();+ assertThat(error.get()).as("no stream error expected").isNull();+ assertThat(received).isNotEmpty();+ assertThat(received.get(0)).isInstanceOf(MessageStartEvent.class);++ List<String> text = received.stream()+ .filter(ContentBlockDeltaEvent.class::isInstance)+ .map(ContentBlockDeltaEvent.class::cast)+ .map(ContentBlockDeltaEvent::delta)+ .map(ContentBlockDelta::text)+ .toList();+ assertThat(text)+ .containsExactly("Floci stub response", " for model=", MODEL_ID);++ assertThat(received).anyMatch(MessageStopEvent.class::isInstance);+ }+ }+}diff --git a/docs/services/bedrock-runtime.md b/docs/services/bedrock-runtime.mdindex 0c38ee18c5..202dd0c270 100644--- a/docs/services/bedrock-runtime.md+++ b/docs/services/bedrock-runtime.md@@ -18,7 +18,7 @@ The Bedrock management plane (`aws bedrock ...`: `ListFoundationModels`, `GetFou|-----------|----------|-------|| `Converse` | `POST /model/{modelId}/converse` | `stub`: returns a static assistant message. `proxy`: forwards to the configured OpenAI-compatible backend and translates the reply. || `InvokeModel` | `POST /model/{modelId}/invoke` | Returns Anthropic-shaped body for `anthropic.*` and `*.anthropic.*` model ids; generic `{"outputs": [...]}` shape otherwise. Not proxied yet — always the stub response, regardless of backend. |-| `ConverseStream` | `POST /model/{modelId}/converse-stream` | `stub`: emits a static assistant turn as `messageStart`/`contentBlockDelta`/`contentBlockStop`/`messageStop`/`metadata` events. `proxy`: forwards a streaming request to the configured OpenAI-compatible backend and translates the SSE response into the same event sequence. Response body is `application/vnd.amazon.eventstream`-framed, built in one shot rather than incrementally chunked. |+| `ConverseStream` | `POST /model/{modelId}/converse-stream` | `stub`: emits a static assistant turn as `messageStart`/`contentBlockDelta`/`contentBlockStop`/`messageStop`/`metadata` events. `proxy`: forwards a streaming request to the configured OpenAI-compatible backend and translates each SSE chunk into a Bedrock event as it arrives. Text deltas reach the client incrementally, not all at once. Response body is `application/vnd.amazon.eventstream`-framed. If the upstream stream ends without a `finish_reason` or `"[DONE]"` (e.g. a dropped connection), the failure is reported as an in-band `modelStreamErrorException` event, since the HTTP 200 is already committed by the time streaming starts, matching real Bedrock's own behavior for a mid-stream failure. || `InvokeModelWithResponseStream` | `POST /model/{modelId}/invoke-with-response-stream` | Returns 501 `UnsupportedOperationException` |`modelId` is URL-decoded by JAX-RS and echoed verbatim. Plain model ids (e.g. `anthropic.claude-3-haiku-20240307-v1:0`), inference-profile ids (e.g. `us.anthropic.claude-3-5-sonnet-20241022-v2:0`), and full ARNs containing slashes (e.g. `arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0`) are all accepted.@@ -89,6 +89,6 @@ print(resp["output"]["message"]["content"][0]["text"])- Real model inference with the `stub` backend (always returns a fixed string).- `InvokeModel` against the `proxy` backend (still returns the stub response).- `InvokeModelWithResponseStream` returns 501, regardless of backend.-- `ConverseStream` responses are built in one shot and returned as a complete event-stream body, not incrementally chunked over the wire.+- `ConverseStream` tool calls are still accumulated across chunks and emitted as a single complete `contentBlockDelta` once the stream ends, rather than as incremental argument fragments — SDKs consume both the same way, since they concatenate `delta.toolUse.input` across events for a given `contentBlockIndex` before parsing it as JSON.- Bedrock management plane (`ListFoundationModels`, `GetFoundationModel`, model customisation).- Bedrock Agents, Knowledge Bases, Guardrails, provisioned throughput.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.javaindex 58ec858d6a..65e3d84be8 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.core.common;import java.io.ByteArrayOutputStream;import java.io.DataOutputStream;+import java.io.IOException;import java.nio.charset.StandardCharsets;import java.util.LinkedHashMap;import java.util.Map;@@ -9,7 +10,7 @@ import java.util.zip.CRC32;public class AwsEventStreamEncoder {- public static byte[] encodeMessage(LinkedHashMap<String, String> headers, byte[] payload) throws Exception {+ public static byte[] encodeMessage(LinkedHashMap<String, String> headers, byte[] payload) throws IOException {byte[] headersBytes = encodeHeaders(headers);int totalLen = 4 + 4 + 4 + headersBytes.length + payload.length + 4;@@ -35,7 +36,7 @@ public class AwsEventStreamEncoder {return buf.toByteArray();}- private static byte[] encodeHeaders(LinkedHashMap<String, String> headers) throws Exception {+ private static byte[] encodeHeaders(LinkedHashMap<String, String> headers) throws IOException {ByteArrayOutputStream h = new ByteArrayOutputStream();for (Map.Entry<String, String> e : headers.entrySet()) {byte[] name = e.getKey().getBytes(StandardCharsets.UTF_8);diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.javaindex f2bfa9a529..579183e4f8 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java@@ -11,10 +11,15 @@ import jakarta.ws.rs.POST;import jakarta.ws.rs.Path;import jakarta.ws.rs.PathParam;import jakarta.ws.rs.Produces;+import jakarta.ws.rs.core.GenericEntity;import jakarta.ws.rs.core.MediaType;import jakarta.ws.rs.core.Response;+import jakarta.ws.rs.core.StreamingOutput;import org.jboss.logging.Logger;+import java.io.OutputStream;+import java.util.function.Consumer;+/*** AWS Bedrock Runtime REST JSON endpoints.*@@ -87,13 +92,17 @@ public class BedrockRuntimeController {JsonNode messages = validateMessages(request);- byte[] response = service.buildConverseStreamResponse(modelId, request);+ Consumer<OutputStream> stream = service.buildConverseStreamResponse(modelId, request);LOG.debugv("Bedrock Converse Stream: modelId={0}, messages={1}", modelId, messages.size());- return Response.ok(response)+ return Response.ok(streaming(stream)).header("Content-Type", "application/vnd.amazon.eventstream").build();}+ private static GenericEntity<StreamingOutput> streaming(Consumer<OutputStream> stream) {+ return new GenericEntity<>(stream::accept, StreamingOutput.class);+ }+private static JsonNode validateMessages(ObjectNode request) {JsonNode messages = request.path("messages");if (!messages.isArray() || messages.isEmpty()) {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.javaindex 2edcd9f4ea..8e65712db9 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java@@ -9,6 +9,9 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.OutputStream;+import java.util.function.Consumer;+/*** Thin orchestration layer for Bedrock Runtime: delegates Converse/InvokeModel to the* backend selected by {@code floci.services.bedrock-runtime.backend}.@@ -37,7 +40,7 @@ public class BedrockRuntimeService {return backend().invokeModel(modelId, body);}- public byte[] buildConverseStreamResponse(String modelId, ObjectNode bedrockRequest) {+ public Consumer<OutputStream> buildConverseStreamResponse(String modelId, ObjectNode bedrockRequest) {return backend().converseStream(modelId, bedrockRequest);}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.javaindex 18c390822d..c6a35a7a97 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java@@ -1,6 +1,10 @@package io.github.hectorvent.floci.services.bedrockruntime.backend;import com.fasterxml.jackson.databind.node.ObjectNode;+import jakarta.ws.rs.core.StreamingOutput;++import java.io.OutputStream;+import java.util.function.Consumer;/*** Executes Bedrock Runtime Converse/InvokeModel requests against a concrete backend@@ -12,5 +16,5 @@ public interface BedrockBackend {byte[] invokeModel(String modelId, byte[] body);- byte[] converseStream(String modelId, ObjectNode bedrockRequest);+ Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest);}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.javaindex 489ee163a6..15f1659335 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java@@ -7,10 +7,13 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import org.jboss.logging.Logger;+import java.io.OutputStream;import java.util.ArrayList;+import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.stream.Stream;/*** Translates between the Bedrock Runtime Converse wire shape and the OpenAI@@ -259,22 +262,27 @@ final class BedrockOpenAiTranslator {}/**- * Translates an OpenAI Chat Completions SSE stream body ({@code data: {...}\n\n} lines,- * terminated by {@code data: [DONE]}) into the ordered Bedrock ConverseStream events- * (messageStart, contentBlockDelta, contentBlockStop, messageStop, metadata), framed as- * {@code application/vnd.amazon.eventstream} bytes via {@link BedrockStreamEncoder}.+ * Translates an OpenAI Chat Completions SSE stream ({@code data: {...}} lines, terminated by+ * {@code data: [DONE]}) into Bedrock ConverseStream events, writing each frame to {@code out}+ * as soon as it's known instead of buffering the whole response - text deltas reach the+ * client as they arrive from the upstream.+ *+ * <p>By the time this runs, JAX-RS has already committed the 200 response (messageStart is+ * about to be written), so a failure discovered here - a truncated stream that never reaches+ * a finish_reason or "[DONE]" - can no longer become an HTTP error status. It's written as an+ * in-band {@code modelStreamErrorException} event instead, matching how real Bedrock reports+ * a mid-stream failure after the response has already started.** <p>Tool calls are accumulated across chunks and emitted as a single complete- * contentBlockDelta once the stream ends, rather than as incremental argument- * fragments - simpler than real Bedrock's fragment-by-fragment streaming, but- * SDKs consume both the same way since they just concatenate delta.toolUse.input- * across events for a given contentBlockIndex before parsing it as JSON.+ * contentBlockDelta once the stream ends, rather than as incremental argument fragments -+ * simpler than real Bedrock's fragment-by-fragment streaming, but SDKs consume both the same+ * way since they just concatenate delta.toolUse.input across events for a given+ * contentBlockIndex before parsing it as JSON.*/- static byte[] toBedrockStreamEvents(ObjectMapper mapper, String sseBody, long latencyMs) {- List<BedrockStreamEncoder.Event> events = new ArrayList<>();- events.add(new BedrockStreamEncoder.Event("messageStart", mapper.createObjectNode().put("role", "assistant")));+ static void streamBedrockEvents(ObjectMapper mapper, Stream<String> sseLines, OutputStream out, long startNanos) {+ BedrockStreamEncoder.writeEvent(mapper, out, "messageStart", mapper.createObjectNode().put("role", "assistant"));- StringBuilder text = new StringBuilder();+ boolean anyContentDeltaWritten = false;Map<Integer, ToolCallAccumulator> toolCalls = new LinkedHashMap<>();String finishReason = "stop";boolean sawFinishReason = false;@@ -282,76 +290,123 @@ final class BedrockOpenAiTranslator {int promptTokens = 0;int completionTokens = 0;int totalTokens = 0;- int parsedChunks = 0;- for (String rawLine : sseBody.split("\n")) {- String line = rawLine.strip();- if (!line.startsWith("data:")) {- continue;- }- String data = line.substring(5).strip();- if (data.isEmpty()) {- continue;- }- if ("[DONE]".equals(data)) {- sawDone = true;- continue;- }- JsonNode chunk;- try {- chunk = mapper.readTree(data);- } catch (Exception e) {- LOG.warnv("Skipping malformed SSE chunk from proxy backend: {0}", e.getMessage());- continue;- }- parsedChunks++;+ try (sseLines) {+ Iterator<String> lineIterator = sseLines.iterator();+ String rawLine;+ while ((rawLine = readLine(lineIterator)) != null) {+ String line = rawLine.strip();+ if (!line.startsWith("data:")) {+ continue;+ }+ String data = line.substring(5).strip();+ if (data.isEmpty()) {+ continue;+ }+ if ("[DONE]".equals(data)) {+ sawDone = true;+ continue;+ }+ JsonNode chunk;+ try {+ chunk = mapper.readTree(data);+ } catch (Exception e) {+ LOG.warnv("Skipping malformed SSE chunk from proxy backend: {0}", e.getMessage());+ continue;+ }- JsonNode usage = chunk.path("usage");- if (usage.isObject()) {- promptTokens = usage.path("prompt_tokens").asInt(promptTokens);- completionTokens = usage.path("completion_tokens").asInt(completionTokens);- totalTokens = usage.path("total_tokens").asInt(totalTokens);- }+ JsonNode usage = chunk.path("usage");+ if (usage.isObject()) {+ promptTokens = usage.path("prompt_tokens").asInt(promptTokens);+ completionTokens = usage.path("completion_tokens").asInt(completionTokens);+ totalTokens = usage.path("total_tokens").asInt(totalTokens);+ }- JsonNode choice = chunk.path("choices").path(0);- String reason = choice.path("finish_reason").asText(null);- if (reason != null) {- finishReason = reason;- sawFinishReason = true;- }+ JsonNode choice = chunk.path("choices").path(0);+ String reason = choice.path("finish_reason").asText(null);+ if (reason != null) {+ finishReason = reason;+ sawFinishReason = true;+ }- JsonNode delta = choice.path("delta");- String contentDelta = delta.path("content").asText(null);- if (contentDelta != null && !contentDelta.isEmpty()) {- text.append(contentDelta);- ObjectNode deltaEvent = mapper.createObjectNode();- deltaEvent.put("contentBlockIndex", 0);- deltaEvent.putObject("delta").put("text", contentDelta);- events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));+ JsonNode delta = choice.path("delta");+ if (writeContentDelta(mapper, out, delta)) {+ anyContentDeltaWritten = true;+ }+ writeToolCalls(delta, toolCalls);}+ }- JsonNode toolCallDeltas = delta.path("tool_calls");- if (toolCallDeltas.isArray()) {- for (JsonNode toolCallDelta : toolCallDeltas) {- int index = toolCallDelta.path("index").asInt(0);- ToolCallAccumulator acc = toolCalls.computeIfAbsent(index, i -> new ToolCallAccumulator());- String id = toolCallDelta.path("id").asText(null);- if (id != null) {- acc.id = id;- }- JsonNode function = toolCallDelta.path("function");- String name = function.path("name").asText(null);- if (name != null) {- acc.name = name;- }- String argsFragment = function.path("arguments").asText(null);- if (argsFragment != null) {- acc.arguments.append(argsFragment);- }- }+ boolean streamCompleted = sawFinishReason || sawDone;+ if (!streamCompleted) {+ writeTruncatedException(mapper, out);+ return;+ }++ if (anyContentDeltaWritten) {+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStop",+ mapper.createObjectNode().put("contentBlockIndex", 0));+ }++ var validToolCalls = collectValidToolCalls(mapper, toolCalls);+ int blockIndex = anyContentDeltaWritten ? 1 : 0;+ for (ToolCallAccumulator acc : validToolCalls) {+ writeToolCalls(mapper, out, acc, blockIndex);+ blockIndex++;+ }++ boolean hasToolCalls = !validToolCalls.isEmpty();+ BedrockStreamEncoder.writeEvent(mapper, out, "messageStop",+ mapper.createObjectNode().put("stopReason", hasToolCalls ? "tool_use" : mapFinishReason(finishReason)));++ writeMetadata(mapper, out, startNanos, promptTokens, completionTokens, totalTokens);+ }++ private static String readLine(Iterator<String> lineIterator) {+ try {+ if (!lineIterator.hasNext()) {+ return null;}+ return lineIterator.next();+ } catch (RuntimeException e) {+ // The upstream connection failed mid-read (e.g. it dropped the connection).+ // Treat exactly like reaching EOF without a finish_reason/[DONE]: fall through+ // to the truncation handling instead of letting an UncheckedIOException+ // propagate and abruptly kill the response mid-stream.+ LOG.warnv(e, "Bedrock proxy backend connection failed while streaming");+ return null;}+ }+ private static void writeMetadata(ObjectMapper mapper, OutputStream out, long startNanos, int promptTokens, int completionTokens, int totalTokens) {+ long latencyMs = (System.nanoTime() - startNanos) / 1_000_000;+ ObjectNode metadata = mapper.createObjectNode();+ metadata.putObject("usage")+ .put("inputTokens", promptTokens)+ .put("outputTokens", completionTokens)+ .put("totalTokens", totalTokens);+ metadata.putObject("metrics").put("latencyMs", latencyMs);+ BedrockStreamEncoder.writeEvent(mapper, out, "metadata", metadata);+ }++ private static void writeToolCalls(ObjectMapper mapper, OutputStream out, ToolCallAccumulator acc, int blockIndex) {+ ObjectNode startEvent = mapper.createObjectNode();+ startEvent.put("contentBlockIndex", blockIndex);+ ObjectNode toolUseStart = startEvent.putObject("start").putObject("toolUse");+ toolUseStart.put("toolUseId", acc.id);+ toolUseStart.put("name", acc.name);+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStart", startEvent);++ ObjectNode deltaEvent = mapper.createObjectNode();+ deltaEvent.put("contentBlockIndex", blockIndex);+ deltaEvent.putObject("delta").putObject("toolUse").put("input", acc.arguments.toString());+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockDelta", deltaEvent);++ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStop",+ mapper.createObjectNode().put("contentBlockIndex", blockIndex));+ }++ private static List<ToolCallAccumulator> collectValidToolCalls(ObjectMapper mapper, Map<Integer, ToolCallAccumulator> toolCalls) {// A tool_calls delta only becomes a usable toolUse block once it has a non-blank id,// a non-blank name, and arguments that actually form a JSON object - a partial or// truncated fragment (e.g. an id with no name, or unclosed argument JSON) is dropped@@ -365,59 +420,54 @@ final class BedrockOpenAiTranslator {acc.id, acc.name);}}+ return validToolCalls;+ }- boolean hasContent = text.length() > 0 || !validToolCalls.isEmpty();- boolean streamCompleted = sawFinishReason || sawDone;- if (!hasContent || !streamCompleted) {- // Either no usable completion content was ever produced (empty/garbled body,- // role/usage/finish_reason-only records, a data:-prefixed error object, or only- // invalid tool_calls fragments), or the stream ended without a finish_reason or a- // "[DONE]" terminator at all - a truncated generation (e.g. the connection dropped- // mid-stream). Neither case is a real completion: surface it as a backend failure- // rather than fabricating a successful message out of whatever partial data arrived.- throw new AwsException("ModelErrorException",- "Proxy backend's ConverseStream response did not complete ("- + parsedChunks + " SSE chunk(s) parsed, streamCompleted=" + streamCompleted- + "): " + truncate(sseBody, 512), 424);- }+ private static void writeTruncatedException(ObjectMapper mapper, OutputStream out) {+ // The stream ended (upstream closed the connection, or we ran out of lines) without+ // ever reaching a finish_reason or "[DONE]" - a truncated generation. The HTTP status+ // is already 200, so this can only be reported in-band.+ ObjectNode errorPayload = mapper.createObjectNode();+ errorPayload.put("message", "Proxy backend's ConverseStream response ended without a finish_reason "+ + "or \"[DONE]\" terminator - the stream may have been truncated.");+ BedrockStreamEncoder.writeException(mapper, out, "modelStreamErrorException", errorPayload);+ }- if (text.length() > 0) {- events.add(new BedrockStreamEncoder.Event("contentBlockStop",- mapper.createObjectNode().put("contentBlockIndex", 0)));+ private static void writeToolCalls(JsonNode delta, Map<Integer, ToolCallAccumulator> toolCalls) {+ JsonNode toolCallDeltas = delta.path("tool_calls");+ if (!toolCallDeltas.isArray()) {+ return;}-- boolean hasToolCalls = !validToolCalls.isEmpty();- int blockIndex = text.length() > 0 ? 1 : 0;- for (ToolCallAccumulator acc : validToolCalls) {- ObjectNode startEvent = mapper.createObjectNode();- startEvent.put("contentBlockIndex", blockIndex);- ObjectNode toolUseStart = startEvent.putObject("start").putObject("toolUse");- toolUseStart.put("toolUseId", acc.id);- toolUseStart.put("name", acc.name);- events.add(new BedrockStreamEncoder.Event("contentBlockStart", startEvent));-- ObjectNode deltaEvent = mapper.createObjectNode();- deltaEvent.put("contentBlockIndex", blockIndex);- deltaEvent.putObject("delta").putObject("toolUse").put("input", acc.arguments.toString());- events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));-- events.add(new BedrockStreamEncoder.Event("contentBlockStop",- mapper.createObjectNode().put("contentBlockIndex", blockIndex)));- blockIndex++;+ for (JsonNode toolCallDelta : toolCallDeltas) {+ int index = toolCallDelta.path("index").asInt(0);+ ToolCallAccumulator acc = toolCalls.computeIfAbsent(index, i -> new ToolCallAccumulator());+ String id = toolCallDelta.path("id").asText(null);+ if (id != null) {+ acc.id = id;+ }+ JsonNode function = toolCallDelta.path("function");+ String name = function.path("name").asText(null);+ if (name != null) {+ acc.name = name;+ }+ String argsFragment = function.path("arguments").asText(null);+ if (argsFragment != null) {+ acc.arguments.append(argsFragment);+ }}+ }- events.add(new BedrockStreamEncoder.Event("messageStop",- mapper.createObjectNode().put("stopReason", hasToolCalls ? "tool_use" : mapFinishReason(finishReason))));-- ObjectNode metadata = mapper.createObjectNode();- metadata.putObject("usage")- .put("inputTokens", promptTokens)- .put("outputTokens", completionTokens)- .put("totalTokens", totalTokens);- metadata.putObject("metrics").put("latencyMs", latencyMs);- events.add(new BedrockStreamEncoder.Event("metadata", metadata));+ private static boolean writeContentDelta(ObjectMapper mapper, OutputStream out, JsonNode delta) {+ String contentDelta = delta.path("content").asText(null);+ if (contentDelta == null || contentDelta.isEmpty()) {+ return false;+ }- return BedrockStreamEncoder.encode(mapper, events);+ ObjectNode deltaEvent = mapper.createObjectNode();+ deltaEvent.put("contentBlockIndex", 0);+ deltaEvent.putObject("delta").put("text", contentDelta);+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockDelta", deltaEvent);+ return true;}private static boolean isUsableToolCall(ObjectMapper mapper, ToolCallAccumulator acc) {@@ -442,20 +492,21 @@ final class BedrockOpenAiTranslator {if (content.isTextual()) {return content.asText("");}- if (content.isArray()) {- StringBuilder sb = new StringBuilder();- for (JsonNode part : content) {- String text = part.path("text").asText(null);- if (text != null) {- if (sb.length() > 0) {- sb.append('\n');- }- sb.append(text);- }+ if (!content.isArray()) {+ return "";+ }+ StringBuilder sb = new StringBuilder();+ for (JsonNode part : content) {+ String text = part.path("text").asText(null);+ if (text == null) {+ continue;}- return sb.toString();+ if (!sb.isEmpty()) {+ sb.append('\n');+ }+ sb.append(text);}- return "";+ return sb.toString();}private static ObjectNode parseToolArguments(ObjectMapper mapper, String argumentsJson) {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.javaindex adb0403ba6..e85f0d7749 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java@@ -3,40 +3,51 @@ package io.github.hectorvent.floci.services.bedrockruntime.backend;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;-import io.github.hectorvent.floci.core.common.AwsException;-import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.OutputStream;+import java.io.UncheckedIOException;import java.util.LinkedHashMap;-import java.util.List;/**- * Frames a sequence of Bedrock ConverseStream events (messageStart, contentBlockDelta, ...)- * as {@code application/vnd.amazon.eventstream} messages, one per event, concatenated into- * a single response body - the same one-shot framing S3 Select and Kinesis SubscribeToShard- * use elsewhere in Floci, rather than true incremental HTTP chunking.+ * Frames Bedrock ConverseStream events (messageStart, contentBlockDelta, ...) as+ * {@code application/vnd.amazon.eventstream} messages and writes each one to the response+ * {@link OutputStream} as soon as it's known, so SDK clients see incremental delivery instead+ * of the whole response landing at once - the same binary framing S3 Select and Kinesis+ * SubscribeToShard build elsewhere in Floci, just written frame-by-frame instead of batched.*/final class BedrockStreamEncoder {private BedrockStreamEncoder() {}- record Event(String type, ObjectNode payload) {+ /** Writes a normal event frame ({@code :message-type: event}), e.g. messageStart, contentBlockDelta. */+ static void writeEvent(ObjectMapper mapper, OutputStream out, String eventType, ObjectNode payload) {+ writeFrame(mapper, out, "event", ":event-type", eventType, payload);}- static byte[] encode(ObjectMapper mapper, List<Event> events) {+ /**+ * Writes a modeled-exception frame ({@code :message-type: exception}) - the wire shape real+ * Bedrock uses to report a stream failure after the HTTP 200 response has already been+ * committed. The AWS SDK's generic JSON error unmarshaller reads {@code :exception-type} to+ * pick the matching modeled exception class (e.g. ModelStreamErrorException), the same way+ * it reads {@code x-amzn-ErrorType} for a regular non-streaming error response.+ */+ static void writeException(ObjectMapper mapper, OutputStream out, String exceptionType, ObjectNode payload) {+ writeFrame(mapper, out, "exception", ":exception-type", exceptionType, payload);+ }++ private static void writeFrame(ObjectMapper mapper, OutputStream out, String messageType, String typeHeaderName,+ String typeValue, ObjectNode payload) {+ LinkedHashMap<String, String> headers = new LinkedHashMap<>();+ headers.put(":message-type", messageType);+ headers.put(typeHeaderName, typeValue);+ headers.put(":content-type", "application/json");try {- ByteArrayOutputStream out = new ByteArrayOutputStream();- for (Event event : events) {- LinkedHashMap<String, String> headers = new LinkedHashMap<>();- headers.put(":message-type", "event");- headers.put(":event-type", event.type());- headers.put(":content-type", "application/json");- out.write(AwsEventStreamEncoder.encodeMessage(headers, mapper.writeValueAsBytes(event.payload())));- }- return out.toByteArray();- } catch (Exception e) {- throw new AwsException("InternalServerException",- "Failed to encode ConverseStream response: " + e.getMessage(), 500);+ out.write(AwsEventStreamEncoder.encodeMessage(headers, mapper.writeValueAsBytes(payload)));+ out.flush();+ } catch (IOException e) {+ throw new UncheckedIOException(e);}}}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.javaindex 9f50142c90..e593f4bcd7 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java@@ -9,6 +9,7 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.OutputStream;import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;@@ -17,6 +18,9 @@ import java.net.http.HttpTimeoutException;import java.time.Duration;import java.util.LinkedHashMap;import java.util.Map;+import java.util.function.Consumer;+import java.util.stream.Collectors;+import java.util.stream.Stream;/*** Forwards Bedrock Converse requests to any OpenAI-compatible {@code /chat/completions}@@ -52,16 +56,24 @@ public class ProxyBackend implements BedrockBackend {EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();String resolvedModel = resolveModel(modelId, proxyConfig);ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, false);- ProxyResult result = send(modelId, proxyConfig, openAiRequest);+ HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);++ long start = System.nanoTime();+ HttpResponse<String> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofString());+ long latencyMs = (System.nanoTime() - start) / 1_000_000;++ if (response.statusCode() >= 300) {+ throw failedRequest(response.statusCode(), response.body());+ }JsonNode openAiResponse;try {- openAiResponse = objectMapper.readTree(result.body());+ openAiResponse = objectMapper.readTree(response.body());} catch (Exception e) {throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);}- return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, result.latencyMs());+ return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, latencyMs);}@Override@@ -71,30 +83,71 @@ public class ProxyBackend implements BedrockBackend {}@Override- public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {+ public Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest) {EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();String resolvedModel = resolveModel(modelId, proxyConfig);ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, true);- ProxyResult result = send(modelId, proxyConfig, openAiRequest);- return BedrockOpenAiTranslator.toBedrockStreamEvents(objectMapper, result.body(), result.latencyMs());+ // Everything that can fail with a proper HTTP status - resolving the model, opening the+ // connection, checking the upstream status code - happens here, before any bytes of our+ // own response are written. Once the returned Consumer is invoked, JAX-RS has+ // already committed a 200, so failures discovered while consuming the body stream can+ // only be reported as an in-band Bedrock stream exception event (see+ // BedrockOpenAiTranslator.streamBedrockEvents).+ HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);+ long startNanos = System.nanoTime();+ HttpResponse<Stream<String>> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofLines());++ if (response.statusCode() >= 300) {+ String errorBody;+ try (Stream<String> body = response.body()) {+ errorBody = body.limit(50).collect(Collectors.joining("\n"));+ }+ throw failedRequest(response.statusCode(), errorBody);+ }++ return output -> BedrockOpenAiTranslator.streamBedrockEvents(objectMapper, response.body(), output, startNanos);}- private record ProxyResult(String body, long latencyMs) {+ private static AwsException failedRequest(int response, String errorBody) {+ LOG.warnv("Bedrock proxy backend returned HTTP {0}: {1}", response, errorBody);+ return new AwsException("ModelErrorException",+ "Proxy backend returned HTTP %d: %s".formatted(response, truncate(errorBody, 512)), 424);}- private ProxyResult send(String modelId, EmulatorConfig.BedrockProxyConfig proxyConfig, ObjectNode openAiRequest) {+ private <T> HttpResponse<T> invoke(String modelId, HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler) {+ try {+ return httpClient.send(request, bodyHandler);+ } catch (HttpTimeoutException e) {+ LOG.warnv("Bedrock proxy backend timed out: modelId={0}, url={1}, error={2}",+ modelId, request.uri(), e.getMessage());+ throw new AwsException("ModelTimeoutException", "Proxy backend timed out: " + e.getMessage(), 408);+ } catch (Exception e) {+ LOG.warnv("Bedrock proxy backend call failed: modelId={0}, url={1}, error={2}",+ modelId, request.uri(), e.getMessage());+ throw new AwsException("ModelErrorException", "Failed to reach proxy backend: " + e.getMessage(), 424);+ }+ }++ private HttpRequest buildHttpRequest(EmulatorConfig.BedrockProxyConfig proxyConfig, ObjectNode openAiRequest) {String baseUrl = proxyConfig.url().filter(url -> !url.isBlank()).orElseThrow(() -> new AwsException("ValidationException","floci.services.bedrock-runtime.proxy.url is required when backend=proxy.", 400));- URI uri;+ byte[] requestBody;+ try {+ requestBody = objectMapper.writeValueAsBytes(openAiRequest);+ } catch (Exception e) {+ throw new AwsException("InternalServerException", "Failed to serialize proxy request: " + e.getMessage(), 500);+ }+HttpRequest.Builder builder;try {- uri = URI.create(stripTrailingSlash(baseUrl) + "/chat/completions");+ URI uri = URI.create(stripTrailingSlash(baseUrl) + "/chat/completions");builder = HttpRequest.newBuilder().uri(uri)+ .POST(HttpRequest.BodyPublishers.ofByteArray(requestBody)).timeout(Duration.ofSeconds(proxyConfig.requestTimeoutSeconds())).header("Content-Type", "application/json");} catch (IllegalArgumentException e) {@@ -105,35 +158,7 @@ public class ProxyBackend implements BedrockBackend {.filter(key -> !key.isBlank()).ifPresent(key -> builder.header("Authorization", "Bearer " + key));- String requestBody;- try {- requestBody = objectMapper.writeValueAsString(openAiRequest);- } catch (Exception e) {- throw new AwsException("InternalServerException", "Failed to serialize proxy request: " + e.getMessage(), 500);- }- builder.POST(HttpRequest.BodyPublishers.ofString(requestBody));-- long start = System.nanoTime();- HttpResponse<String> response;- try {- response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString());- } catch (HttpTimeoutException e) {- LOG.warnv("Bedrock proxy backend timed out: modelId={0}, url={1}, error={2}", modelId, uri, e.getMessage());- throw new AwsException("ModelTimeoutException", "Proxy backend timed out: " + e.getMessage(), 408);- } catch (Exception e) {- LOG.warnv("Bedrock proxy backend call failed: modelId={0}, url={1}, error={2}", modelId, uri, e.getMessage());- throw new AwsException("ModelErrorException", "Failed to reach proxy backend: " + e.getMessage(), 424);- }- long latencyMs = (System.nanoTime() - start) / 1_000_000;-- if (response.statusCode() >= 300) {- LOG.warnv("Bedrock proxy backend returned HTTP {0}: {1}", response.statusCode(), response.body());- throw new AwsException("ModelErrorException",- "Proxy backend returned HTTP " + response.statusCode() + ": " + truncate(response.body(), 512),- 424);- }-- return new ProxyResult(response.body(), latencyMs);+ return builder.build();}String resolveModel(String bedrockModelId, EmulatorConfig.BedrockProxyConfig proxyConfig) {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.javaindex d517b95e57..a8126b9410 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java@@ -6,7 +6,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.io.OutputStream;import java.util.List;+import java.util.function.Consumer;/*** Dummy response builder for Bedrock Runtime. Stateless.@@ -78,32 +80,32 @@ public class StubBackend implements BedrockBackend {}@Override- public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {- ObjectNode messageStart = objectMapper.createObjectNode();- messageStart.put("role", "assistant");-- ObjectNode contentBlockDelta = objectMapper.createObjectNode();- contentBlockDelta.put("contentBlockIndex", 0);- contentBlockDelta.putObject("delta").put("text", "Floci stub response for model=" + modelId);-- ObjectNode contentBlockStop = objectMapper.createObjectNode();- contentBlockStop.put("contentBlockIndex", 0);-- ObjectNode messageStop = objectMapper.createObjectNode();- messageStop.put("stopReason", "end_turn");-- ObjectNode metadata = objectMapper.createObjectNode();- metadata.putObject("usage")- .put("inputTokens", 10)- .put("outputTokens", 12)- .put("totalTokens", 22);- metadata.putObject("metrics").put("latencyMs", 1);-- return BedrockStreamEncoder.encode(objectMapper, List.of(- new BedrockStreamEncoder.Event("messageStart", messageStart),- new BedrockStreamEncoder.Event("contentBlockDelta", contentBlockDelta),- new BedrockStreamEncoder.Event("contentBlockStop", contentBlockStop),- new BedrockStreamEncoder.Event("messageStop", messageStop),- new BedrockStreamEncoder.Event("metadata", metadata)));+ public Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest) {+ return output -> {+ BedrockStreamEncoder.writeEvent(objectMapper, output, "messageStart",+ objectMapper.createObjectNode().put("role", "assistant"));++ List<String> flociStubResponse = List.of("Floci stub response", " for model=", modelId);+ for (String fragment : flociStubResponse) {+ ObjectNode contentBlockDelta = objectMapper.createObjectNode();+ contentBlockDelta.put("contentBlockIndex", 0);+ contentBlockDelta.putObject("delta").put("text", fragment);+ BedrockStreamEncoder.writeEvent(objectMapper, output, "contentBlockDelta", contentBlockDelta);+ }++ BedrockStreamEncoder.writeEvent(objectMapper, output, "contentBlockStop",+ objectMapper.createObjectNode().put("contentBlockIndex", 0));++ BedrockStreamEncoder.writeEvent(objectMapper, output, "messageStop",+ objectMapper.createObjectNode().put("stopReason", "end_turn"));++ ObjectNode metadata = objectMapper.createObjectNode();+ metadata.putObject("usage")+ .put("inputTokens", 10)+ .put("outputTokens", 12)+ .put("totalTokens", 22);+ metadata.putObject("metrics").put("latencyMs", 1);+ BedrockStreamEncoder.writeEvent(objectMapper, output, "metadata", metadata);+ };}}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.javaindex d7808d1e26..39de2cb03a 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java@@ -775,14 +775,16 @@ class BedrockProxyIntegrationTest {}@Test- void converseStream_noUsableSseChunks_returns424InsteadOfFakeSuccess() {+ void converseStream_noUsableSseChunks_emitsInBandStreamError() {// A 2xx response with a body that isn't SSE-shaped at all (e.g. a plain JSON error- // object) must not be silently translated into a "successful" empty completion.+ // object) never reaches a finish_reason or "[DONE]". The HTTP status is already 200 by+ // the time messageStart is written, so the failure surfaces as an in-band+ // modelStreamErrorException event rather than an HTTP error status.nextResponseBody.set("""{"error": {"message": "model not found"}}""");- given()+ String body = given().contentType("application/json").header("Authorization", AUTH_HEADER).body("""@@ -791,16 +793,20 @@ class BedrockProxyIntegrationTest {.when().post("/model/" + MAPPED_MODEL_ID + "/converse-stream").then()- .statusCode(424)- .body("__type", equalTo("ModelErrorException"));+ .statusCode(200)+ .header("Content-Type", containsString("application/vnd.amazon.eventstream"))+ .extract().body().asString();++ assertThat(body, containsString("messageStart"));+ assertThat(body, containsString("modelStreamErrorException"));+ assertThat(body, org.hamcrest.Matchers.not(containsString("messageStop")));}@Test- void converseStream_parsableChunksWithNoContent_returns424InsteadOfFakeSuccess() {- // Every line here parses as valid JSON (parsedChunks > 0), but none of them carry- // actual completion content - a role-only chunk, then a bare finish_reason with no- // text or tool_calls. A finish_reason alone must not be treated as proof of a real- // completion; this is indistinguishable from a truncated/failed generation.+ void converseStream_completedWithNoContent_isValidEmptyCompletion() {+ // A role-only chunk, then a bare finish_reason with no text or tool_calls, then [DONE].+ // The stream properly completed (finish_reason and "[DONE]" were both seen) - real+ // Bedrock delivers this as a legitimate empty message, not an error.nextResponseBody.set("""data: {"choices":[{"delta":{"role":"assistant"}}]}@@ -809,7 +815,7 @@ class BedrockProxyIntegrationTest {data: [DONE]""");- given()+ String body = given().contentType("application/json").header("Authorization", AUTH_HEADER).body("""@@ -818,22 +824,30 @@ class BedrockProxyIntegrationTest {.when().post("/model/" + MAPPED_MODEL_ID + "/converse-stream").then()- .statusCode(424)- .body("__type", equalTo("ModelErrorException"));+ .statusCode(200)+ .extract().body().asString();++ assertThat(body, containsString("messageStart"));+ assertThat(body, containsString("messageStop"));+ assertThat(body, containsString("end_turn"));+ assertThat(body, containsString("metadata"));+ assertThat(body, org.hamcrest.Matchers.not(containsString("modelStreamErrorException")));+ assertThat(body, org.hamcrest.Matchers.not(containsString("contentBlockDelta")));}@Test- void converseStream_truncatedTextWithNoFinishReason_returns424() {+ void converseStream_truncatedTextWithNoFinishReason_emitsInBandStreamErrorAfterPartialText() {// A real text delta arrives, but the stream ends (EOF, no [DONE], no finish_reason) -- // e.g. the upstream connection dropped mid-generation. Accumulated text alone must not- // be treated as a completed answer.+ // e.g. the upstream connection dropped mid-generation. The already-streamed text and the+ // 200 status can't be undone, so the truncation is reported as a trailing in-band+ // modelStreamErrorException event instead of a clean messageStop.nextResponseBody.set("""data: {"choices":[{"delta":{"role":"assistant"}}]}data: {"choices":[{"delta":{"content":"Hello"}}]}""");- given()+ String body = given().contentType("application/json").header("Authorization", AUTH_HEADER).body("""@@ -842,15 +856,21 @@ class BedrockProxyIntegrationTest {.when().post("/model/" + MAPPED_MODEL_ID + "/converse-stream").then()- .statusCode(424)- .body("__type", equalTo("ModelErrorException"));+ .statusCode(200)+ .extract().body().asString();++ assertThat(body, containsString("messageStart"));+ assertThat(body, containsString("Hello"));+ assertThat(body, containsString("modelStreamErrorException"));+ assertThat(body, org.hamcrest.Matchers.not(containsString("messageStop")));}@Test- void converseStream_incompleteToolCallFragment_returns424() {- // The tool_calls delta never carries a "name" and its arguments never close into- // valid JSON - a partial/corrupted fragment, not a usable tool_use block. It must be- // dropped rather than surfacing as a 200 with an empty-name toolUse block.+ void converseStream_incompleteToolCallFragment_dropsFragmentAndCompletesNormally() {+ // The tool_calls delta never carries a "name" and its arguments never close into valid+ // JSON - a partial/corrupted fragment, not a usable tool_use block. It's dropped rather+ // than surfacing as a toolUse block with empty identity fields; since the stream did+ // reach "[DONE]"/finish_reason, it still completes normally (as an empty message).nextResponseBody.set("""data: {"choices":[{"delta":{"role":"assistant"}}]}@@ -861,7 +881,7 @@ class BedrockProxyIntegrationTest {data: [DONE]""");- given()+ String body = given().contentType("application/json").header("Authorization", AUTH_HEADER).body("""@@ -870,8 +890,14 @@ class BedrockProxyIntegrationTest {.when().post("/model/" + MAPPED_MODEL_ID + "/converse-stream").then()- .statusCode(424)- .body("__type", equalTo("ModelErrorException"));+ .statusCode(200)+ .extract().body().asString();++ assertThat(body, containsString("messageStop"));+ assertThat(body, containsString("end_turn"));+ assertThat(body, org.hamcrest.Matchers.not(containsString("call_1")));+ assertThat(body, org.hamcrest.Matchers.not(containsString("toolUse")));+ assertThat(body, org.hamcrest.Matchers.not(containsString("modelStreamErrorException")));}@Test