Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(bedrock-runtime): stream ConverseStream incrementally (#3043)

Scott Steen0471d6f488f4
diff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xml
index 14afc79ae0..188257bc69 100644
--- a/compatibility-tests/sdk-test-java/pom.xml
+++ b/compatibility-tests/sdk-test-java/pom.xml
@@ -337,6 +337,11 @@
<groupId>software.amazon.awssdk</groupId>
<artifactId>bedrockagentcore</artifactId>
</dependency>
+ <!-- Bedrock Runtime client (Converse/ConverseStream) -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bedrockruntime</artifactId>
+ </dependency>
<!-- AWS IoT client -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
index 7c09e3a095..fb85a938b6 100644
--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
@@ -68,6 +68,7 @@ import software.amazon.awssdk.services.ec2.Ec2Client;
import software.amazon.awssdk.services.ecr.EcrClient;
import software.amazon.awssdk.services.bedrockagentcore.BedrockAgentCoreClient;
import software.amazon.awssdk.services.bedrockagentcorecontrol.BedrockAgentCoreControlClient;
+import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient;
import software.amazon.awssdk.services.pipes.PipesClient;
import software.amazon.awssdk.services.codebuild.CodeBuildClient;
import software.amazon.awssdk.services.codedeploy.CodeDeployClient;
@@ -925,6 +926,16 @@ public final class TestFixtures {
.build();
}
+ public static BedrockRuntimeAsyncClient bedrockRuntimeAsyncClient() {
+ return BedrockRuntimeAsyncClient.builder()
+ .endpointOverride(ENDPOINT)
+ .region(REGION)
+ .credentialsProvider(CREDENTIALS)
+ .httpClientBuilder(NettyNioAsyncHttpClient.builder()
+ .protocol(Protocol.HTTP1_1))
+ .build();
+ }
+
public static IotClient iotClient() {
return IotClient.builder()
.endpointOverride(ENDPOINT)
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.java
new file mode 100644
index 0000000000..fb9351d31d
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/BedrockRuntimeConverseStreamTest.java
@@ -0,0 +1,70 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient;
+import software.amazon.awssdk.services.bedrockruntime.model.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Verifies Bedrock Runtime's ConverseStream against the real AWS SDK's event-stream decoder,
+ * not just Floci's own raw-byte assertions - proves the {@code application/vnd.amazon.eventstream}
+ * frames Floci emits actually round-trip through {@link BedrockRuntimeAsyncClient}'s generated
+ * unmarshaller end to end.
+ */
+@DisplayName("Bedrock Runtime ConverseStream")
+class BedrockRuntimeConverseStreamTest {
+
+ private static final String MODEL_ID = "anthropic.claude-3-haiku-20240307-v1:0";
+
+ @Test
+ void converseStream_deliversTextDeltasAndCompletes() throws Exception {
+ try (BedrockRuntimeAsyncClient client = TestFixtures.bedrockRuntimeAsyncClient()) {
+ List<ConverseStreamOutput> received = new ArrayList<>();
+ AtomicReference<Throwable> error = new AtomicReference<>();
+ CountDownLatch complete = new CountDownLatch(1);
+
+ ConverseStreamResponseHandler handler = ConverseStreamResponseHandler.builder()
+ .subscriber(received::add)
+ .onError(error::set)
+ .onComplete(complete::countDown)
+ .build();
+
+ ConverseStreamRequest request = ConverseStreamRequest.builder()
+ .modelId(MODEL_ID)
+ .messages(Message.builder()
+ .role(ConversationRole.USER)
+ .content(List.of(ContentBlock.fromText("hi")))
+ .build())
+ .build();
+
+ CompletableFuture<Void> future = client.converseStream(request, handler);
+ future.get(15, TimeUnit.SECONDS);
+
+ assertThat(complete.await(15, TimeUnit.SECONDS)).as("onComplete should fire").isTrue();
+ assertThat(error.get()).as("no stream error expected").isNull();
+ assertThat(received).isNotEmpty();
+ assertThat(received.get(0)).isInstanceOf(MessageStartEvent.class);
+
+ List<String> text = received.stream()
+ .filter(ContentBlockDeltaEvent.class::isInstance)
+ .map(ContentBlockDeltaEvent.class::cast)
+ .map(ContentBlockDeltaEvent::delta)
+ .map(ContentBlockDelta::text)
+ .toList();
+ assertThat(text)
+ .containsExactly("Floci stub response", " for model=", MODEL_ID);
+
+ assertThat(received).anyMatch(MessageStopEvent.class::isInstance);
+ }
+ }
+}
diff --git a/docs/services/bedrock-runtime.md b/docs/services/bedrock-runtime.md
index 0c38ee18c5..202dd0c270 100644
--- a/docs/services/bedrock-runtime.md
+++ b/docs/services/bedrock-runtime.md
@@ -18,7 +18,7 @@ The Bedrock management plane (`aws bedrock ...`: `ListFoundationModels`, `GetFou
|-----------|----------|-------|
| `Converse` | `POST /model/{modelId}/converse` | `stub`: returns a static assistant message. `proxy`: forwards to the configured OpenAI-compatible backend and translates the reply. |
| `InvokeModel` | `POST /model/{modelId}/invoke` | Returns Anthropic-shaped body for `anthropic.*` and `*.anthropic.*` model ids; generic `{"outputs": [...]}` shape otherwise. Not proxied yet — always the stub response, regardless of backend. |
-| `ConverseStream` | `POST /model/{modelId}/converse-stream` | `stub`: emits a static assistant turn as `messageStart`/`contentBlockDelta`/`contentBlockStop`/`messageStop`/`metadata` events. `proxy`: forwards a streaming request to the configured OpenAI-compatible backend and translates the SSE response into the same event sequence. Response body is `application/vnd.amazon.eventstream`-framed, built in one shot rather than incrementally chunked. |
+| `ConverseStream` | `POST /model/{modelId}/converse-stream` | `stub`: emits a static assistant turn as `messageStart`/`contentBlockDelta`/`contentBlockStop`/`messageStop`/`metadata` events. `proxy`: forwards a streaming request to the configured OpenAI-compatible backend and translates each SSE chunk into a Bedrock event as it arrives. Text deltas reach the client incrementally, not all at once. Response body is `application/vnd.amazon.eventstream`-framed. If the upstream stream ends without a `finish_reason` or `"[DONE]"` (e.g. a dropped connection), the failure is reported as an in-band `modelStreamErrorException` event, since the HTTP 200 is already committed by the time streaming starts, matching real Bedrock's own behavior for a mid-stream failure. |
| `InvokeModelWithResponseStream` | `POST /model/{modelId}/invoke-with-response-stream` | Returns 501 `UnsupportedOperationException` |
`modelId` is URL-decoded by JAX-RS and echoed verbatim. Plain model ids (e.g. `anthropic.claude-3-haiku-20240307-v1:0`), inference-profile ids (e.g. `us.anthropic.claude-3-5-sonnet-20241022-v2:0`), and full ARNs containing slashes (e.g. `arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0`) are all accepted.
@@ -89,6 +89,6 @@ print(resp["output"]["message"]["content"][0]["text"])
- Real model inference with the `stub` backend (always returns a fixed string).
- `InvokeModel` against the `proxy` backend (still returns the stub response).
- `InvokeModelWithResponseStream` returns 501, regardless of backend.
-- `ConverseStream` responses are built in one shot and returned as a complete event-stream body, not incrementally chunked over the wire.
+- `ConverseStream` tool calls are still accumulated across chunks and emitted as a single complete `contentBlockDelta` once the stream ends, rather than as incremental argument fragments — SDKs consume both the same way, since they concatenate `delta.toolUse.input` across events for a given `contentBlockIndex` before parsing it as JSON.
- Bedrock management plane (`ListFoundationModels`, `GetFoundationModel`, model customisation).
- Bedrock Agents, Knowledge Bases, Guardrails, provisioned throughput.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java
index 58ec858d6a..65e3d84be8 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsEventStreamEncoder.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.core.common;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -9,7 +10,7 @@ import java.util.zip.CRC32;
public class AwsEventStreamEncoder {
- public static byte[] encodeMessage(LinkedHashMap<String, String> headers, byte[] payload) throws Exception {
+ public static byte[] encodeMessage(LinkedHashMap<String, String> headers, byte[] payload) throws IOException {
byte[] headersBytes = encodeHeaders(headers);
int totalLen = 4 + 4 + 4 + headersBytes.length + payload.length + 4;
@@ -35,7 +36,7 @@ public class AwsEventStreamEncoder {
return buf.toByteArray();
}
- private static byte[] encodeHeaders(LinkedHashMap<String, String> headers) throws Exception {
+ private static byte[] encodeHeaders(LinkedHashMap<String, String> headers) throws IOException {
ByteArrayOutputStream h = new ByteArrayOutputStream();
for (Map.Entry<String, String> e : headers.entrySet()) {
byte[] name = e.getKey().getBytes(StandardCharsets.UTF_8);
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java
index f2bfa9a529..579183e4f8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java
@@ -11,10 +11,15 @@ import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.GenericEntity;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.StreamingOutput;
import org.jboss.logging.Logger;
+import java.io.OutputStream;
+import java.util.function.Consumer;
+
/**
* AWS Bedrock Runtime REST JSON endpoints.
*
@@ -87,13 +92,17 @@ public class BedrockRuntimeController {
JsonNode messages = validateMessages(request);
- byte[] response = service.buildConverseStreamResponse(modelId, request);
+ Consumer<OutputStream> stream = service.buildConverseStreamResponse(modelId, request);
LOG.debugv("Bedrock Converse Stream: modelId={0}, messages={1}", modelId, messages.size());
- return Response.ok(response)
+ return Response.ok(streaming(stream))
.header("Content-Type", "application/vnd.amazon.eventstream")
.build();
}
+ private static GenericEntity<StreamingOutput> streaming(Consumer<OutputStream> stream) {
+ return new GenericEntity<>(stream::accept, StreamingOutput.class);
+ }
+
private static JsonNode validateMessages(ObjectNode request) {
JsonNode messages = request.path("messages");
if (!messages.isArray() || messages.isEmpty()) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java
index 2edcd9f4ea..8e65712db9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java
@@ -9,6 +9,9 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.OutputStream;
+import java.util.function.Consumer;
+
/**
* Thin orchestration layer for Bedrock Runtime: delegates Converse/InvokeModel to the
* backend selected by {@code floci.services.bedrock-runtime.backend}.
@@ -37,7 +40,7 @@ public class BedrockRuntimeService {
return backend().invokeModel(modelId, body);
}
- public byte[] buildConverseStreamResponse(String modelId, ObjectNode bedrockRequest) {
+ public Consumer<OutputStream> buildConverseStreamResponse(String modelId, ObjectNode bedrockRequest) {
return backend().converseStream(modelId, bedrockRequest);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java
index 18c390822d..c6a35a7a97 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java
@@ -1,6 +1,10 @@
package io.github.hectorvent.floci.services.bedrockruntime.backend;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import jakarta.ws.rs.core.StreamingOutput;
+
+import java.io.OutputStream;
+import java.util.function.Consumer;
/**
* Executes Bedrock Runtime Converse/InvokeModel requests against a concrete backend
@@ -12,5 +16,5 @@ public interface BedrockBackend {
byte[] invokeModel(String modelId, byte[] body);
- byte[] converseStream(String modelId, ObjectNode bedrockRequest);
+ Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java
index 489ee163a6..15f1659335 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java
@@ -7,10 +7,13 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import org.jboss.logging.Logger;
+import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Stream;
/**
* Translates between the Bedrock Runtime Converse wire shape and the OpenAI
@@ -259,22 +262,27 @@ final class BedrockOpenAiTranslator {
}
/**
- * Translates an OpenAI Chat Completions SSE stream body ({@code data: {...}\n\n} lines,
- * terminated by {@code data: [DONE]}) into the ordered Bedrock ConverseStream events
- * (messageStart, contentBlockDelta, contentBlockStop, messageStop, metadata), framed as
- * {@code application/vnd.amazon.eventstream} bytes via {@link BedrockStreamEncoder}.
+ * Translates an OpenAI Chat Completions SSE stream ({@code data: {...}} lines, terminated by
+ * {@code data: [DONE]}) into Bedrock ConverseStream events, writing each frame to {@code out}
+ * as soon as it's known instead of buffering the whole response - text deltas reach the
+ * client as they arrive from the upstream.
+ *
+ * <p>By the time this runs, JAX-RS has already committed the 200 response (messageStart is
+ * about to be written), so a failure discovered here - a truncated stream that never reaches
+ * a finish_reason or "[DONE]" - can no longer become an HTTP error status. It's written as an
+ * in-band {@code modelStreamErrorException} event instead, matching how real Bedrock reports
+ * a mid-stream failure after the response has already started.
*
* <p>Tool calls are accumulated across chunks and emitted as a single complete
- * contentBlockDelta once the stream ends, rather than as incremental argument
- * fragments - simpler than real Bedrock's fragment-by-fragment streaming, but
- * SDKs consume both the same way since they just concatenate delta.toolUse.input
- * across events for a given contentBlockIndex before parsing it as JSON.
+ * contentBlockDelta once the stream ends, rather than as incremental argument fragments -
+ * simpler than real Bedrock's fragment-by-fragment streaming, but SDKs consume both the same
+ * way since they just concatenate delta.toolUse.input across events for a given
+ * contentBlockIndex before parsing it as JSON.
*/
- static byte[] toBedrockStreamEvents(ObjectMapper mapper, String sseBody, long latencyMs) {
- List<BedrockStreamEncoder.Event> events = new ArrayList<>();
- events.add(new BedrockStreamEncoder.Event("messageStart", mapper.createObjectNode().put("role", "assistant")));
+ static void streamBedrockEvents(ObjectMapper mapper, Stream<String> sseLines, OutputStream out, long startNanos) {
+ BedrockStreamEncoder.writeEvent(mapper, out, "messageStart", mapper.createObjectNode().put("role", "assistant"));
- StringBuilder text = new StringBuilder();
+ boolean anyContentDeltaWritten = false;
Map<Integer, ToolCallAccumulator> toolCalls = new LinkedHashMap<>();
String finishReason = "stop";
boolean sawFinishReason = false;
@@ -282,76 +290,123 @@ final class BedrockOpenAiTranslator {
int promptTokens = 0;
int completionTokens = 0;
int totalTokens = 0;
- int parsedChunks = 0;
- for (String rawLine : sseBody.split("\n")) {
- String line = rawLine.strip();
- if (!line.startsWith("data:")) {
- continue;
- }
- String data = line.substring(5).strip();
- if (data.isEmpty()) {
- continue;
- }
- if ("[DONE]".equals(data)) {
- sawDone = true;
- continue;
- }
- JsonNode chunk;
- try {
- chunk = mapper.readTree(data);
- } catch (Exception e) {
- LOG.warnv("Skipping malformed SSE chunk from proxy backend: {0}", e.getMessage());
- continue;
- }
- parsedChunks++;
+ try (sseLines) {
+ Iterator<String> lineIterator = sseLines.iterator();
+ String rawLine;
+ while ((rawLine = readLine(lineIterator)) != null) {
+ String line = rawLine.strip();
+ if (!line.startsWith("data:")) {
+ continue;
+ }
+ String data = line.substring(5).strip();
+ if (data.isEmpty()) {
+ continue;
+ }
+ if ("[DONE]".equals(data)) {
+ sawDone = true;
+ continue;
+ }
+ JsonNode chunk;
+ try {
+ chunk = mapper.readTree(data);
+ } catch (Exception e) {
+ LOG.warnv("Skipping malformed SSE chunk from proxy backend: {0}", e.getMessage());
+ continue;
+ }
- JsonNode usage = chunk.path("usage");
- if (usage.isObject()) {
- promptTokens = usage.path("prompt_tokens").asInt(promptTokens);
- completionTokens = usage.path("completion_tokens").asInt(completionTokens);
- totalTokens = usage.path("total_tokens").asInt(totalTokens);
- }
+ JsonNode usage = chunk.path("usage");
+ if (usage.isObject()) {
+ promptTokens = usage.path("prompt_tokens").asInt(promptTokens);
+ completionTokens = usage.path("completion_tokens").asInt(completionTokens);
+ totalTokens = usage.path("total_tokens").asInt(totalTokens);
+ }
- JsonNode choice = chunk.path("choices").path(0);
- String reason = choice.path("finish_reason").asText(null);
- if (reason != null) {
- finishReason = reason;
- sawFinishReason = true;
- }
+ JsonNode choice = chunk.path("choices").path(0);
+ String reason = choice.path("finish_reason").asText(null);
+ if (reason != null) {
+ finishReason = reason;
+ sawFinishReason = true;
+ }
- JsonNode delta = choice.path("delta");
- String contentDelta = delta.path("content").asText(null);
- if (contentDelta != null && !contentDelta.isEmpty()) {
- text.append(contentDelta);
- ObjectNode deltaEvent = mapper.createObjectNode();
- deltaEvent.put("contentBlockIndex", 0);
- deltaEvent.putObject("delta").put("text", contentDelta);
- events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));
+ JsonNode delta = choice.path("delta");
+ if (writeContentDelta(mapper, out, delta)) {
+ anyContentDeltaWritten = true;
+ }
+ writeToolCalls(delta, toolCalls);
}
+ }
- JsonNode toolCallDeltas = delta.path("tool_calls");
- if (toolCallDeltas.isArray()) {
- for (JsonNode toolCallDelta : toolCallDeltas) {
- int index = toolCallDelta.path("index").asInt(0);
- ToolCallAccumulator acc = toolCalls.computeIfAbsent(index, i -> new ToolCallAccumulator());
- String id = toolCallDelta.path("id").asText(null);
- if (id != null) {
- acc.id = id;
- }
- JsonNode function = toolCallDelta.path("function");
- String name = function.path("name").asText(null);
- if (name != null) {
- acc.name = name;
- }
- String argsFragment = function.path("arguments").asText(null);
- if (argsFragment != null) {
- acc.arguments.append(argsFragment);
- }
- }
+ boolean streamCompleted = sawFinishReason || sawDone;
+ if (!streamCompleted) {
+ writeTruncatedException(mapper, out);
+ return;
+ }
+
+ if (anyContentDeltaWritten) {
+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStop",
+ mapper.createObjectNode().put("contentBlockIndex", 0));
+ }
+
+ var validToolCalls = collectValidToolCalls(mapper, toolCalls);
+ int blockIndex = anyContentDeltaWritten ? 1 : 0;
+ for (ToolCallAccumulator acc : validToolCalls) {
+ writeToolCalls(mapper, out, acc, blockIndex);
+ blockIndex++;
+ }
+
+ boolean hasToolCalls = !validToolCalls.isEmpty();
+ BedrockStreamEncoder.writeEvent(mapper, out, "messageStop",
+ mapper.createObjectNode().put("stopReason", hasToolCalls ? "tool_use" : mapFinishReason(finishReason)));
+
+ writeMetadata(mapper, out, startNanos, promptTokens, completionTokens, totalTokens);
+ }
+
+ private static String readLine(Iterator<String> lineIterator) {
+ try {
+ if (!lineIterator.hasNext()) {
+ return null;
}
+ return lineIterator.next();
+ } catch (RuntimeException e) {
+ // The upstream connection failed mid-read (e.g. it dropped the connection).
+ // Treat exactly like reaching EOF without a finish_reason/[DONE]: fall through
+ // to the truncation handling instead of letting an UncheckedIOException
+ // propagate and abruptly kill the response mid-stream.
+ LOG.warnv(e, "Bedrock proxy backend connection failed while streaming");
+ return null;
}
+ }
+ private static void writeMetadata(ObjectMapper mapper, OutputStream out, long startNanos, int promptTokens, int completionTokens, int totalTokens) {
+ long latencyMs = (System.nanoTime() - startNanos) / 1_000_000;
+ ObjectNode metadata = mapper.createObjectNode();
+ metadata.putObject("usage")
+ .put("inputTokens", promptTokens)
+ .put("outputTokens", completionTokens)
+ .put("totalTokens", totalTokens);
+ metadata.putObject("metrics").put("latencyMs", latencyMs);
+ BedrockStreamEncoder.writeEvent(mapper, out, "metadata", metadata);
+ }
+
+ private static void writeToolCalls(ObjectMapper mapper, OutputStream out, ToolCallAccumulator acc, int blockIndex) {
+ ObjectNode startEvent = mapper.createObjectNode();
+ startEvent.put("contentBlockIndex", blockIndex);
+ ObjectNode toolUseStart = startEvent.putObject("start").putObject("toolUse");
+ toolUseStart.put("toolUseId", acc.id);
+ toolUseStart.put("name", acc.name);
+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStart", startEvent);
+
+ ObjectNode deltaEvent = mapper.createObjectNode();
+ deltaEvent.put("contentBlockIndex", blockIndex);
+ deltaEvent.putObject("delta").putObject("toolUse").put("input", acc.arguments.toString());
+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockDelta", deltaEvent);
+
+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockStop",
+ mapper.createObjectNode().put("contentBlockIndex", blockIndex));
+ }
+
+ private static List<ToolCallAccumulator> collectValidToolCalls(ObjectMapper mapper, Map<Integer, ToolCallAccumulator> toolCalls) {
// A tool_calls delta only becomes a usable toolUse block once it has a non-blank id,
// a non-blank name, and arguments that actually form a JSON object - a partial or
// truncated fragment (e.g. an id with no name, or unclosed argument JSON) is dropped
@@ -365,59 +420,54 @@ final class BedrockOpenAiTranslator {
acc.id, acc.name);
}
}
+ return validToolCalls;
+ }
- boolean hasContent = text.length() > 0 || !validToolCalls.isEmpty();
- boolean streamCompleted = sawFinishReason || sawDone;
- if (!hasContent || !streamCompleted) {
- // Either no usable completion content was ever produced (empty/garbled body,
- // role/usage/finish_reason-only records, a data:-prefixed error object, or only
- // invalid tool_calls fragments), or the stream ended without a finish_reason or a
- // "[DONE]" terminator at all - a truncated generation (e.g. the connection dropped
- // mid-stream). Neither case is a real completion: surface it as a backend failure
- // rather than fabricating a successful message out of whatever partial data arrived.
- throw new AwsException("ModelErrorException",
- "Proxy backend's ConverseStream response did not complete ("
- + parsedChunks + " SSE chunk(s) parsed, streamCompleted=" + streamCompleted
- + "): " + truncate(sseBody, 512), 424);
- }
+ private static void writeTruncatedException(ObjectMapper mapper, OutputStream out) {
+ // The stream ended (upstream closed the connection, or we ran out of lines) without
+ // ever reaching a finish_reason or "[DONE]" - a truncated generation. The HTTP status
+ // is already 200, so this can only be reported in-band.
+ ObjectNode errorPayload = mapper.createObjectNode();
+ errorPayload.put("message", "Proxy backend's ConverseStream response ended without a finish_reason "
+ + "or \"[DONE]\" terminator - the stream may have been truncated.");
+ BedrockStreamEncoder.writeException(mapper, out, "modelStreamErrorException", errorPayload);
+ }
- if (text.length() > 0) {
- events.add(new BedrockStreamEncoder.Event("contentBlockStop",
- mapper.createObjectNode().put("contentBlockIndex", 0)));
+ private static void writeToolCalls(JsonNode delta, Map<Integer, ToolCallAccumulator> toolCalls) {
+ JsonNode toolCallDeltas = delta.path("tool_calls");
+ if (!toolCallDeltas.isArray()) {
+ return;
}
-
- boolean hasToolCalls = !validToolCalls.isEmpty();
- int blockIndex = text.length() > 0 ? 1 : 0;
- for (ToolCallAccumulator acc : validToolCalls) {
- ObjectNode startEvent = mapper.createObjectNode();
- startEvent.put("contentBlockIndex", blockIndex);
- ObjectNode toolUseStart = startEvent.putObject("start").putObject("toolUse");
- toolUseStart.put("toolUseId", acc.id);
- toolUseStart.put("name", acc.name);
- events.add(new BedrockStreamEncoder.Event("contentBlockStart", startEvent));
-
- ObjectNode deltaEvent = mapper.createObjectNode();
- deltaEvent.put("contentBlockIndex", blockIndex);
- deltaEvent.putObject("delta").putObject("toolUse").put("input", acc.arguments.toString());
- events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));
-
- events.add(new BedrockStreamEncoder.Event("contentBlockStop",
- mapper.createObjectNode().put("contentBlockIndex", blockIndex)));
- blockIndex++;
+ for (JsonNode toolCallDelta : toolCallDeltas) {
+ int index = toolCallDelta.path("index").asInt(0);
+ ToolCallAccumulator acc = toolCalls.computeIfAbsent(index, i -> new ToolCallAccumulator());
+ String id = toolCallDelta.path("id").asText(null);
+ if (id != null) {
+ acc.id = id;
+ }
+ JsonNode function = toolCallDelta.path("function");
+ String name = function.path("name").asText(null);
+ if (name != null) {
+ acc.name = name;
+ }
+ String argsFragment = function.path("arguments").asText(null);
+ if (argsFragment != null) {
+ acc.arguments.append(argsFragment);
+ }
}
+ }
- events.add(new BedrockStreamEncoder.Event("messageStop",
- mapper.createObjectNode().put("stopReason", hasToolCalls ? "tool_use" : mapFinishReason(finishReason))));
-
- ObjectNode metadata = mapper.createObjectNode();
- metadata.putObject("usage")
- .put("inputTokens", promptTokens)
- .put("outputTokens", completionTokens)
- .put("totalTokens", totalTokens);
- metadata.putObject("metrics").put("latencyMs", latencyMs);
- events.add(new BedrockStreamEncoder.Event("metadata", metadata));
+ private static boolean writeContentDelta(ObjectMapper mapper, OutputStream out, JsonNode delta) {
+ String contentDelta = delta.path("content").asText(null);
+ if (contentDelta == null || contentDelta.isEmpty()) {
+ return false;
+ }
- return BedrockStreamEncoder.encode(mapper, events);
+ ObjectNode deltaEvent = mapper.createObjectNode();
+ deltaEvent.put("contentBlockIndex", 0);
+ deltaEvent.putObject("delta").put("text", contentDelta);
+ BedrockStreamEncoder.writeEvent(mapper, out, "contentBlockDelta", deltaEvent);
+ return true;
}
private static boolean isUsableToolCall(ObjectMapper mapper, ToolCallAccumulator acc) {
@@ -442,20 +492,21 @@ final class BedrockOpenAiTranslator {
if (content.isTextual()) {
return content.asText("");
}
- if (content.isArray()) {
- StringBuilder sb = new StringBuilder();
- for (JsonNode part : content) {
- String text = part.path("text").asText(null);
- if (text != null) {
- if (sb.length() > 0) {
- sb.append('\n');
- }
- sb.append(text);
- }
+ if (!content.isArray()) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (JsonNode part : content) {
+ String text = part.path("text").asText(null);
+ if (text == null) {
+ continue;
}
- return sb.toString();
+ if (!sb.isEmpty()) {
+ sb.append('\n');
+ }
+ sb.append(text);
}
- return "";
+ return sb.toString();
}
private static ObjectNode parseToolArguments(ObjectMapper mapper, String argumentsJson) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java
index adb0403ba6..e85f0d7749 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java
@@ -3,40 +3,51 @@ package io.github.hectorvent.floci.services.bedrockruntime.backend;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;
-import io.github.hectorvent.floci.core.common.AwsException;
-import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
import java.util.LinkedHashMap;
-import java.util.List;
/**
- * Frames a sequence of Bedrock ConverseStream events (messageStart, contentBlockDelta, ...)
- * as {@code application/vnd.amazon.eventstream} messages, one per event, concatenated into
- * a single response body - the same one-shot framing S3 Select and Kinesis SubscribeToShard
- * use elsewhere in Floci, rather than true incremental HTTP chunking.
+ * Frames Bedrock ConverseStream events (messageStart, contentBlockDelta, ...) as
+ * {@code application/vnd.amazon.eventstream} messages and writes each one to the response
+ * {@link OutputStream} as soon as it's known, so SDK clients see incremental delivery instead
+ * of the whole response landing at once - the same binary framing S3 Select and Kinesis
+ * SubscribeToShard build elsewhere in Floci, just written frame-by-frame instead of batched.
*/
final class BedrockStreamEncoder {
private BedrockStreamEncoder() {
}
- record Event(String type, ObjectNode payload) {
+ /** Writes a normal event frame ({@code :message-type: event}), e.g. messageStart, contentBlockDelta. */
+ static void writeEvent(ObjectMapper mapper, OutputStream out, String eventType, ObjectNode payload) {
+ writeFrame(mapper, out, "event", ":event-type", eventType, payload);
}
- static byte[] encode(ObjectMapper mapper, List<Event> events) {
+ /**
+ * Writes a modeled-exception frame ({@code :message-type: exception}) - the wire shape real
+ * Bedrock uses to report a stream failure after the HTTP 200 response has already been
+ * committed. The AWS SDK's generic JSON error unmarshaller reads {@code :exception-type} to
+ * pick the matching modeled exception class (e.g. ModelStreamErrorException), the same way
+ * it reads {@code x-amzn-ErrorType} for a regular non-streaming error response.
+ */
+ static void writeException(ObjectMapper mapper, OutputStream out, String exceptionType, ObjectNode payload) {
+ writeFrame(mapper, out, "exception", ":exception-type", exceptionType, payload);
+ }
+
+ private static void writeFrame(ObjectMapper mapper, OutputStream out, String messageType, String typeHeaderName,
+ String typeValue, ObjectNode payload) {
+ LinkedHashMap<String, String> headers = new LinkedHashMap<>();
+ headers.put(":message-type", messageType);
+ headers.put(typeHeaderName, typeValue);
+ headers.put(":content-type", "application/json");
try {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- for (Event event : events) {
- LinkedHashMap<String, String> headers = new LinkedHashMap<>();
- headers.put(":message-type", "event");
- headers.put(":event-type", event.type());
- headers.put(":content-type", "application/json");
- out.write(AwsEventStreamEncoder.encodeMessage(headers, mapper.writeValueAsBytes(event.payload())));
- }
- return out.toByteArray();
- } catch (Exception e) {
- throw new AwsException("InternalServerException",
- "Failed to encode ConverseStream response: " + e.getMessage(), 500);
+ out.write(AwsEventStreamEncoder.encodeMessage(headers, mapper.writeValueAsBytes(payload)));
+ out.flush();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
index 9f50142c90..e593f4bcd7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java
@@ -9,6 +9,7 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.OutputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@@ -17,6 +18,9 @@ import java.net.http.HttpTimeoutException;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* Forwards Bedrock Converse requests to any OpenAI-compatible {@code /chat/completions}
@@ -52,16 +56,24 @@ public class ProxyBackend implements BedrockBackend {
EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();
String resolvedModel = resolveModel(modelId, proxyConfig);
ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, false);
- ProxyResult result = send(modelId, proxyConfig, openAiRequest);
+ HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);
+
+ long start = System.nanoTime();
+ HttpResponse<String> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofString());
+ long latencyMs = (System.nanoTime() - start) / 1_000_000;
+
+ if (response.statusCode() >= 300) {
+ throw failedRequest(response.statusCode(), response.body());
+ }
JsonNode openAiResponse;
try {
- openAiResponse = objectMapper.readTree(result.body());
+ openAiResponse = objectMapper.readTree(response.body());
} catch (Exception e) {
throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);
}
- return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, result.latencyMs());
+ return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, latencyMs);
}
@Override
@@ -71,30 +83,71 @@ public class ProxyBackend implements BedrockBackend {
}
@Override
- public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {
+ public Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest) {
EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();
String resolvedModel = resolveModel(modelId, proxyConfig);
ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, true);
- ProxyResult result = send(modelId, proxyConfig, openAiRequest);
- return BedrockOpenAiTranslator.toBedrockStreamEvents(objectMapper, result.body(), result.latencyMs());
+ // Everything that can fail with a proper HTTP status - resolving the model, opening the
+ // connection, checking the upstream status code - happens here, before any bytes of our
+ // own response are written. Once the returned Consumer is invoked, JAX-RS has
+ // already committed a 200, so failures discovered while consuming the body stream can
+ // only be reported as an in-band Bedrock stream exception event (see
+ // BedrockOpenAiTranslator.streamBedrockEvents).
+ HttpRequest request = buildHttpRequest(proxyConfig, openAiRequest);
+ long startNanos = System.nanoTime();
+ HttpResponse<Stream<String>> response = invoke(modelId, request, HttpResponse.BodyHandlers.ofLines());
+
+ if (response.statusCode() >= 300) {
+ String errorBody;
+ try (Stream<String> body = response.body()) {
+ errorBody = body.limit(50).collect(Collectors.joining("\n"));
+ }
+ throw failedRequest(response.statusCode(), errorBody);
+ }
+
+ return output -> BedrockOpenAiTranslator.streamBedrockEvents(objectMapper, response.body(), output, startNanos);
}
- private record ProxyResult(String body, long latencyMs) {
+ private static AwsException failedRequest(int response, String errorBody) {
+ LOG.warnv("Bedrock proxy backend returned HTTP {0}: {1}", response, errorBody);
+ return new AwsException("ModelErrorException",
+ "Proxy backend returned HTTP %d: %s".formatted(response, truncate(errorBody, 512)), 424);
}
- private ProxyResult send(String modelId, EmulatorConfig.BedrockProxyConfig proxyConfig, ObjectNode openAiRequest) {
+ private <T> HttpResponse<T> invoke(String modelId, HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler) {
+ try {
+ return httpClient.send(request, bodyHandler);
+ } catch (HttpTimeoutException e) {
+ LOG.warnv("Bedrock proxy backend timed out: modelId={0}, url={1}, error={2}",
+ modelId, request.uri(), e.getMessage());
+ throw new AwsException("ModelTimeoutException", "Proxy backend timed out: " + e.getMessage(), 408);
+ } catch (Exception e) {
+ LOG.warnv("Bedrock proxy backend call failed: modelId={0}, url={1}, error={2}",
+ modelId, request.uri(), e.getMessage());
+ throw new AwsException("ModelErrorException", "Failed to reach proxy backend: " + e.getMessage(), 424);
+ }
+ }
+
+ private HttpRequest buildHttpRequest(EmulatorConfig.BedrockProxyConfig proxyConfig, ObjectNode openAiRequest) {
String baseUrl = proxyConfig.url()
.filter(url -> !url.isBlank())
.orElseThrow(() -> new AwsException("ValidationException",
"floci.services.bedrock-runtime.proxy.url is required when backend=proxy.", 400));
- URI uri;
+ byte[] requestBody;
+ try {
+ requestBody = objectMapper.writeValueAsBytes(openAiRequest);
+ } catch (Exception e) {
+ throw new AwsException("InternalServerException", "Failed to serialize proxy request: " + e.getMessage(), 500);
+ }
+
HttpRequest.Builder builder;
try {
- uri = URI.create(stripTrailingSlash(baseUrl) + "/chat/completions");
+ URI uri = URI.create(stripTrailingSlash(baseUrl) + "/chat/completions");
builder = HttpRequest.newBuilder()
.uri(uri)
+ .POST(HttpRequest.BodyPublishers.ofByteArray(requestBody))
.timeout(Duration.ofSeconds(proxyConfig.requestTimeoutSeconds()))
.header("Content-Type", "application/json");
} catch (IllegalArgumentException e) {
@@ -105,35 +158,7 @@ public class ProxyBackend implements BedrockBackend {
.filter(key -> !key.isBlank())
.ifPresent(key -> builder.header("Authorization", "Bearer " + key));
- String requestBody;
- try {
- requestBody = objectMapper.writeValueAsString(openAiRequest);
- } catch (Exception e) {
- throw new AwsException("InternalServerException", "Failed to serialize proxy request: " + e.getMessage(), 500);
- }
- builder.POST(HttpRequest.BodyPublishers.ofString(requestBody));
-
- long start = System.nanoTime();
- HttpResponse<String> response;
- try {
- response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString());
- } catch (HttpTimeoutException e) {
- LOG.warnv("Bedrock proxy backend timed out: modelId={0}, url={1}, error={2}", modelId, uri, e.getMessage());
- throw new AwsException("ModelTimeoutException", "Proxy backend timed out: " + e.getMessage(), 408);
- } catch (Exception e) {
- LOG.warnv("Bedrock proxy backend call failed: modelId={0}, url={1}, error={2}", modelId, uri, e.getMessage());
- throw new AwsException("ModelErrorException", "Failed to reach proxy backend: " + e.getMessage(), 424);
- }
- long latencyMs = (System.nanoTime() - start) / 1_000_000;
-
- if (response.statusCode() >= 300) {
- LOG.warnv("Bedrock proxy backend returned HTTP {0}: {1}", response.statusCode(), response.body());
- throw new AwsException("ModelErrorException",
- "Proxy backend returned HTTP " + response.statusCode() + ": " + truncate(response.body(), 512),
- 424);
- }
-
- return new ProxyResult(response.body(), latencyMs);
+ return builder.build();
}
String resolveModel(String bedrockModelId, EmulatorConfig.BedrockProxyConfig proxyConfig) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java
index d517b95e57..a8126b9410 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java
@@ -6,7 +6,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.io.OutputStream;
import java.util.List;
+import java.util.function.Consumer;
/**
* Dummy response builder for Bedrock Runtime. Stateless.
@@ -78,32 +80,32 @@ public class StubBackend implements BedrockBackend {
}
@Override
- public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {
- ObjectNode messageStart = objectMapper.createObjectNode();
- messageStart.put("role", "assistant");
-
- ObjectNode contentBlockDelta = objectMapper.createObjectNode();
- contentBlockDelta.put("contentBlockIndex", 0);
- contentBlockDelta.putObject("delta").put("text", "Floci stub response for model=" + modelId);
-
- ObjectNode contentBlockStop = objectMapper.createObjectNode();
- contentBlockStop.put("contentBlockIndex", 0);
-
- ObjectNode messageStop = objectMapper.createObjectNode();
- messageStop.put("stopReason", "end_turn");
-
- ObjectNode metadata = objectMapper.createObjectNode();
- metadata.putObject("usage")
- .put("inputTokens", 10)
- .put("outputTokens", 12)
- .put("totalTokens", 22);
- metadata.putObject("metrics").put("latencyMs", 1);
-
- return BedrockStreamEncoder.encode(objectMapper, List.of(
- new BedrockStreamEncoder.Event("messageStart", messageStart),
- new BedrockStreamEncoder.Event("contentBlockDelta", contentBlockDelta),
- new BedrockStreamEncoder.Event("contentBlockStop", contentBlockStop),
- new BedrockStreamEncoder.Event("messageStop", messageStop),
- new BedrockStreamEncoder.Event("metadata", metadata)));
+ public Consumer<OutputStream> converseStream(String modelId, ObjectNode bedrockRequest) {
+ return output -> {
+ BedrockStreamEncoder.writeEvent(objectMapper, output, "messageStart",
+ objectMapper.createObjectNode().put("role", "assistant"));
+
+ List<String> flociStubResponse = List.of("Floci stub response", " for model=", modelId);
+ for (String fragment : flociStubResponse) {
+ ObjectNode contentBlockDelta = objectMapper.createObjectNode();
+ contentBlockDelta.put("contentBlockIndex", 0);
+ contentBlockDelta.putObject("delta").put("text", fragment);
+ BedrockStreamEncoder.writeEvent(objectMapper, output, "contentBlockDelta", contentBlockDelta);
+ }
+
+ BedrockStreamEncoder.writeEvent(objectMapper, output, "contentBlockStop",
+ objectMapper.createObjectNode().put("contentBlockIndex", 0));
+
+ BedrockStreamEncoder.writeEvent(objectMapper, output, "messageStop",
+ objectMapper.createObjectNode().put("stopReason", "end_turn"));
+
+ ObjectNode metadata = objectMapper.createObjectNode();
+ metadata.putObject("usage")
+ .put("inputTokens", 10)
+ .put("outputTokens", 12)
+ .put("totalTokens", 22);
+ metadata.putObject("metrics").put("latencyMs", 1);
+ BedrockStreamEncoder.writeEvent(objectMapper, output, "metadata", metadata);
+ };
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java
index d7808d1e26..39de2cb03a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java
@@ -775,14 +775,16 @@ class BedrockProxyIntegrationTest {
}
@Test
- void converseStream_noUsableSseChunks_returns424InsteadOfFakeSuccess() {
+ void converseStream_noUsableSseChunks_emitsInBandStreamError() {
// A 2xx response with a body that isn't SSE-shaped at all (e.g. a plain JSON error
- // object) must not be silently translated into a "successful" empty completion.
+ // object) never reaches a finish_reason or "[DONE]". The HTTP status is already 200 by
+ // the time messageStart is written, so the failure surfaces as an in-band
+ // modelStreamErrorException event rather than an HTTP error status.
nextResponseBody.set("""
{"error": {"message": "model not found"}}
""");
- given()
+ String body = given()
.contentType("application/json")
.header("Authorization", AUTH_HEADER)
.body("""
@@ -791,16 +793,20 @@ class BedrockProxyIntegrationTest {
.when()
.post("/model/" + MAPPED_MODEL_ID + "/converse-stream")
.then()
- .statusCode(424)
- .body("__type", equalTo("ModelErrorException"));
+ .statusCode(200)
+ .header("Content-Type", containsString("application/vnd.amazon.eventstream"))
+ .extract().body().asString();
+
+ assertThat(body, containsString("messageStart"));
+ assertThat(body, containsString("modelStreamErrorException"));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("messageStop")));
}
@Test
- void converseStream_parsableChunksWithNoContent_returns424InsteadOfFakeSuccess() {
- // Every line here parses as valid JSON (parsedChunks > 0), but none of them carry
- // actual completion content - a role-only chunk, then a bare finish_reason with no
- // text or tool_calls. A finish_reason alone must not be treated as proof of a real
- // completion; this is indistinguishable from a truncated/failed generation.
+ void converseStream_completedWithNoContent_isValidEmptyCompletion() {
+ // A role-only chunk, then a bare finish_reason with no text or tool_calls, then [DONE].
+ // The stream properly completed (finish_reason and "[DONE]" were both seen) - real
+ // Bedrock delivers this as a legitimate empty message, not an error.
nextResponseBody.set("""
data: {"choices":[{"delta":{"role":"assistant"}}]}
@@ -809,7 +815,7 @@ class BedrockProxyIntegrationTest {
data: [DONE]
""");
- given()
+ String body = given()
.contentType("application/json")
.header("Authorization", AUTH_HEADER)
.body("""
@@ -818,22 +824,30 @@ class BedrockProxyIntegrationTest {
.when()
.post("/model/" + MAPPED_MODEL_ID + "/converse-stream")
.then()
- .statusCode(424)
- .body("__type", equalTo("ModelErrorException"));
+ .statusCode(200)
+ .extract().body().asString();
+
+ assertThat(body, containsString("messageStart"));
+ assertThat(body, containsString("messageStop"));
+ assertThat(body, containsString("end_turn"));
+ assertThat(body, containsString("metadata"));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("modelStreamErrorException")));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("contentBlockDelta")));
}
@Test
- void converseStream_truncatedTextWithNoFinishReason_returns424() {
+ void converseStream_truncatedTextWithNoFinishReason_emitsInBandStreamErrorAfterPartialText() {
// A real text delta arrives, but the stream ends (EOF, no [DONE], no finish_reason) -
- // e.g. the upstream connection dropped mid-generation. Accumulated text alone must not
- // be treated as a completed answer.
+ // e.g. the upstream connection dropped mid-generation. The already-streamed text and the
+ // 200 status can't be undone, so the truncation is reported as a trailing in-band
+ // modelStreamErrorException event instead of a clean messageStop.
nextResponseBody.set("""
data: {"choices":[{"delta":{"role":"assistant"}}]}
data: {"choices":[{"delta":{"content":"Hello"}}]}
""");
- given()
+ String body = given()
.contentType("application/json")
.header("Authorization", AUTH_HEADER)
.body("""
@@ -842,15 +856,21 @@ class BedrockProxyIntegrationTest {
.when()
.post("/model/" + MAPPED_MODEL_ID + "/converse-stream")
.then()
- .statusCode(424)
- .body("__type", equalTo("ModelErrorException"));
+ .statusCode(200)
+ .extract().body().asString();
+
+ assertThat(body, containsString("messageStart"));
+ assertThat(body, containsString("Hello"));
+ assertThat(body, containsString("modelStreamErrorException"));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("messageStop")));
}
@Test
- void converseStream_incompleteToolCallFragment_returns424() {
- // The tool_calls delta never carries a "name" and its arguments never close into
- // valid JSON - a partial/corrupted fragment, not a usable tool_use block. It must be
- // dropped rather than surfacing as a 200 with an empty-name toolUse block.
+ void converseStream_incompleteToolCallFragment_dropsFragmentAndCompletesNormally() {
+ // The tool_calls delta never carries a "name" and its arguments never close into valid
+ // JSON - a partial/corrupted fragment, not a usable tool_use block. It's dropped rather
+ // than surfacing as a toolUse block with empty identity fields; since the stream did
+ // reach "[DONE]"/finish_reason, it still completes normally (as an empty message).
nextResponseBody.set("""
data: {"choices":[{"delta":{"role":"assistant"}}]}
@@ -861,7 +881,7 @@ class BedrockProxyIntegrationTest {
data: [DONE]
""");
- given()
+ String body = given()
.contentType("application/json")
.header("Authorization", AUTH_HEADER)
.body("""
@@ -870,8 +890,14 @@ class BedrockProxyIntegrationTest {
.when()
.post("/model/" + MAPPED_MODEL_ID + "/converse-stream")
.then()
- .statusCode(424)
- .body("__type", equalTo("ModelErrorException"));
+ .statusCode(200)
+ .extract().body().asString();
+
+ assertThat(body, containsString("messageStop"));
+ assertThat(body, containsString("end_turn"));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("call_1")));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("toolUse")));
+ assertThat(body, org.hamcrest.Matchers.not(containsString("modelStreamErrorException")));
}
@Test