Light, fluffy, and always free - The AWS Local Emulator alternative
feat(bedrock-runtime): implement ConverseStream for stub and proxy backends (#2889)
Scott Steen44655751c674
diff --git a/docs/services/bedrock-runtime.md b/docs/services/bedrock-runtime.mdindex 43bfbca00c..0c38ee18c5 100644--- a/docs/services/bedrock-runtime.md+++ b/docs/services/bedrock-runtime.md@@ -8,7 +8,7 @@ Floci emulates the AWS Bedrock Runtime data-plane API. The response shape matcheTwo backends are available, selected via `floci.services.bedrock-runtime.backend`:- `stub` (default) — no real model inference: every call returns a fixed assistant turn plus synthetic token usage metadata.-- `proxy` — forwards `Converse` requests to any OpenAI-compatible `/chat/completions` endpoint (Ollama, OpenRouter, LiteLLM, vLLM, ...), translating request/response shapes including tool use. `InvokeModel` still falls back to the stub response in this mode. See [Proxy Backend](#proxy-backend) below.+- `proxy` — forwards `Converse` and `ConverseStream` requests to any OpenAI-compatible `/chat/completions` endpoint (Ollama, OpenRouter, LiteLLM, vLLM, ...), translating request/response shapes including tool use. `InvokeModel` still falls back to the stub response in this mode. See [Proxy Backend](#proxy-backend) below.The Bedrock management plane (`aws bedrock ...`: `ListFoundationModels`, `GetFoundationModel`, customization) is not yet emulated.@@ -18,7 +18,7 @@ The Bedrock management plane (`aws bedrock ...`: `ListFoundationModels`, `GetFou|-----------|----------|-------|| `Converse` | `POST /model/{modelId}/converse` | `stub`: returns a static assistant message. `proxy`: forwards to the configured OpenAI-compatible backend and translates the reply. || `InvokeModel` | `POST /model/{modelId}/invoke` | Returns Anthropic-shaped body for `anthropic.*` and `*.anthropic.*` model ids; generic `{"outputs": [...]}` shape otherwise. Not proxied yet — always the stub response, regardless of backend. |-| `ConverseStream` | `POST /model/{modelId}/converse-stream` | Returns 501 `UnsupportedOperationException` |+| `ConverseStream` | `POST /model/{modelId}/converse-stream` | `stub`: emits a static assistant turn as `messageStart`/`contentBlockDelta`/`contentBlockStop`/`messageStop`/`metadata` events. `proxy`: forwards a streaming request to the configured OpenAI-compatible backend and translates the SSE response into the same event sequence. Response body is `application/vnd.amazon.eventstream`-framed, built in one shot rather than incrementally chunked. || `InvokeModelWithResponseStream` | `POST /model/{modelId}/invoke-with-response-stream` | Returns 501 `UnsupportedOperationException` |`modelId` is URL-decoded by JAX-RS and echoed verbatim. Plain model ids (e.g. `anthropic.claude-3-haiku-20240307-v1:0`), inference-profile ids (e.g. `us.anthropic.claude-3-5-sonnet-20241022-v2:0`), and full ARNs containing slashes (e.g. `arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0`) are all accepted.@@ -88,6 +88,7 @@ print(resp["output"]["message"]["content"][0]["text"])- Real model inference with the `stub` backend (always returns a fixed string).- `InvokeModel` against the `proxy` backend (still returns the stub response).-- Streaming (`ConverseStream`, `InvokeModelWithResponseStream`) return 501, regardless of backend.+- `InvokeModelWithResponseStream` returns 501, regardless of backend.+- `ConverseStream` responses are built in one shot and returned as a complete event-stream body, not incrementally chunked over the wire.- Bedrock management plane (`ListFoundationModels`, `GetFoundationModel`, model customisation).- Bedrock Agents, Knowledge Bases, Guardrails, provisioned throughput.diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.javaindex 2884bc4f8f..f2bfa9a529 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeController.java@@ -45,26 +45,11 @@ public class BedrockRuntimeController {@Blocking@Path("/{modelId:.+}/converse")public Response converse(@PathParam("modelId") String modelId, String body) {- if (modelId == null || modelId.isBlank()) {- throw new AwsException("ValidationException", "modelId is required.", 400);- }- JsonNode request;- try {- request = body == null || body.isBlank()- ? objectMapper.createObjectNode()- : objectMapper.readTree(body);- } catch (Exception e) {- throw new AwsException("ValidationException",- "Malformed request body: " + e.getMessage(), 400);- }+ ObjectNode request = readRequest(modelId, body);- JsonNode messages = request.path("messages");- if (!messages.isArray() || messages.isEmpty()) {- throw new AwsException("ValidationException",- "messages is required and must be a non-empty array.", 400);- }+ JsonNode messages = validateMessages(request);- ObjectNode response = service.buildConverseResponse(modelId, (ObjectNode) request);+ ObjectNode response = service.buildConverseResponse(modelId, request);LOG.debugv("Bedrock Converse: modelId={0}, messages={1}", modelId, messages.size());return Response.ok(response).build();}@@ -94,11 +79,42 @@ public class BedrockRuntimeController {}@POST+ @Blocking@Path("/{modelId:.+}/converse-stream")@Consumes(MediaType.WILDCARD)- public Response converseStream(@PathParam("modelId") String modelId) {- throw new AwsException("UnsupportedOperationException",- "ConverseStream is not supported by Floci yet. "- + "Use Converse instead.", 501);+ public Response converseStream(@PathParam("modelId") String modelId, String body) {+ ObjectNode request = readRequest(modelId, body);++ JsonNode messages = validateMessages(request);++ byte[] response = service.buildConverseStreamResponse(modelId, request);+ LOG.debugv("Bedrock Converse Stream: modelId={0}, messages={1}", modelId, messages.size());+ return Response.ok(response)+ .header("Content-Type", "application/vnd.amazon.eventstream")+ .build();+ }++ private static JsonNode validateMessages(ObjectNode request) {+ JsonNode messages = request.path("messages");+ if (!messages.isArray() || messages.isEmpty()) {+ throw new AwsException("ValidationException",+ "messages is required and must be a non-empty array.", 400);+ }+ return messages;+ }++ private ObjectNode readRequest(String modelId, String body) {+ if (modelId == null || modelId.isBlank()) {+ throw new AwsException("ValidationException", "modelId is required.", 400);+ }++ try {+ return body == null || body.isBlank()+ ? objectMapper.createObjectNode()+ : (ObjectNode) objectMapper.readTree(body);+ } catch (Exception e) {+ throw new AwsException("ValidationException",+ "Malformed request body: " + e.getMessage(), 400);+ }}}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.javaindex 58ea435708..2edcd9f4ea 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeService.java@@ -37,6 +37,10 @@ public class BedrockRuntimeService {return backend().invokeModel(modelId, body);}+ public byte[] buildConverseStreamResponse(String modelId, ObjectNode bedrockRequest) {+ return backend().converseStream(modelId, bedrockRequest);+ }+private BedrockBackend backend() {String backend = config.services().bedrockRuntime().backend();if ("proxy".equalsIgnoreCase(backend)) {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.javaindex a4211fb833..18c390822d 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockBackend.java@@ -11,4 +11,6 @@ public interface BedrockBackend {ObjectNode converse(String modelId, ObjectNode bedrockRequest);byte[] invokeModel(String modelId, byte[] body);++ byte[] converseStream(String modelId, ObjectNode bedrockRequest);}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.javaindex 183d472d76..489ee163a6 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockOpenAiTranslator.java@@ -4,8 +4,14 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;import org.jboss.logging.Logger;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+/*** Translates between the Bedrock Runtime Converse wire shape and the OpenAI* Chat Completions wire shape used by Ollama, OpenRouter, LiteLLM, vLLM, etc.@@ -17,10 +23,15 @@ final class BedrockOpenAiTranslator {private BedrockOpenAiTranslator() {}- static ObjectNode toOpenAiRequest(ObjectMapper mapper, ObjectNode bedrockRequest, String resolvedModel) {+ static ObjectNode toOpenAiRequest(ObjectMapper mapper, ObjectNode bedrockRequest, String resolvedModel, boolean stream) {ObjectNode openAi = mapper.createObjectNode();openAi.put("model", resolvedModel);- openAi.put("stream", false);+ openAi.put("stream", stream);+ if (stream) {+ // Not all OpenAI-compatible backends honor this, but the ones that do (including+ // OpenAI itself) only include token usage in the final chunk when asked for it.+ openAi.putObject("stream_options").put("include_usage", true);+ }ArrayNode openAiMessages = openAi.putArray("messages");JsonNode system = bedrockRequest.path("system");@@ -193,6 +204,13 @@ final class BedrockOpenAiTranslator {static ObjectNode toBedrockResponse(ObjectMapper mapper, JsonNode openAiResponse, long latencyMs) {JsonNode choice = openAiResponse.path("choices").path(0);+ if (!choice.isObject()) {+ // A 2xx response with no choices at all (e.g. an error object the backend still+ // returned with a success status) is not a completion - don't fabricate an empty+ // assistant message out of it.+ throw new AwsException("ModelErrorException",+ "Proxy backend's response had no choices: " + truncate(openAiResponse.toString(), 512), 424);+ }JsonNode message = choice.path("message");String finishReason = choice.path("finish_reason").asText("stop");JsonNode toolCallsNode = message.path("tool_calls");@@ -240,6 +258,185 @@ final class BedrockOpenAiTranslator {return root;}+ /**+ * Translates an OpenAI Chat Completions SSE stream body ({@code data: {...}\n\n} lines,+ * terminated by {@code data: [DONE]}) into the ordered Bedrock ConverseStream events+ * (messageStart, contentBlockDelta, contentBlockStop, messageStop, metadata), framed as+ * {@code application/vnd.amazon.eventstream} bytes via {@link BedrockStreamEncoder}.+ *+ * <p>Tool calls are accumulated across chunks and emitted as a single complete+ * contentBlockDelta once the stream ends, rather than as incremental argument+ * fragments - simpler than real Bedrock's fragment-by-fragment streaming, but+ * SDKs consume both the same way since they just concatenate delta.toolUse.input+ * across events for a given contentBlockIndex before parsing it as JSON.+ */+ static byte[] toBedrockStreamEvents(ObjectMapper mapper, String sseBody, long latencyMs) {+ List<BedrockStreamEncoder.Event> events = new ArrayList<>();+ events.add(new BedrockStreamEncoder.Event("messageStart", mapper.createObjectNode().put("role", "assistant")));++ StringBuilder text = new StringBuilder();+ Map<Integer, ToolCallAccumulator> toolCalls = new LinkedHashMap<>();+ String finishReason = "stop";+ boolean sawFinishReason = false;+ boolean sawDone = false;+ int promptTokens = 0;+ int completionTokens = 0;+ int totalTokens = 0;+ int parsedChunks = 0;++ for (String rawLine : sseBody.split("\n")) {+ String line = rawLine.strip();+ if (!line.startsWith("data:")) {+ continue;+ }+ String data = line.substring(5).strip();+ if (data.isEmpty()) {+ continue;+ }+ if ("[DONE]".equals(data)) {+ sawDone = true;+ continue;+ }+ JsonNode chunk;+ try {+ chunk = mapper.readTree(data);+ } catch (Exception e) {+ LOG.warnv("Skipping malformed SSE chunk from proxy backend: {0}", e.getMessage());+ continue;+ }+ parsedChunks++;++ JsonNode usage = chunk.path("usage");+ if (usage.isObject()) {+ promptTokens = usage.path("prompt_tokens").asInt(promptTokens);+ completionTokens = usage.path("completion_tokens").asInt(completionTokens);+ totalTokens = usage.path("total_tokens").asInt(totalTokens);+ }++ JsonNode choice = chunk.path("choices").path(0);+ String reason = choice.path("finish_reason").asText(null);+ if (reason != null) {+ finishReason = reason;+ sawFinishReason = true;+ }++ JsonNode delta = choice.path("delta");+ String contentDelta = delta.path("content").asText(null);+ if (contentDelta != null && !contentDelta.isEmpty()) {+ text.append(contentDelta);+ ObjectNode deltaEvent = mapper.createObjectNode();+ deltaEvent.put("contentBlockIndex", 0);+ deltaEvent.putObject("delta").put("text", contentDelta);+ events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));+ }++ JsonNode toolCallDeltas = delta.path("tool_calls");+ if (toolCallDeltas.isArray()) {+ for (JsonNode toolCallDelta : toolCallDeltas) {+ int index = toolCallDelta.path("index").asInt(0);+ ToolCallAccumulator acc = toolCalls.computeIfAbsent(index, i -> new ToolCallAccumulator());+ String id = toolCallDelta.path("id").asText(null);+ if (id != null) {+ acc.id = id;+ }+ JsonNode function = toolCallDelta.path("function");+ String name = function.path("name").asText(null);+ if (name != null) {+ acc.name = name;+ }+ String argsFragment = function.path("arguments").asText(null);+ if (argsFragment != null) {+ acc.arguments.append(argsFragment);+ }+ }+ }+ }++ // A tool_calls delta only becomes a usable toolUse block once it has a non-blank id,+ // a non-blank name, and arguments that actually form a JSON object - a partial or+ // truncated fragment (e.g. an id with no name, or unclosed argument JSON) is dropped+ // instead of being surfaced as a broken tool_use block with empty identity fields.+ List<ToolCallAccumulator> validToolCalls = new ArrayList<>();+ for (ToolCallAccumulator acc : toolCalls.values()) {+ if (isUsableToolCall(mapper, acc)) {+ validToolCalls.add(acc);+ } else {+ LOG.warnv("Dropping incomplete or invalid tool_calls fragment from proxy backend: id={0}, name={1}",+ acc.id, acc.name);+ }+ }++ boolean hasContent = text.length() > 0 || !validToolCalls.isEmpty();+ boolean streamCompleted = sawFinishReason || sawDone;+ if (!hasContent || !streamCompleted) {+ // Either no usable completion content was ever produced (empty/garbled body,+ // role/usage/finish_reason-only records, a data:-prefixed error object, or only+ // invalid tool_calls fragments), or the stream ended without a finish_reason or a+ // "[DONE]" terminator at all - a truncated generation (e.g. the connection dropped+ // mid-stream). Neither case is a real completion: surface it as a backend failure+ // rather than fabricating a successful message out of whatever partial data arrived.+ throw new AwsException("ModelErrorException",+ "Proxy backend's ConverseStream response did not complete ("+ + parsedChunks + " SSE chunk(s) parsed, streamCompleted=" + streamCompleted+ + "): " + truncate(sseBody, 512), 424);+ }++ if (text.length() > 0) {+ events.add(new BedrockStreamEncoder.Event("contentBlockStop",+ mapper.createObjectNode().put("contentBlockIndex", 0)));+ }++ boolean hasToolCalls = !validToolCalls.isEmpty();+ int blockIndex = text.length() > 0 ? 1 : 0;+ for (ToolCallAccumulator acc : validToolCalls) {+ ObjectNode startEvent = mapper.createObjectNode();+ startEvent.put("contentBlockIndex", blockIndex);+ ObjectNode toolUseStart = startEvent.putObject("start").putObject("toolUse");+ toolUseStart.put("toolUseId", acc.id);+ toolUseStart.put("name", acc.name);+ events.add(new BedrockStreamEncoder.Event("contentBlockStart", startEvent));++ ObjectNode deltaEvent = mapper.createObjectNode();+ deltaEvent.put("contentBlockIndex", blockIndex);+ deltaEvent.putObject("delta").putObject("toolUse").put("input", acc.arguments.toString());+ events.add(new BedrockStreamEncoder.Event("contentBlockDelta", deltaEvent));++ events.add(new BedrockStreamEncoder.Event("contentBlockStop",+ mapper.createObjectNode().put("contentBlockIndex", blockIndex)));+ blockIndex++;+ }++ events.add(new BedrockStreamEncoder.Event("messageStop",+ mapper.createObjectNode().put("stopReason", hasToolCalls ? "tool_use" : mapFinishReason(finishReason))));++ ObjectNode metadata = mapper.createObjectNode();+ metadata.putObject("usage")+ .put("inputTokens", promptTokens)+ .put("outputTokens", completionTokens)+ .put("totalTokens", totalTokens);+ metadata.putObject("metrics").put("latencyMs", latencyMs);+ events.add(new BedrockStreamEncoder.Event("metadata", metadata));++ return BedrockStreamEncoder.encode(mapper, events);+ }++ private static boolean isUsableToolCall(ObjectMapper mapper, ToolCallAccumulator acc) {+ if (acc.id == null || acc.id.isBlank() || acc.name == null || acc.name.isBlank()) {+ return false;+ }+ try {+ return mapper.readTree(acc.arguments.toString()).isObject();+ } catch (Exception e) {+ return false;+ }+ }++ private static final class ToolCallAccumulator {+ String id;+ String name;+ final StringBuilder arguments = new StringBuilder();+ }+private static String extractMessageText(JsonNode message) {JsonNode content = message.path("content");if (content.isTextual()) {@@ -276,4 +473,8 @@ final class BedrockOpenAiTranslator {private static String mapFinishReason(String finishReason) {return "length".equals(finishReason) ? "max_tokens" : "end_turn";}++ private static String truncate(String value, int maxLength) {+ return value != null && value.length() > maxLength ? value.substring(0, maxLength) + "…" : value;+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.javanew file mode 100644index 0000000000..adb0403ba6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/BedrockStreamEncoder.java@@ -0,0 +1,42 @@+package io.github.hectorvent.floci.services.bedrockruntime.backend;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;+import io.github.hectorvent.floci.core.common.AwsException;++import java.io.ByteArrayOutputStream;+import java.util.LinkedHashMap;+import java.util.List;++/**+ * Frames a sequence of Bedrock ConverseStream events (messageStart, contentBlockDelta, ...)+ * as {@code application/vnd.amazon.eventstream} messages, one per event, concatenated into+ * a single response body - the same one-shot framing S3 Select and Kinesis SubscribeToShard+ * use elsewhere in Floci, rather than true incremental HTTP chunking.+ */+final class BedrockStreamEncoder {++ private BedrockStreamEncoder() {+ }++ record Event(String type, ObjectNode payload) {+ }++ static byte[] encode(ObjectMapper mapper, List<Event> events) {+ try {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ for (Event event : events) {+ LinkedHashMap<String, String> headers = new LinkedHashMap<>();+ headers.put(":message-type", "event");+ headers.put(":event-type", event.type());+ headers.put(":content-type", "application/json");+ out.write(AwsEventStreamEncoder.encodeMessage(headers, mapper.writeValueAsBytes(event.payload())));+ }+ return out.toByteArray();+ } catch (Exception e) {+ throw new AwsException("InternalServerException",+ "Failed to encode ConverseStream response: " + e.getMessage(), 500);+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.javaindex 6d9436c3a5..9f50142c90 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/ProxyBackend.java@@ -51,13 +51,44 @@ public class ProxyBackend implements BedrockBackend {public ObjectNode converse(String modelId, ObjectNode bedrockRequest) {EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();String resolvedModel = resolveModel(modelId, proxyConfig);+ ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, false);+ ProxyResult result = send(modelId, proxyConfig, openAiRequest);++ JsonNode openAiResponse;+ try {+ openAiResponse = objectMapper.readTree(result.body());+ } catch (Exception e) {+ throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);+ }++ return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, result.latencyMs());+ }++ @Override+ public byte[] invokeModel(String modelId, byte[] body) {+ throw new AwsException("ValidationException",+ "InvokeModel is not supported by the bedrock-runtime proxy backend; use Converse.", 400);+ }++ @Override+ public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {+ EmulatorConfig.BedrockProxyConfig proxyConfig = config.services().bedrockRuntime().proxy();+ String resolvedModel = resolveModel(modelId, proxyConfig);+ ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel, true);+ ProxyResult result = send(modelId, proxyConfig, openAiRequest);++ return BedrockOpenAiTranslator.toBedrockStreamEvents(objectMapper, result.body(), result.latencyMs());+ }++ private record ProxyResult(String body, long latencyMs) {+ }++ private ProxyResult send(String modelId, EmulatorConfig.BedrockProxyConfig proxyConfig, ObjectNode openAiRequest) {String baseUrl = proxyConfig.url().filter(url -> !url.isBlank()).orElseThrow(() -> new AwsException("ValidationException","floci.services.bedrock-runtime.proxy.url is required when backend=proxy.", 400));- ObjectNode openAiRequest = BedrockOpenAiTranslator.toOpenAiRequest(objectMapper, bedrockRequest, resolvedModel);-URI uri;HttpRequest.Builder builder;try {@@ -102,20 +133,7 @@ public class ProxyBackend implements BedrockBackend {424);}- JsonNode openAiResponse;- try {- openAiResponse = objectMapper.readTree(response.body());- } catch (Exception e) {- throw new AwsException("ModelErrorException", "Proxy backend returned malformed JSON: " + e.getMessage(), 424);- }-- return BedrockOpenAiTranslator.toBedrockResponse(objectMapper, openAiResponse, latencyMs);- }-- @Override- public byte[] invokeModel(String modelId, byte[] body) {- throw new AwsException("ValidationException",- "InvokeModel is not supported by the bedrock-runtime proxy backend; use Converse.", 400);+ return new ProxyResult(response.body(), latencyMs);}String resolveModel(String bedrockModelId, EmulatorConfig.BedrockProxyConfig proxyConfig) {diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.javaindex cb2e84d9fd..d517b95e57 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockruntime/backend/StubBackend.java@@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.util.List;+/*** Dummy response builder for Bedrock Runtime. Stateless.* No real model inference: returns a fixed assistant turn plus token usage metadata.@@ -74,4 +76,34 @@ public class StubBackend implements BedrockBackend {throw new RuntimeException("Failed to serialize InvokeModel response", e);}}++ @Override+ public byte[] converseStream(String modelId, ObjectNode bedrockRequest) {+ ObjectNode messageStart = objectMapper.createObjectNode();+ messageStart.put("role", "assistant");++ ObjectNode contentBlockDelta = objectMapper.createObjectNode();+ contentBlockDelta.put("contentBlockIndex", 0);+ contentBlockDelta.putObject("delta").put("text", "Floci stub response for model=" + modelId);++ ObjectNode contentBlockStop = objectMapper.createObjectNode();+ contentBlockStop.put("contentBlockIndex", 0);++ ObjectNode messageStop = objectMapper.createObjectNode();+ messageStop.put("stopReason", "end_turn");++ ObjectNode metadata = objectMapper.createObjectNode();+ metadata.putObject("usage")+ .put("inputTokens", 10)+ .put("outputTokens", 12)+ .put("totalTokens", 22);+ metadata.putObject("metrics").put("latencyMs", 1);++ return BedrockStreamEncoder.encode(objectMapper, List.of(+ new BedrockStreamEncoder.Event("messageStart", messageStart),+ new BedrockStreamEncoder.Event("contentBlockDelta", contentBlockDelta),+ new BedrockStreamEncoder.Event("contentBlockStop", contentBlockStop),+ new BedrockStreamEncoder.Event("messageStop", messageStop),+ new BedrockStreamEncoder.Event("metadata", metadata)));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.javaindex 67ca258a21..d7808d1e26 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockProxyIntegrationTest.java@@ -19,6 +19,8 @@ import java.util.Map;import java.util.concurrent.atomic.AtomicReference;import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -368,6 +370,27 @@ class BedrockProxyIntegrationTest {org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("x".repeat(1000)))));}+ @Test+ void converse_noChoicesInResponse_returns424InsteadOfFakeSuccess() {+ // A 2xx response with no "choices" at all (e.g. an error object returned with a+ // success status) must not be translated into a fake successful empty completion.+ nextResponseBody.set("""+ {"error": {"message": "model not found"}}+ """);++ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse")+ .then()+ .statusCode(424)+ .body("__type", equalTo("ModelErrorException"));+ }+@Testvoid converse_omitsToolsWhenNoToolConfig() throws IOException {nextResponseBody.set("""@@ -678,6 +701,179 @@ class BedrockProxyIntegrationTest {assertFalse(sentRequest.has("tool_choice"));}+ @Test+ void converseStream_basicTextResponse_translatesSseToEventStream() throws IOException {+ nextResponseBody.set("""+ data: {"choices":[{"delta":{"role":"assistant"}}]}++ data: {"choices":[{"delta":{"content":"Hello"}}]}++ data: {"choices":[{"delta":{"content":" world"}}]}++ data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":7,"completion_tokens":9,"total_tokens":16}}++ data: [DONE]+ """);++ String body = given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(200)+ .header("Content-Type", containsString("application/vnd.amazon.eventstream"))+ .extract().body().asString();++ assertThat(body, containsString("messageStart"));+ assertThat(body, containsString("Hello"));+ assertThat(body, containsString(" world"));+ assertThat(body, containsString("contentBlockStop"));+ assertThat(body, containsString("messageStop"));+ assertThat(body, containsString("end_turn"));+ assertThat(body, containsString("metadata"));++ JsonNode sentRequest = objectMapper.readTree(received.get().body());+ assertTrue(sentRequest.path("stream").asBoolean());+ assertEquals("claude-3-haiku", sentRequest.path("model").asText());+ }++ @Test+ void converseStream_toolCallsTranslateToToolUseContentBlock() {+ nextResponseBody.set("""+ data: {"choices":[{"delta":{"role":"assistant"}}]}++ data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_abc123","function":{"name":"get_weather","arguments":"{\\"city\\":"}}]}}]}++ data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\\"Berlin\\"}"}}]}}]}++ data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":5,"completion_tokens":8,"total_tokens":13}}++ data: [DONE]+ """);++ String body = given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "weather in Berlin?"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(200)+ .extract().body().asString();++ assertThat(body, containsString("contentBlockStart"));+ assertThat(body, containsString("call_abc123"));+ assertThat(body, containsString("get_weather"));+ assertThat(body, containsString("Berlin"));+ assertThat(body, containsString("tool_use"));+ }++ @Test+ void converseStream_noUsableSseChunks_returns424InsteadOfFakeSuccess() {+ // A 2xx response with a body that isn't SSE-shaped at all (e.g. a plain JSON error+ // object) must not be silently translated into a "successful" empty completion.+ nextResponseBody.set("""+ {"error": {"message": "model not found"}}+ """);++ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(424)+ .body("__type", equalTo("ModelErrorException"));+ }++ @Test+ void converseStream_parsableChunksWithNoContent_returns424InsteadOfFakeSuccess() {+ // Every line here parses as valid JSON (parsedChunks > 0), but none of them carry+ // actual completion content - a role-only chunk, then a bare finish_reason with no+ // text or tool_calls. A finish_reason alone must not be treated as proof of a real+ // completion; this is indistinguishable from a truncated/failed generation.+ nextResponseBody.set("""+ data: {"choices":[{"delta":{"role":"assistant"}}]}++ data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":0,"total_tokens":3}}++ data: [DONE]+ """);++ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(424)+ .body("__type", equalTo("ModelErrorException"));+ }++ @Test+ void converseStream_truncatedTextWithNoFinishReason_returns424() {+ // A real text delta arrives, but the stream ends (EOF, no [DONE], no finish_reason) -+ // e.g. the upstream connection dropped mid-generation. Accumulated text alone must not+ // be treated as a completed answer.+ nextResponseBody.set("""+ data: {"choices":[{"delta":{"role":"assistant"}}]}++ data: {"choices":[{"delta":{"content":"Hello"}}]}+ """);++ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(424)+ .body("__type", equalTo("ModelErrorException"));+ }++ @Test+ void converseStream_incompleteToolCallFragment_returns424() {+ // The tool_calls delta never carries a "name" and its arguments never close into+ // valid JSON - a partial/corrupted fragment, not a usable tool_use block. It must be+ // dropped rather than surfacing as a 200 with an empty-name toolUse block.+ nextResponseBody.set("""+ data: {"choices":[{"delta":{"role":"assistant"}}]}++ data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"arguments":"{\\"city\\":"}}]}}]}++ data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}++ data: [DONE]+ """);++ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("""+ {"messages": [{"role": "user", "content": [{"text": "weather?"}]}]}+ """)+ .when()+ .post("/model/" + MAPPED_MODEL_ID + "/converse-stream")+ .then()+ .statusCode(424)+ .body("__type", equalTo("ModelErrorException"));+ }+@Testvoid invokeModel_notSupportedByProxyBackend_returns400() {given()diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeIntegrationTest.javaindex 73c2533e5d..be56cefb39 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockruntime/BedrockRuntimeIntegrationTest.java@@ -4,6 +4,7 @@ import io.quarkus.test.junit.QuarkusTest;import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.greaterThan;@@ -220,8 +221,8 @@ class BedrockRuntimeIntegrationTest {}@Test- void converseStream_returns501() {- given()+ void converseStream_happyPath() {+ String body = given().contentType("application/json").header("Authorization", AUTH_HEADER).body("""@@ -230,8 +231,32 @@ class BedrockRuntimeIntegrationTest {.when().post("/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream").then()- .statusCode(501)- .body("__type", equalTo("UnsupportedOperationException"));+ .statusCode(200)+ .header("Content-Type", containsString("application/vnd.amazon.eventstream"))+ .extract().body().asString();++ // The response is binary event-stream framing, not JSON; the event names and JSON+ // payload text still appear as recognizable UTF-8 substrings around the framing bytes.+ assertThat(body, containsString("messageStart"));+ assertThat(body, containsString("contentBlockDelta"));+ assertThat(body, containsString("anthropic.claude-3-haiku-20240307-v1:0"));+ assertThat(body, containsString("contentBlockStop"));+ assertThat(body, containsString("messageStop"));+ assertThat(body, containsString("end_turn"));+ assertThat(body, containsString("metadata"));+ }++ @Test+ void converseStream_missingMessages_returns400() {+ given()+ .contentType("application/json")+ .header("Authorization", AUTH_HEADER)+ .body("{}")+ .when()+ .post("/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"));}@Test