Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis-cbor): timestamps mis-scaled 1000x on the CBOR path (#2432)
sheldon-nadarajah-sft39ed7712a719
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.javaindex 9ca686e319..076d61dffc 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java@@ -3,6 +3,9 @@ package io.github.hectorvent.floci.core.common;import com.fasterxml.jackson.core.JacksonException;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.DecimalNode;+import com.fasterxml.jackson.databind.node.ObjectNode;import com.fasterxml.jackson.dataformat.cbor.CBORFactory;import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;import com.google.gson.JsonParseException;@@ -22,6 +25,8 @@ import org.jboss.logging.Logger;import java.io.ByteArrayOutputStream;import java.io.IOException;+import java.math.BigDecimal;+import java.math.RoundingMode;import java.util.Iterator;import java.util.List;import java.util.Map;@@ -76,18 +81,40 @@ public class AwsJsonCborController {/**- * Serializes a JsonNode to CBOR bytes, encoding timestamp shapes with CBOR tag 1- * (epoch seconds, RFC 8949 section 3.4.2) as required by the smithy-rpc-v2-cbor- * protocol specification.+ * Serializes a JsonNode to CBOR bytes for the smithy-rpc-v2-cbor protocol, encoding+ * timestamp shapes with CBOR tag 1 as plain epoch seconds (RFC 8949 section 3.4.2,+ * unmodified) — this protocol uses RFC 8949's own native tag(1) convention, unlike+ * the older, separate {@code application/x-amz-cbor-1.1} dialect (see+ * {@link #nodeToLegacyCbor}), which encodes tag(1) as epoch milliseconds instead.+ * These are two genuinely different wire conventions sharing the same CBOR tag+ * mechanism, not one protocol with an inconsistency — see floci-io/floci#2368's+ * resolution for the investigation that distinguished them (CloudWatch Metrics, on+ * this protocol, needs the value unscaled; Kinesis, on the legacy dialect, needs it+ * scaled to milliseconds).* <p>* Package-private so the timestamp-tagging behaviour can be unit-tested directly* without booting Quarkus.*/static byte[] nodeToSmithyCbor(JsonNode node) throws Exception {+ return writeCbor(node, false);+ }++ /**+ * Serializes a JsonNode to CBOR bytes for the legacy {@code application/x-amz-cbor-1.1}+ * dialect (routed through {@link #handleCborRequest}) — encodes timestamp shapes as+ * CBOR tag(1) epoch <em>milliseconds</em>, AWS's convention for this specific,+ * pre-Smithy dialect. See {@link #nodeToSmithyCbor} for how this differs from the+ * newer smithy-rpc-v2-cbor protocol, and {@link #writeCborTimestamp} for why.+ */+ static byte[] nodeToLegacyCbor(JsonNode node) throws Exception {+ return writeCbor(node, true);+ }++ private static byte[] writeCbor(JsonNode node, boolean legacyDialect) throws Exception {ByteArrayOutputStream out = new ByteArrayOutputStream();CBORFactory factory = (CBORFactory) CBOR_MAPPER.getFactory();try (CBORGenerator gen = factory.createGenerator(out)) {- writeNodeToCbor(gen, node, false);+ writeNodeToCbor(gen, node, false, legacyDialect);}return out.toByteArray();}@@ -98,16 +125,18 @@ public class AwsJsonCborController {* up the Smithy shape): when set and the node is a number, it is emitted as a CBOR* tag(1) timestamp. The flag is set by {@link #isTimestampField(String)} for matching* object fields and is propagated into array elements, so both scalar timestamps and- * elements of timestamp lists are tagged.+ * elements of timestamp lists are tagged. {@code legacyDialect} selects which of the+ * two wire conventions tag(1) uses - see {@link #nodeToSmithyCbor}/{@link #nodeToLegacyCbor}.*/- private static void writeNodeToCbor(CBORGenerator gen, JsonNode node, boolean numberIsTimestamp) throws Exception {+ private static void writeNodeToCbor(CBORGenerator gen, JsonNode node, boolean numberIsTimestamp,+ boolean legacyDialect) throws Exception {if (node.isObject()) {gen.writeStartObject();Iterator<Map.Entry<String, JsonNode>> fields = node.fields();while (fields.hasNext()) {Map.Entry<String, JsonNode> entry = fields.next();gen.writeFieldName(entry.getKey());- writeNodeToCbor(gen, entry.getValue(), isTimestampField(entry.getKey()));+ writeNodeToCbor(gen, entry.getValue(), isTimestampField(entry.getKey()), legacyDialect);}gen.writeEndObject();} else if (node.isArray()) {@@ -115,11 +144,11 @@ public class AwsJsonCborController {for (JsonNode item : node) {// Propagate the timestamp context so every element of a timestamp list// (e.g. CloudWatch GetMetricData MetricDataResults[].Timestamps) is tagged.- writeNodeToCbor(gen, item, numberIsTimestamp);+ writeNodeToCbor(gen, item, numberIsTimestamp, legacyDialect);}gen.writeEndArray();} else if (numberIsTimestamp && node.isNumber()) {- writeCborTimestamp(gen, node);+ writeCborTimestamp(gen, node, legacyDialect);} else if (node.isTextual()) {gen.writeString(node.textValue());} else if (node.isDouble() || node.isFloat()) {@@ -153,18 +182,95 @@ public class AwsJsonCborController {}/**- * Writes a numeric node as a CBOR tag(1) epoch-seconds timestamp, preserving- * integer-ness. Integral epoch seconds are emitted as a tag(1) integer and- * fractional values as a tag(1) float; both are spec-valid (RFC 8949 section 3.4.2)- * and accepted by AWS SDK decoders.+ * Writes a numeric node as a CBOR tag(1) timestamp. On the legacy+ * {@code application/x-amz-cbor-1.1} dialect, converts from this codebase's internal+ * convention (fractional epoch seconds, matching the plain-JSON protocol) to epoch+ * milliseconds - AWS's convention for tag(1) on that specific, pre-Smithy dialect.+ * See floci-io/floci#2368: without this conversion, an unmodified AWS SDK for Java v2+ * Kinesis client (which uses this legacy dialect by default) decodes every timestamp+ * 1000x too small, and an {@code AT_TIMESTAMP} shard iterator built from a corrupted+ * request-side timestamp never matches any real record. On smithy-rpc-v2-cbor,+ * RFC 8949's own tag(1) convention (plain epoch seconds) already applies unmodified -+ * no conversion needed, or correct, there.+ * <p>+ * {@code BigDecimal} (not double arithmetic) preserves millisecond precision exactly+ * and avoids reintroducing the same class of float-precision defect fixed on the+ * read side of this exact value by #2173/#2359 — {@link JsonNode#decimalValue()}+ * already handles both integral and floating-point nodes correctly.*/- private static void writeCborTimestamp(CBORGenerator gen, JsonNode node) throws Exception {+ private static void writeCborTimestamp(CBORGenerator gen, JsonNode node, boolean legacyDialect) throws Exception {gen.writeTag(1);- if (node.isIntegralNumber()) {- gen.writeNumber(node.longValue());- } else {- gen.writeNumber(node.doubleValue());+ if (!legacyDialect) {+ if (node.isIntegralNumber()) {+ gen.writeNumber(node.longValue());+ } else {+ gen.writeNumber(node.doubleValue());+ }+ return;}+ BigDecimal epochSeconds = node.decimalValue();+ long epochMillis = epochSeconds.movePointRight(3).setScale(0, RoundingMode.HALF_UP).longValueExact();+ gen.writeNumber(epochMillis);+ }++ /**+ * Converts every timestamp-shaped field's value in a CBOR request/response decoded+ * off the legacy {@code application/x-amz-cbor-1.1} dialect from epoch milliseconds+ * (that dialect's tag(1) wire representation) back to this codebase's internal+ * convention of fractional epoch seconds — the inverse of {@link #writeCborTimestamp}+ * for that same dialect. See floci-io/floci#2368, and {@link #nodeToSmithyCbor} for+ * why this must NOT be applied to smithy-rpc-v2-cbor traffic (its tag(1) is already+ * plain epoch seconds, unscaled).+ * <p>+ * Jackson's CBOR module does not preserve tag information into the decoded+ * {@link JsonNode} tree ({@link #bodyToJson} uses a plain {@code readTree}), so by+ * the time a request reaches this method the CBOR tag(1) marker is already gone and+ * a decoded timestamp's raw numeric value is indistinguishable from any other number.+ * This reuses the same schema-less field-name heuristic ({@link #isTimestampField})+ * {@link #writeNodeToCbor} already relies on for the identical reason on the response+ * side, so every service handler continues to see the one convention it already+ * expects regardless of which wire protocol (legacy CBOR or plain JSON) actually+ * carried the request — no handler needs to change.+ * <p>+ * Mutates the given tree in place (both {@link ObjectNode} and {@link ArrayNode} are+ * mutable) and returns it, purely for convenient chaining at the call site.+ */+ static JsonNode normalizeLegacyCborTimestampsFromMillis(JsonNode node) {+ normalizeLegacyCborTimestampsFromMillis(node, false);+ return node;+ }++ private static void normalizeLegacyCborTimestampsFromMillis(JsonNode node, boolean isTimestamp) {+ if (node.isObject()) {+ ObjectNode object = (ObjectNode) node;+ Iterator<Map.Entry<String, JsonNode>> fields = object.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> entry = fields.next();+ boolean fieldIsTimestamp = isTimestampField(entry.getKey());+ JsonNode value = entry.getValue();+ if (fieldIsTimestamp && value.isNumber()) {+ object.set(entry.getKey(), millisToSeconds(value));+ } else {+ normalizeLegacyCborTimestampsFromMillis(value, fieldIsTimestamp);+ }+ }+ } else if (node.isArray()) {+ ArrayNode array = (ArrayNode) node;+ for (int i = 0; i < array.size(); i++) {+ JsonNode item = array.get(i);+ if (isTimestamp && item.isNumber()) {+ array.set(i, millisToSeconds(item));+ } else {+ normalizeLegacyCborTimestampsFromMillis(item, isTimestamp);+ }+ }+ }+ }++ /** Exact millis-to-seconds conversion (see {@link #normalizeLegacyCborTimestampsFromMillis}). */+ private static JsonNode millisToSeconds(JsonNode millisNode) {+ BigDecimal epochMillis = millisNode.decimalValue();+ return DecimalNode.valueOf(epochMillis.movePointLeft(3));}/**@@ -217,12 +323,24 @@ public class AwsJsonCborController {}JsonNode bodyToJson(HttpHeaders httpHeaders, byte[] body) throws IOException {+ return bodyToJson(httpHeaders, body, false);+ }++ /**+ * {@code legacyDialect} selects whether a decoded numeric timestamp field is+ * interpreted as the legacy {@code application/x-amz-cbor-1.1} dialect's epoch+ * milliseconds ({@link #normalizeLegacyCborTimestampsFromMillis}) or left as-is+ * (smithy-rpc-v2-cbor's own plain epoch seconds, already correct unmodified) — see+ * {@link #nodeToSmithyCbor} for why these differ.+ */+ private JsonNode bodyToJson(HttpHeaders httpHeaders, byte[] body, boolean legacyDialect) throws IOException {JsonNode request;if (body != null && body.length > 0) {if( httpHeaders.getRequestHeader("Content-encoding") != null && isGZipped(httpHeaders.getRequestHeader("Content-encoding"))) {body = decodeBody(body);}- request = CBOR_MAPPER.readTree(body);+ JsonNode decoded = CBOR_MAPPER.readTree(body);+ request = legacyDialect ? normalizeLegacyCborTimestampsFromMillis(decoded) : decoded;} else {request = objectMapper.createObjectNode();}@@ -245,6 +363,14 @@ public class AwsJsonCborController {/*** Handles AWS services that migrated to the smithy-rpc-v2-cbor protocol at root path.* Fallback handler for X-Amz-Target based routing with CBOR body.+ * <p>+ * Confirmed (via a real wire capture, floci-io/floci#2368) that Kinesis reaches this+ * handler using the older, separate {@code application/x-amz-cbor-1.1} dialect, whose+ * tag(1) timestamps are epoch milliseconds rather than smithy-rpc-v2-cbor's plain+ * epoch seconds — see {@link #nodeToLegacyCbor}/{@link #bodyToJson}'s {@code+ * legacyDialect} parameter, applied here for that reason. Not reconfirmed for every+ * other service dispatched below; none currently has test coverage asserting an+ * exact numeric timestamp value over this specific path.*/@POST@Consumes({GENERIC_CBOR_MEDIA_TYPE, AWS_CBOR_1_1_MEDIA_TYPE})@@ -272,7 +398,7 @@ public class AwsJsonCborController {LOG.debugv("{0} CBOR action: {1}", serviceKey, action);try {- JsonNode request = bodyToJson(httpHeaders, body);+ JsonNode request = bodyToJson(httpHeaders, body, true);String region = regionResolver.resolveRegion(httpHeaders);Response delegated = switch (serviceKey) {@@ -296,7 +422,7 @@ public class AwsJsonCborController {JsonNode responseNode = delegated.getEntity() instanceof JsonNode? (JsonNode) delegated.getEntity(): objectMapper.valueToTree(delegated.getEntity());- byte[] cborBytes = nodeToSmithyCbor(responseNode);+ byte[] cborBytes = nodeToLegacyCbor(responseNode);String responseContentType = responseContentType(httpHeaders);return Response.status(delegated.getStatus()).header("smithy-protocol", "rpc-v2-cbor")diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.javanew file mode 100644index 0000000000..3cbf6c925f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.java@@ -0,0 +1,96 @@+package io.github.hectorvent.floci.core.common;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.time.Instant;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Regression guard for the exact bug this repo's own {@code AwsJsonCborSerializerTest}+ * originally introduced while fixing floci-io/floci#2368: scaling smithy-rpc-v2-cbor+ * timestamps to milliseconds (correct for the legacy {@code application/x-amz-cbor-1.1}+ * dialect, per the same issue) broke CloudWatch Metrics, whose {@code PutMetricData}/+ * {@code GetMetricStatistics} round-trip a {@code Timestamp} field over the+ * smithy-rpc-v2-cbor protocol - which uses RFC 8949's plain, unscaled epoch-seconds+ * tag(1) convention. A wrongly-scaled timestamp lands outside any query window, so+ * {@code GetMetricStatistics} silently returns zero datapoints (matches the real Go SDK+ * compat test failure this was caught by, {@code TestCloudWatch/GetMetricStatistics}+ * asserting {@code Datapoints} non-empty).+ * <p>+ * Drives the real HTTP endpoint end-to-end via the same {@code+ * POST /service/GraniteServiceVersion20100801/operation/{action}} routing+ * {@link SmithyRpcV2RoutingIntegrationTest#cloudWatchRoutesViaGraniteServiceName} uses,+ * with request bodies built via {@link AwsJsonCborController#nodeToSmithyCbor} - the+ * correct (unscaled) encoder for this protocol - so this test would fail the same way+ * the real regression did if that scaling were ever mistakenly reapplied here.+ */+@QuarkusTest+class AwsJsonCborCloudWatchTimestampIntegrationTest {++ private static final ObjectMapper JSON = new ObjectMapper();+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());++ @Test+ void getMetricStatisticsFindsADatapointPublishedJustNow() throws Exception {+ String namespace = "CborRpcV2TimestampTest-" + System.nanoTime();+ Instant now = Instant.now();++ ObjectNode putRequest = JSON.createObjectNode();+ putRequest.put("Namespace", namespace);+ ArrayNode metricData = putRequest.putArray("MetricData");+ ObjectNode datum = metricData.addObject();+ datum.put("MetricName", "RequestCount");+ datum.put("Value", 42.0);+ datum.put("Unit", "Count");+ datum.put("Timestamp", now.getEpochSecond());+ rpcV2Call("PutMetricData", putRequest);++ ObjectNode statsRequest = JSON.createObjectNode();+ statsRequest.put("Namespace", namespace);+ statsRequest.put("MetricName", "RequestCount");+ statsRequest.put("StartTime", now.minusSeconds(300).getEpochSecond());+ statsRequest.put("EndTime", now.plusSeconds(60).getEpochSecond());+ statsRequest.put("Period", 60);+ statsRequest.putArray("Statistics").add("Sum");+ JsonNode statsResponse = rpcV2Call("GetMetricStatistics", statsRequest);++ // Before this fix, a stray millisecond-scaling of "Timestamp" here would have+ // stored the datapoint far outside [StartTime, EndTime], and this would be 0 -+ // exactly the real Go SDK compat test failure this class guards against.+ assertFalse(statsResponse.path("Datapoints").isEmpty(),+ "GetMetricStatistics must find the datapoint just published: " + statsResponse);++ // Not an exact match: CloudWatch buckets a datapoint's own returned Timestamp+ // to the start of its Period (60s here), so it legitimately differs from the+ // submitted value by up to one period - a real, unrelated behaviour, not this+ // fix's concern. What this DOES still catch: a 1000x scale error would push the+ // value thousands of years away, far outside this window.+ JsonNode datapoint = statsResponse.path("Datapoints").get(0);+ long bucketedEpochSecond = datapoint.path("Timestamp").asLong();+ long driftSeconds = Math.abs(now.getEpochSecond() - bucketedEpochSecond);+ assertTrue(driftSeconds <= 60,+ "the round-tripped Timestamp must stay in plain epoch seconds, unscaled (drifted "+ + driftSeconds + "s): " + datapoint);+ }++ private static JsonNode rpcV2Call(String action, ObjectNode body) throws Exception {+ byte[] responseCbor = given()+ .contentType("application/cbor")+ .accept("application/cbor")+ .header("smithy-protocol", "rpc-v2-cbor")+ .body(AwsJsonCborController.nodeToSmithyCbor(body))+ .when().post("/service/GraniteServiceVersion20100801/operation/" + action)+ .then().statusCode(200)+ .extract().asByteArray();+ return CBOR.readTree(responseCbor);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.javanew file mode 100644index 0000000000..5c43755fdf--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java@@ -0,0 +1,120 @@+package io.github.hectorvent.floci.core.common;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.time.Instant;+import java.util.Base64;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;++/**+ * End-to-end reproduction of floci-io/floci#2368's actual reported symptom, driven+ * through the real HTTP CBOR endpoint (not just the unit-level encode/decode round-trip+ * already covered by {@link AwsJsonCborSerializerTest}): an {@code AT_TIMESTAMP} Kinesis+ * shard iterator built over the CBOR wire protocol - the AWS SDK for Java v2's default+ * transport - for a timestamp strictly before an already-written record.+ * <p>+ * Before the fix, this never found that record: {@code handleGetShardIterator} treated+ * the decoded {@code Timestamp} as epoch seconds when the CBOR wire value a real client+ * actually sends is epoch milliseconds, landing the iterator around the year 58598 -+ * past every real record - so {@code GetRecords} returned an empty page forever, with no+ * error at all.+ * <p>+ * The {@code GetShardIterator} request's {@code Timestamp} is hand-built with a raw+ * tag(1) epoch-<em>millisecond</em> value via {@link #cborTag1MillisTimestampRequest},+ * deliberately bypassing {@link AwsJsonCborController#nodeToLegacyCbor} (the method that+ * would normally build this exact request, correctly, for the legacy+ * {@code application/x-amz-cbor-1.1} dialect Kinesis uses) - building the request+ * through Floci's own encoder would only prove Floci's own encoder and decoder agree+ * with each other, not that Floci actually speaks the real AWS wire format a genuine+ * CBOR-default SDK client sends. Hand-building the millis value directly is what makes+ * this test fail against the pre-fix code and pass against the fix, matching the+ * issue's own wire capture (tag(1) followed by an 8-byte integer, e.g.+ * {@code c1 1b 000001a013accc38}).+ * <p>+ * The other requests here (none carry a timestamp field, so the choice of encoder makes+ * no difference to their bytes) go through {@link AwsJsonCborController#nodeToSmithyCbor}+ * as a matter of convenience, and CBOR response bodies are decoded with a plain CBOR+ * {@link ObjectMapper} since the server mirrors the request's own content type back on+ * the response.+ */+@QuarkusTest+class AwsJsonCborKinesisAtTimestampIntegrationTest {++ private static final String CBOR_CONTENT_TYPE = "application/x-amz-cbor-1.1";+ private static final ObjectMapper JSON = new ObjectMapper();+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());++ @Test+ void atTimestampShardIteratorOverCborFindsAnAlreadyWrittenRecord() throws Exception {+ String streamName = "cbor-at-timestamp-" + System.nanoTime();++ cborCall("Kinesis_20131202.CreateStream",+ AwsJsonCborController.nodeToSmithyCbor(+ JSON.createObjectNode().put("StreamName", streamName).put("ShardCount", 1)));++ // Cutoff strictly before the record about to be written.+ Instant cutoff = Instant.now().minusSeconds(30);++ cborCall("Kinesis_20131202.PutRecord", AwsJsonCborController.nodeToSmithyCbor(JSON.createObjectNode()+ .put("StreamName", streamName)+ .put("Data", Base64.getEncoder().encodeToString("hello".getBytes()))+ .put("PartitionKey", "pk")));++ byte[] getShardIteratorRequest = cborTag1MillisTimestampRequest(+ streamName, "shardId-000000000000", cutoff.toEpochMilli());+ JsonNode shardIteratorResponse = cborCall("Kinesis_20131202.GetShardIterator", getShardIteratorRequest);+ String shardIterator = shardIteratorResponse.path("ShardIterator").asText();+ assertFalse(shardIterator.isEmpty(), "GetShardIterator must actually return an iterator");++ JsonNode records = cborCall("Kinesis_20131202.GetRecords", AwsJsonCborController.nodeToSmithyCbor(+ JSON.createObjectNode().put("ShardIterator", shardIterator)));++ // Before the #2368 fix, this was always 0 regardless of how long the caller waited.+ assertEquals(1, records.path("Records").size(),+ "AT_TIMESTAMP over CBOR must find the record written after the cutoff");+ }++ /**+ * Hand-builds a {@code GetShardIterator} CBOR request body with {@code Timestamp}+ * encoded exactly as a real AWS SDK for Java v2 client sends it: CBOR tag(1)+ * immediately followed by an integer epoch-<em>millisecond</em> value - see this+ * class's own javadoc for why this must not go through+ * {@link AwsJsonCborController#nodeToSmithyCbor}.+ */+ private static byte[] cborTag1MillisTimestampRequest(String streamName, String shardId, long epochMillis)+ throws Exception {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ CBORFactory factory = (CBORFactory) CBOR.getFactory();+ try (CBORGenerator gen = factory.createGenerator(out)) {+ gen.writeStartObject();+ gen.writeStringField("StreamName", streamName);+ gen.writeStringField("ShardId", shardId);+ gen.writeStringField("ShardIteratorType", "AT_TIMESTAMP");+ gen.writeFieldName("Timestamp");+ gen.writeTag(1);+ gen.writeNumber(epochMillis);+ gen.writeEndObject();+ }+ return out.toByteArray();+ }++ private static JsonNode cborCall(String target, byte[] requestCbor) throws Exception {+ byte[] responseCbor = given()+ .header("X-Amz-Target", target)+ .contentType(CBOR_CONTENT_TYPE)+ .body(requestCbor)+ .when().post("/")+ .then().statusCode(200)+ .extract().asByteArray();+ return CBOR.readTree(responseCbor);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.javaindex c2a0e6c78f..0927e7e69f 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java@@ -10,13 +10,30 @@ import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;/**- * Fast, schema-less unit test for {@link AwsJsonCborController#nodeToSmithyCbor(JsonNode)}.+ * Fast, schema-less unit test for {@link AwsJsonCborController}'s two CBOR timestamp+ * dialects.* <p>- * Verifies that timestamp shapes are encoded as CBOR tag(1) values per the- * smithy-rpc-v2-cbor spec, for both scalar timestamps and timestamp lists, and that- * integral epoch seconds are preserved as CBOR integers (not floats). CBOR tag 1 is the- * single byte {@code 0xC1} (major type 6, value 1) immediately preceding the encoded- * number. These assertions are deterministic and do not boot Quarkus.+ * There are genuinely two different tag(1) conventions sharing the same CBOR tag+ * mechanism, not one protocol with an inconsistency (see floci-io/floci#2368's+ * resolution):+ * <ul>+ * <li>{@link AwsJsonCborController#nodeToSmithyCbor} / plain {@link+ * AwsJsonCborController#bodyToJson} - the smithy-rpc-v2-cbor protocol (CloudWatch+ * Metrics, confirmed via a Go SDK compat test), which uses RFC 8949 section 3.4.2's+ * own tag(1) convention: plain, unscaled epoch seconds.</li>+ * <li>{@link AwsJsonCborController#nodeToLegacyCbor} / {@link+ * AwsJsonCborController#normalizeLegacyCborTimestampsFromMillis} - the older,+ * separate {@code application/x-amz-cbor-1.1} dialect (Kinesis, confirmed via a real+ * wire capture), whose tag(1) is epoch <em>milliseconds</em> instead.</li>+ * </ul>+ * This file previously (before #2368's resolution was itself corrected) asserted that+ * {@code nodeToSmithyCbor} scales to milliseconds - that was wrong for the+ * smithy-rpc-v2-cbor protocol it actually covers (it broke CloudWatch's+ * {@code GetMetricStatistics}/{@code GetMetricData}, whose timestamps came back+ * outside any query window once wrongly divided by 1000) and is why the two dialects+ * now have distinctly-named methods rather than one shared, ambiguous pair. CBOR tag 1+ * is the single byte {@code 0xC1} (major type 6, value 1) immediately preceding the+ * encoded number. These assertions are deterministic and do not boot Quarkus.* <p>* The end-to-end acceptance gate that tag(1)+integer is actually accepted by* aws-sdk-go-v2 / smithy-go is the OTel awscloudwatch receiver run; this test guards the@@ -47,14 +64,14 @@ class AwsJsonCborSerializerTest {return (headByte & 0xFF) >> 5;}+ // --- smithy-rpc-v2-cbor (nodeToSmithyCbor) - plain, unscaled epoch seconds ---+@Test- void scalarTimestampIsTaggedAsInteger() throws Exception {+ void smithyRpcV2ScalarTimestampIsTaggedAsIntegerUnscaled() throws Exception {JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000}");byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);- // Exactly one tag(1) byte, and it is immediately followed by an unsigned integer- // (major type 0) — i.e. integer-ness is preserved, not coerced to a float.assertEquals(1, countTag1(cbor), "scalar Timestamp must be tagged exactly once");int tagIndex = indexOf(cbor, CBOR_TAG_1);assertTrue(tagIndex >= 0, "tag(1) byte must be present");@@ -62,21 +79,86 @@ class AwsJsonCborSerializerTest {assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),"integral epoch seconds must be encoded as a CBOR integer, not a float");+ // smithy-rpc-v2-cbor's tag(1) is plain epoch seconds - the value round-trips+ // unchanged, unlike the legacy dialect's millisecond scaling below.JsonNode decoded = cborMapper.readTree(cbor);assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded timestamp must be integral");assertEquals(1700000000L, decoded.path("Timestamp").asLong());}@Test- void timestampListTagsEveryElementAsInteger() throws Exception {- // Core regression guard for CloudWatch GetMetricData: every element of a- // Timestamps list must be tagged (not just a scalar named "Timestamp").- JsonNode node = jsonMapper.readTree("{\"Timestamps\": [1700000000, 1700000060]}");+ void smithyRpcV2FractionalTimestampIsTaggedAsFloat() throws Exception {+ // A non-integral timestamp is a valid tag(1) floating-point value (RFC 8949 3.4.2).+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.5}");++ byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);++ assertEquals(1, countTag1(cbor), "fractional Timestamp must still be tagged");+ int tagIndex = indexOf(cbor, CBOR_TAG_1);+ assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");+ assertEquals(CBOR_MAJOR_FLOAT, majorType(cbor[tagIndex + 1]),+ "fractional epoch seconds must be encoded as a CBOR float");+ assertEquals(1700000000.5, cborMapper.readTree(cbor).path("Timestamp").asDouble());+ }++ @Test+ void smithyRpcV2NonTimestampNumberIsNotTagged() throws Exception {+ // A plain numeric field (e.g. CloudWatch Period) must remain an untagged integer.+ JsonNode node = jsonMapper.readTree("{\"Period\": 60}");byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);+ assertEquals(0, countTag1(cbor), "non-timestamp numbers must not carry a tag(1) byte");+ assertFalse(contains(cbor, CBOR_TAG_1));+ assertEquals(60L, cborMapper.readTree(cbor).path("Period").asLong());+ }++ @Test+ void smithyRpcV2DecodedRequestTimestampIsNotScaled() throws Exception {+ // The read side (plain bodyToJson, no legacyDialect) must leave a decoded+ // timestamp-shaped field completely untouched - this is the exact regression+ // that broke CloudWatch's GetMetricStatistics/GetMetricData: a real value+ // wrongly divided by 1000 landed outside any query window.+ byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(jsonMapper.readTree("{\"Timestamp\": 1700000000}"));++ JsonNode decoded = cborMapper.readTree(cbor);++ assertEquals(1700000000L, decoded.path("Timestamp").asLong());+ }++ // --- legacy application/x-amz-cbor-1.1 (nodeToLegacyCbor) - epoch milliseconds ---++ @Test+ void legacyScalarTimestampIsTaggedAsIntegerMillis() throws Exception {+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000}");++ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);++ // Exactly one tag(1) byte, and it is immediately followed by an unsigned integer+ // (major type 0) — i.e. integer-ness is preserved, not coerced to a float.+ assertEquals(1, countTag1(cbor), "scalar Timestamp must be tagged exactly once");+ int tagIndex = indexOf(cbor, CBOR_TAG_1);+ assertTrue(tagIndex >= 0, "tag(1) byte must be present");+ assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");+ assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),+ "epoch millis must be encoded as a CBOR integer, not a float");++ // 1700000000 epoch SECONDS (this codebase's internal convention) must be written+ // under tag(1) as 1700000000000 epoch MILLISECONDS (the legacy dialect's real+ // convention) — not the same number unchanged, which is exactly the+ // pre-#2368-fix bug for this dialect.+ JsonNode decoded = cborMapper.readTree(cbor);+ assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded timestamp must be integral");+ assertEquals(1700000000000L, decoded.path("Timestamp").asLong());+ }++ @Test+ void legacyTimestampListTagsEveryElementAsIntegerMillis() throws Exception {+ JsonNode node = jsonMapper.readTree("{\"Timestamps\": [1700000000, 1700000060]}");++ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);+assertEquals(2, countTag1(cbor), "each Timestamps element must be tagged with tag(1)");- // Both tagged elements must be integers, not floats.for (int i = 0; i < cbor.length; i++) {if (cbor[i] == CBOR_TAG_1) {assertTrue(i + 1 < cbor.length, "tag(1) byte at index " + i + " must not be the last byte in the output");@@ -89,43 +171,81 @@ class AwsJsonCborSerializerTest {JsonNode ts = decoded.path("Timestamps");assertTrue(ts.isArray());assertEquals(2, ts.size());- assertEquals(1700000000L, ts.get(0).asLong());- assertEquals(1700000060L, ts.get(1).asLong());+ assertEquals(1700000000000L, ts.get(0).asLong());+ assertEquals(1700000060000L, ts.get(1).asLong());}@Test- void suffixedTimestampFieldIsTagged() throws Exception {- // Future-proofing: the heuristic matches by "Timestamp" suffix, so AWS fields such- // as StateUpdatedTimestamp are tagged even though Floci does not emit them today.+ void legacySuffixedTimestampFieldIsTagged() throws Exception {+ // Future-proofing: the heuristic matches by "Timestamp" suffix.JsonNode node = jsonMapper.readTree("{\"StateUpdatedTimestamp\": 1700000000}");- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);assertEquals(1, countTag1(cbor), "a *Timestamp-suffixed field must be tagged");- assertEquals(1700000000L, cborMapper.readTree(cbor).path("StateUpdatedTimestamp").asLong());+ assertEquals(1700000000000L, cborMapper.readTree(cbor).path("StateUpdatedTimestamp").asLong());}@Test- void fractionalTimestampIsTaggedAsFloat() throws Exception {- // A non-integral timestamp is a valid tag(1) floating-point value (RFC 8949 3.4.2).- JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.5}");+ void legacyMillisecondPrecisionFractionalTimestampBecomesAnExactIntegerMillisValue() throws Exception {+ // A fractional epoch-SECONDS value with real AWS's own millisecond-precision limit+ // (3 decimal places) converts to an EXACT integer number of milliseconds - not a+ // float - once written under tag(1). This is the realistic case (AWS timestamps+ // never carry sub-millisecond precision) and the actual reproducer from+ // floci-io/floci#2368: 1700000000.712s -> 1700000000712ms, exactly.+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.712}");- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);- assertEquals(1, countTag1(cbor), "fractional Timestamp must still be tagged");int tagIndex = indexOf(cbor, CBOR_TAG_1);+ assertTrue(tagIndex >= 0, "tag(1) byte must be present");assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");- assertEquals(CBOR_MAJOR_FLOAT, majorType(cbor[tagIndex + 1]),- "fractional epoch seconds must be encoded as a CBOR float");- assertEquals(1700000000.5, cborMapper.readTree(cbor).path("Timestamp").asDouble());+ assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),+ "millisecond-precision epoch seconds must convert to an exact integer number "+ + "of milliseconds, not a float");++ JsonNode decoded = cborMapper.readTree(cbor);+ assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded millis value must be integral");+ assertEquals(1700000000712L, decoded.path("Timestamp").asLong());+ }++ @Test+ void legacyRoundTripsThroughEncodeAndDecodeBackToTheOriginalEpochSecondsValue() throws Exception {+ // The actual end-to-end guarantee that matters: whatever+ // normalizeLegacyCborTimestampsFromMillis decodes back out must equal what+ // nodeToLegacyCbor was given, for both an integral and a millisecond-precision+ // fractional epoch-seconds value - proving the two directions are genuine+ // inverses of each other, not just independently plausible.+ JsonNode integral = jsonMapper.readTree("{\"Timestamp\": 1700000000}");+ JsonNode fractional = jsonMapper.readTree("{\"Timestamp\": 1700000000.712}");++ JsonNode decodedIntegral = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(+ cborMapper.readTree(AwsJsonCborController.nodeToLegacyCbor(integral)));+ JsonNode decodedFractional = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(+ cborMapper.readTree(AwsJsonCborController.nodeToLegacyCbor(fractional)));++ assertEquals(1700000000, decodedIntegral.path("Timestamp").decimalValue().doubleValue());+ assertEquals(1700000000.712, decodedFractional.path("Timestamp").decimalValue().doubleValue());}@Test- void nonTimestampNumberIsNotTagged() throws Exception {+ void legacyNonTimestampFieldIsNotConvertedByNormalization() {+ // normalizeLegacyCborTimestampsFromMillis must use the identical field-name+ // heuristic as the writer - a plain numeric field (e.g. CloudWatch Period) must+ // pass through completely unchanged.+ JsonNode node = jsonMapper.createObjectNode().put("Period", 60);++ JsonNode normalized = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(node);++ assertEquals(60L, normalized.path("Period").asLong());+ }++ @Test+ void legacyNonTimestampNumberIsNotTagged() throws Exception {// A plain numeric field (e.g. CloudWatch Period) must remain an untagged integer.JsonNode node = jsonMapper.readTree("{\"Period\": 60}");- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);assertEquals(0, countTag1(cbor), "non-timestamp numbers must not carry a tag(1) byte");assertFalse(contains(cbor, CBOR_TAG_1));