Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis-cbor): timestamps mis-scaled 1000x on the CBOR path (#2432)

sheldon-nadarajah-sft39ed7712a719
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java
index 9ca686e319..076d61dffc 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonCborController.java
@@ -3,6 +3,9 @@ package io.github.hectorvent.floci.core.common;
import com.fasterxml.jackson.core.JacksonException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.DecimalNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
import com.google.gson.JsonParseException;
@@ -22,6 +25,8 @@ import org.jboss.logging.Logger;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -76,18 +81,40 @@ public class AwsJsonCborController {
/**
- * Serializes a JsonNode to CBOR bytes, encoding timestamp shapes with CBOR tag 1
- * (epoch seconds, RFC 8949 section 3.4.2) as required by the smithy-rpc-v2-cbor
- * protocol specification.
+ * Serializes a JsonNode to CBOR bytes for the smithy-rpc-v2-cbor protocol, encoding
+ * timestamp shapes with CBOR tag 1 as plain epoch seconds (RFC 8949 section 3.4.2,
+ * unmodified) — this protocol uses RFC 8949's own native tag(1) convention, unlike
+ * the older, separate {@code application/x-amz-cbor-1.1} dialect (see
+ * {@link #nodeToLegacyCbor}), which encodes tag(1) as epoch milliseconds instead.
+ * These are two genuinely different wire conventions sharing the same CBOR tag
+ * mechanism, not one protocol with an inconsistency — see floci-io/floci#2368's
+ * resolution for the investigation that distinguished them (CloudWatch Metrics, on
+ * this protocol, needs the value unscaled; Kinesis, on the legacy dialect, needs it
+ * scaled to milliseconds).
* <p>
* Package-private so the timestamp-tagging behaviour can be unit-tested directly
* without booting Quarkus.
*/
static byte[] nodeToSmithyCbor(JsonNode node) throws Exception {
+ return writeCbor(node, false);
+ }
+
+ /**
+ * Serializes a JsonNode to CBOR bytes for the legacy {@code application/x-amz-cbor-1.1}
+ * dialect (routed through {@link #handleCborRequest}) — encodes timestamp shapes as
+ * CBOR tag(1) epoch <em>milliseconds</em>, AWS's convention for this specific,
+ * pre-Smithy dialect. See {@link #nodeToSmithyCbor} for how this differs from the
+ * newer smithy-rpc-v2-cbor protocol, and {@link #writeCborTimestamp} for why.
+ */
+ static byte[] nodeToLegacyCbor(JsonNode node) throws Exception {
+ return writeCbor(node, true);
+ }
+
+ private static byte[] writeCbor(JsonNode node, boolean legacyDialect) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
CBORFactory factory = (CBORFactory) CBOR_MAPPER.getFactory();
try (CBORGenerator gen = factory.createGenerator(out)) {
- writeNodeToCbor(gen, node, false);
+ writeNodeToCbor(gen, node, false, legacyDialect);
}
return out.toByteArray();
}
@@ -98,16 +125,18 @@ public class AwsJsonCborController {
* up the Smithy shape): when set and the node is a number, it is emitted as a CBOR
* tag(1) timestamp. The flag is set by {@link #isTimestampField(String)} for matching
* object fields and is propagated into array elements, so both scalar timestamps and
- * elements of timestamp lists are tagged.
+ * elements of timestamp lists are tagged. {@code legacyDialect} selects which of the
+ * two wire conventions tag(1) uses - see {@link #nodeToSmithyCbor}/{@link #nodeToLegacyCbor}.
*/
- private static void writeNodeToCbor(CBORGenerator gen, JsonNode node, boolean numberIsTimestamp) throws Exception {
+ private static void writeNodeToCbor(CBORGenerator gen, JsonNode node, boolean numberIsTimestamp,
+ boolean legacyDialect) throws Exception {
if (node.isObject()) {
gen.writeStartObject();
Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
while (fields.hasNext()) {
Map.Entry<String, JsonNode> entry = fields.next();
gen.writeFieldName(entry.getKey());
- writeNodeToCbor(gen, entry.getValue(), isTimestampField(entry.getKey()));
+ writeNodeToCbor(gen, entry.getValue(), isTimestampField(entry.getKey()), legacyDialect);
}
gen.writeEndObject();
} else if (node.isArray()) {
@@ -115,11 +144,11 @@ public class AwsJsonCborController {
for (JsonNode item : node) {
// Propagate the timestamp context so every element of a timestamp list
// (e.g. CloudWatch GetMetricData MetricDataResults[].Timestamps) is tagged.
- writeNodeToCbor(gen, item, numberIsTimestamp);
+ writeNodeToCbor(gen, item, numberIsTimestamp, legacyDialect);
}
gen.writeEndArray();
} else if (numberIsTimestamp && node.isNumber()) {
- writeCborTimestamp(gen, node);
+ writeCborTimestamp(gen, node, legacyDialect);
} else if (node.isTextual()) {
gen.writeString(node.textValue());
} else if (node.isDouble() || node.isFloat()) {
@@ -153,18 +182,95 @@ public class AwsJsonCborController {
}
/**
- * Writes a numeric node as a CBOR tag(1) epoch-seconds timestamp, preserving
- * integer-ness. Integral epoch seconds are emitted as a tag(1) integer and
- * fractional values as a tag(1) float; both are spec-valid (RFC 8949 section 3.4.2)
- * and accepted by AWS SDK decoders.
+ * Writes a numeric node as a CBOR tag(1) timestamp. On the legacy
+ * {@code application/x-amz-cbor-1.1} dialect, converts from this codebase's internal
+ * convention (fractional epoch seconds, matching the plain-JSON protocol) to epoch
+ * milliseconds - AWS's convention for tag(1) on that specific, pre-Smithy dialect.
+ * See floci-io/floci#2368: without this conversion, an unmodified AWS SDK for Java v2
+ * Kinesis client (which uses this legacy dialect by default) decodes every timestamp
+ * 1000x too small, and an {@code AT_TIMESTAMP} shard iterator built from a corrupted
+ * request-side timestamp never matches any real record. On smithy-rpc-v2-cbor,
+ * RFC 8949's own tag(1) convention (plain epoch seconds) already applies unmodified -
+ * no conversion needed, or correct, there.
+ * <p>
+ * {@code BigDecimal} (not double arithmetic) preserves millisecond precision exactly
+ * and avoids reintroducing the same class of float-precision defect fixed on the
+ * read side of this exact value by #2173/#2359 — {@link JsonNode#decimalValue()}
+ * already handles both integral and floating-point nodes correctly.
*/
- private static void writeCborTimestamp(CBORGenerator gen, JsonNode node) throws Exception {
+ private static void writeCborTimestamp(CBORGenerator gen, JsonNode node, boolean legacyDialect) throws Exception {
gen.writeTag(1);
- if (node.isIntegralNumber()) {
- gen.writeNumber(node.longValue());
- } else {
- gen.writeNumber(node.doubleValue());
+ if (!legacyDialect) {
+ if (node.isIntegralNumber()) {
+ gen.writeNumber(node.longValue());
+ } else {
+ gen.writeNumber(node.doubleValue());
+ }
+ return;
}
+ BigDecimal epochSeconds = node.decimalValue();
+ long epochMillis = epochSeconds.movePointRight(3).setScale(0, RoundingMode.HALF_UP).longValueExact();
+ gen.writeNumber(epochMillis);
+ }
+
+ /**
+ * Converts every timestamp-shaped field's value in a CBOR request/response decoded
+ * off the legacy {@code application/x-amz-cbor-1.1} dialect from epoch milliseconds
+ * (that dialect's tag(1) wire representation) back to this codebase's internal
+ * convention of fractional epoch seconds — the inverse of {@link #writeCborTimestamp}
+ * for that same dialect. See floci-io/floci#2368, and {@link #nodeToSmithyCbor} for
+ * why this must NOT be applied to smithy-rpc-v2-cbor traffic (its tag(1) is already
+ * plain epoch seconds, unscaled).
+ * <p>
+ * Jackson's CBOR module does not preserve tag information into the decoded
+ * {@link JsonNode} tree ({@link #bodyToJson} uses a plain {@code readTree}), so by
+ * the time a request reaches this method the CBOR tag(1) marker is already gone and
+ * a decoded timestamp's raw numeric value is indistinguishable from any other number.
+ * This reuses the same schema-less field-name heuristic ({@link #isTimestampField})
+ * {@link #writeNodeToCbor} already relies on for the identical reason on the response
+ * side, so every service handler continues to see the one convention it already
+ * expects regardless of which wire protocol (legacy CBOR or plain JSON) actually
+ * carried the request — no handler needs to change.
+ * <p>
+ * Mutates the given tree in place (both {@link ObjectNode} and {@link ArrayNode} are
+ * mutable) and returns it, purely for convenient chaining at the call site.
+ */
+ static JsonNode normalizeLegacyCborTimestampsFromMillis(JsonNode node) {
+ normalizeLegacyCborTimestampsFromMillis(node, false);
+ return node;
+ }
+
+ private static void normalizeLegacyCborTimestampsFromMillis(JsonNode node, boolean isTimestamp) {
+ if (node.isObject()) {
+ ObjectNode object = (ObjectNode) node;
+ Iterator<Map.Entry<String, JsonNode>> fields = object.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> entry = fields.next();
+ boolean fieldIsTimestamp = isTimestampField(entry.getKey());
+ JsonNode value = entry.getValue();
+ if (fieldIsTimestamp && value.isNumber()) {
+ object.set(entry.getKey(), millisToSeconds(value));
+ } else {
+ normalizeLegacyCborTimestampsFromMillis(value, fieldIsTimestamp);
+ }
+ }
+ } else if (node.isArray()) {
+ ArrayNode array = (ArrayNode) node;
+ for (int i = 0; i < array.size(); i++) {
+ JsonNode item = array.get(i);
+ if (isTimestamp && item.isNumber()) {
+ array.set(i, millisToSeconds(item));
+ } else {
+ normalizeLegacyCborTimestampsFromMillis(item, isTimestamp);
+ }
+ }
+ }
+ }
+
+ /** Exact millis-to-seconds conversion (see {@link #normalizeLegacyCborTimestampsFromMillis}). */
+ private static JsonNode millisToSeconds(JsonNode millisNode) {
+ BigDecimal epochMillis = millisNode.decimalValue();
+ return DecimalNode.valueOf(epochMillis.movePointLeft(3));
}
/**
@@ -217,12 +323,24 @@ public class AwsJsonCborController {
}
JsonNode bodyToJson(HttpHeaders httpHeaders, byte[] body) throws IOException {
+ return bodyToJson(httpHeaders, body, false);
+ }
+
+ /**
+ * {@code legacyDialect} selects whether a decoded numeric timestamp field is
+ * interpreted as the legacy {@code application/x-amz-cbor-1.1} dialect's epoch
+ * milliseconds ({@link #normalizeLegacyCborTimestampsFromMillis}) or left as-is
+ * (smithy-rpc-v2-cbor's own plain epoch seconds, already correct unmodified) — see
+ * {@link #nodeToSmithyCbor} for why these differ.
+ */
+ private JsonNode bodyToJson(HttpHeaders httpHeaders, byte[] body, boolean legacyDialect) throws IOException {
JsonNode request;
if (body != null && body.length > 0) {
if( httpHeaders.getRequestHeader("Content-encoding") != null && isGZipped(httpHeaders.getRequestHeader("Content-encoding"))) {
body = decodeBody(body);
}
- request = CBOR_MAPPER.readTree(body);
+ JsonNode decoded = CBOR_MAPPER.readTree(body);
+ request = legacyDialect ? normalizeLegacyCborTimestampsFromMillis(decoded) : decoded;
} else {
request = objectMapper.createObjectNode();
}
@@ -245,6 +363,14 @@ public class AwsJsonCborController {
/**
* Handles AWS services that migrated to the smithy-rpc-v2-cbor protocol at root path.
* Fallback handler for X-Amz-Target based routing with CBOR body.
+ * <p>
+ * Confirmed (via a real wire capture, floci-io/floci#2368) that Kinesis reaches this
+ * handler using the older, separate {@code application/x-amz-cbor-1.1} dialect, whose
+ * tag(1) timestamps are epoch milliseconds rather than smithy-rpc-v2-cbor's plain
+ * epoch seconds — see {@link #nodeToLegacyCbor}/{@link #bodyToJson}'s {@code
+ * legacyDialect} parameter, applied here for that reason. Not reconfirmed for every
+ * other service dispatched below; none currently has test coverage asserting an
+ * exact numeric timestamp value over this specific path.
*/
@POST
@Consumes({GENERIC_CBOR_MEDIA_TYPE, AWS_CBOR_1_1_MEDIA_TYPE})
@@ -272,7 +398,7 @@ public class AwsJsonCborController {
LOG.debugv("{0} CBOR action: {1}", serviceKey, action);
try {
- JsonNode request = bodyToJson(httpHeaders, body);
+ JsonNode request = bodyToJson(httpHeaders, body, true);
String region = regionResolver.resolveRegion(httpHeaders);
Response delegated = switch (serviceKey) {
@@ -296,7 +422,7 @@ public class AwsJsonCborController {
JsonNode responseNode = delegated.getEntity() instanceof JsonNode
? (JsonNode) delegated.getEntity()
: objectMapper.valueToTree(delegated.getEntity());
- byte[] cborBytes = nodeToSmithyCbor(responseNode);
+ byte[] cborBytes = nodeToLegacyCbor(responseNode);
String responseContentType = responseContentType(httpHeaders);
return Response.status(delegated.getStatus())
.header("smithy-protocol", "rpc-v2-cbor")
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.java
new file mode 100644
index 0000000000..3cbf6c925f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborCloudWatchTimestampIntegrationTest.java
@@ -0,0 +1,96 @@
+package io.github.hectorvent.floci.core.common;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Regression guard for the exact bug this repo's own {@code AwsJsonCborSerializerTest}
+ * originally introduced while fixing floci-io/floci#2368: scaling smithy-rpc-v2-cbor
+ * timestamps to milliseconds (correct for the legacy {@code application/x-amz-cbor-1.1}
+ * dialect, per the same issue) broke CloudWatch Metrics, whose {@code PutMetricData}/
+ * {@code GetMetricStatistics} round-trip a {@code Timestamp} field over the
+ * smithy-rpc-v2-cbor protocol - which uses RFC 8949's plain, unscaled epoch-seconds
+ * tag(1) convention. A wrongly-scaled timestamp lands outside any query window, so
+ * {@code GetMetricStatistics} silently returns zero datapoints (matches the real Go SDK
+ * compat test failure this was caught by, {@code TestCloudWatch/GetMetricStatistics}
+ * asserting {@code Datapoints} non-empty).
+ * <p>
+ * Drives the real HTTP endpoint end-to-end via the same {@code
+ * POST /service/GraniteServiceVersion20100801/operation/{action}} routing
+ * {@link SmithyRpcV2RoutingIntegrationTest#cloudWatchRoutesViaGraniteServiceName} uses,
+ * with request bodies built via {@link AwsJsonCborController#nodeToSmithyCbor} - the
+ * correct (unscaled) encoder for this protocol - so this test would fail the same way
+ * the real regression did if that scaling were ever mistakenly reapplied here.
+ */
+@QuarkusTest
+class AwsJsonCborCloudWatchTimestampIntegrationTest {
+
+ private static final ObjectMapper JSON = new ObjectMapper();
+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());
+
+ @Test
+ void getMetricStatisticsFindsADatapointPublishedJustNow() throws Exception {
+ String namespace = "CborRpcV2TimestampTest-" + System.nanoTime();
+ Instant now = Instant.now();
+
+ ObjectNode putRequest = JSON.createObjectNode();
+ putRequest.put("Namespace", namespace);
+ ArrayNode metricData = putRequest.putArray("MetricData");
+ ObjectNode datum = metricData.addObject();
+ datum.put("MetricName", "RequestCount");
+ datum.put("Value", 42.0);
+ datum.put("Unit", "Count");
+ datum.put("Timestamp", now.getEpochSecond());
+ rpcV2Call("PutMetricData", putRequest);
+
+ ObjectNode statsRequest = JSON.createObjectNode();
+ statsRequest.put("Namespace", namespace);
+ statsRequest.put("MetricName", "RequestCount");
+ statsRequest.put("StartTime", now.minusSeconds(300).getEpochSecond());
+ statsRequest.put("EndTime", now.plusSeconds(60).getEpochSecond());
+ statsRequest.put("Period", 60);
+ statsRequest.putArray("Statistics").add("Sum");
+ JsonNode statsResponse = rpcV2Call("GetMetricStatistics", statsRequest);
+
+ // Before this fix, a stray millisecond-scaling of "Timestamp" here would have
+ // stored the datapoint far outside [StartTime, EndTime], and this would be 0 -
+ // exactly the real Go SDK compat test failure this class guards against.
+ assertFalse(statsResponse.path("Datapoints").isEmpty(),
+ "GetMetricStatistics must find the datapoint just published: " + statsResponse);
+
+ // Not an exact match: CloudWatch buckets a datapoint's own returned Timestamp
+ // to the start of its Period (60s here), so it legitimately differs from the
+ // submitted value by up to one period - a real, unrelated behaviour, not this
+ // fix's concern. What this DOES still catch: a 1000x scale error would push the
+ // value thousands of years away, far outside this window.
+ JsonNode datapoint = statsResponse.path("Datapoints").get(0);
+ long bucketedEpochSecond = datapoint.path("Timestamp").asLong();
+ long driftSeconds = Math.abs(now.getEpochSecond() - bucketedEpochSecond);
+ assertTrue(driftSeconds <= 60,
+ "the round-tripped Timestamp must stay in plain epoch seconds, unscaled (drifted "
+ + driftSeconds + "s): " + datapoint);
+ }
+
+ private static JsonNode rpcV2Call(String action, ObjectNode body) throws Exception {
+ byte[] responseCbor = given()
+ .contentType("application/cbor")
+ .accept("application/cbor")
+ .header("smithy-protocol", "rpc-v2-cbor")
+ .body(AwsJsonCborController.nodeToSmithyCbor(body))
+ .when().post("/service/GraniteServiceVersion20100801/operation/" + action)
+ .then().statusCode(200)
+ .extract().asByteArray();
+ return CBOR.readTree(responseCbor);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java
new file mode 100644
index 0000000000..5c43755fdf
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java
@@ -0,0 +1,120 @@
+package io.github.hectorvent.floci.core.common;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.time.Instant;
+import java.util.Base64;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+/**
+ * End-to-end reproduction of floci-io/floci#2368's actual reported symptom, driven
+ * through the real HTTP CBOR endpoint (not just the unit-level encode/decode round-trip
+ * already covered by {@link AwsJsonCborSerializerTest}): an {@code AT_TIMESTAMP} Kinesis
+ * shard iterator built over the CBOR wire protocol - the AWS SDK for Java v2's default
+ * transport - for a timestamp strictly before an already-written record.
+ * <p>
+ * Before the fix, this never found that record: {@code handleGetShardIterator} treated
+ * the decoded {@code Timestamp} as epoch seconds when the CBOR wire value a real client
+ * actually sends is epoch milliseconds, landing the iterator around the year 58598 -
+ * past every real record - so {@code GetRecords} returned an empty page forever, with no
+ * error at all.
+ * <p>
+ * The {@code GetShardIterator} request's {@code Timestamp} is hand-built with a raw
+ * tag(1) epoch-<em>millisecond</em> value via {@link #cborTag1MillisTimestampRequest},
+ * deliberately bypassing {@link AwsJsonCborController#nodeToLegacyCbor} (the method that
+ * would normally build this exact request, correctly, for the legacy
+ * {@code application/x-amz-cbor-1.1} dialect Kinesis uses) - building the request
+ * through Floci's own encoder would only prove Floci's own encoder and decoder agree
+ * with each other, not that Floci actually speaks the real AWS wire format a genuine
+ * CBOR-default SDK client sends. Hand-building the millis value directly is what makes
+ * this test fail against the pre-fix code and pass against the fix, matching the
+ * issue's own wire capture (tag(1) followed by an 8-byte integer, e.g.
+ * {@code c1 1b 000001a013accc38}).
+ * <p>
+ * The other requests here (none carry a timestamp field, so the choice of encoder makes
+ * no difference to their bytes) go through {@link AwsJsonCborController#nodeToSmithyCbor}
+ * as a matter of convenience, and CBOR response bodies are decoded with a plain CBOR
+ * {@link ObjectMapper} since the server mirrors the request's own content type back on
+ * the response.
+ */
+@QuarkusTest
+class AwsJsonCborKinesisAtTimestampIntegrationTest {
+
+ private static final String CBOR_CONTENT_TYPE = "application/x-amz-cbor-1.1";
+ private static final ObjectMapper JSON = new ObjectMapper();
+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());
+
+ @Test
+ void atTimestampShardIteratorOverCborFindsAnAlreadyWrittenRecord() throws Exception {
+ String streamName = "cbor-at-timestamp-" + System.nanoTime();
+
+ cborCall("Kinesis_20131202.CreateStream",
+ AwsJsonCborController.nodeToSmithyCbor(
+ JSON.createObjectNode().put("StreamName", streamName).put("ShardCount", 1)));
+
+ // Cutoff strictly before the record about to be written.
+ Instant cutoff = Instant.now().minusSeconds(30);
+
+ cborCall("Kinesis_20131202.PutRecord", AwsJsonCborController.nodeToSmithyCbor(JSON.createObjectNode()
+ .put("StreamName", streamName)
+ .put("Data", Base64.getEncoder().encodeToString("hello".getBytes()))
+ .put("PartitionKey", "pk")));
+
+ byte[] getShardIteratorRequest = cborTag1MillisTimestampRequest(
+ streamName, "shardId-000000000000", cutoff.toEpochMilli());
+ JsonNode shardIteratorResponse = cborCall("Kinesis_20131202.GetShardIterator", getShardIteratorRequest);
+ String shardIterator = shardIteratorResponse.path("ShardIterator").asText();
+ assertFalse(shardIterator.isEmpty(), "GetShardIterator must actually return an iterator");
+
+ JsonNode records = cborCall("Kinesis_20131202.GetRecords", AwsJsonCborController.nodeToSmithyCbor(
+ JSON.createObjectNode().put("ShardIterator", shardIterator)));
+
+ // Before the #2368 fix, this was always 0 regardless of how long the caller waited.
+ assertEquals(1, records.path("Records").size(),
+ "AT_TIMESTAMP over CBOR must find the record written after the cutoff");
+ }
+
+ /**
+ * Hand-builds a {@code GetShardIterator} CBOR request body with {@code Timestamp}
+ * encoded exactly as a real AWS SDK for Java v2 client sends it: CBOR tag(1)
+ * immediately followed by an integer epoch-<em>millisecond</em> value - see this
+ * class's own javadoc for why this must not go through
+ * {@link AwsJsonCborController#nodeToSmithyCbor}.
+ */
+ private static byte[] cborTag1MillisTimestampRequest(String streamName, String shardId, long epochMillis)
+ throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ CBORFactory factory = (CBORFactory) CBOR.getFactory();
+ try (CBORGenerator gen = factory.createGenerator(out)) {
+ gen.writeStartObject();
+ gen.writeStringField("StreamName", streamName);
+ gen.writeStringField("ShardId", shardId);
+ gen.writeStringField("ShardIteratorType", "AT_TIMESTAMP");
+ gen.writeFieldName("Timestamp");
+ gen.writeTag(1);
+ gen.writeNumber(epochMillis);
+ gen.writeEndObject();
+ }
+ return out.toByteArray();
+ }
+
+ private static JsonNode cborCall(String target, byte[] requestCbor) throws Exception {
+ byte[] responseCbor = given()
+ .header("X-Amz-Target", target)
+ .contentType(CBOR_CONTENT_TYPE)
+ .body(requestCbor)
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().asByteArray();
+ return CBOR.readTree(responseCbor);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java
index c2a0e6c78f..0927e7e69f 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborSerializerTest.java
@@ -10,13 +10,30 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
- * Fast, schema-less unit test for {@link AwsJsonCborController#nodeToSmithyCbor(JsonNode)}.
+ * Fast, schema-less unit test for {@link AwsJsonCborController}'s two CBOR timestamp
+ * dialects.
* <p>
- * Verifies that timestamp shapes are encoded as CBOR tag(1) values per the
- * smithy-rpc-v2-cbor spec, for both scalar timestamps and timestamp lists, and that
- * integral epoch seconds are preserved as CBOR integers (not floats). CBOR tag 1 is the
- * single byte {@code 0xC1} (major type 6, value 1) immediately preceding the encoded
- * number. These assertions are deterministic and do not boot Quarkus.
+ * There are genuinely two different tag(1) conventions sharing the same CBOR tag
+ * mechanism, not one protocol with an inconsistency (see floci-io/floci#2368's
+ * resolution):
+ * <ul>
+ * <li>{@link AwsJsonCborController#nodeToSmithyCbor} / plain {@link
+ * AwsJsonCborController#bodyToJson} - the smithy-rpc-v2-cbor protocol (CloudWatch
+ * Metrics, confirmed via a Go SDK compat test), which uses RFC 8949 section 3.4.2's
+ * own tag(1) convention: plain, unscaled epoch seconds.</li>
+ * <li>{@link AwsJsonCborController#nodeToLegacyCbor} / {@link
+ * AwsJsonCborController#normalizeLegacyCborTimestampsFromMillis} - the older,
+ * separate {@code application/x-amz-cbor-1.1} dialect (Kinesis, confirmed via a real
+ * wire capture), whose tag(1) is epoch <em>milliseconds</em> instead.</li>
+ * </ul>
+ * This file previously (before #2368's resolution was itself corrected) asserted that
+ * {@code nodeToSmithyCbor} scales to milliseconds - that was wrong for the
+ * smithy-rpc-v2-cbor protocol it actually covers (it broke CloudWatch's
+ * {@code GetMetricStatistics}/{@code GetMetricData}, whose timestamps came back
+ * outside any query window once wrongly divided by 1000) and is why the two dialects
+ * now have distinctly-named methods rather than one shared, ambiguous pair. CBOR tag 1
+ * is the single byte {@code 0xC1} (major type 6, value 1) immediately preceding the
+ * encoded number. These assertions are deterministic and do not boot Quarkus.
* <p>
* The end-to-end acceptance gate that tag(1)+integer is actually accepted by
* aws-sdk-go-v2 / smithy-go is the OTel awscloudwatch receiver run; this test guards the
@@ -47,14 +64,14 @@ class AwsJsonCborSerializerTest {
return (headByte & 0xFF) >> 5;
}
+ // --- smithy-rpc-v2-cbor (nodeToSmithyCbor) - plain, unscaled epoch seconds ---
+
@Test
- void scalarTimestampIsTaggedAsInteger() throws Exception {
+ void smithyRpcV2ScalarTimestampIsTaggedAsIntegerUnscaled() throws Exception {
JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000}");
byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
- // Exactly one tag(1) byte, and it is immediately followed by an unsigned integer
- // (major type 0) — i.e. integer-ness is preserved, not coerced to a float.
assertEquals(1, countTag1(cbor), "scalar Timestamp must be tagged exactly once");
int tagIndex = indexOf(cbor, CBOR_TAG_1);
assertTrue(tagIndex >= 0, "tag(1) byte must be present");
@@ -62,21 +79,86 @@ class AwsJsonCborSerializerTest {
assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),
"integral epoch seconds must be encoded as a CBOR integer, not a float");
+ // smithy-rpc-v2-cbor's tag(1) is plain epoch seconds - the value round-trips
+ // unchanged, unlike the legacy dialect's millisecond scaling below.
JsonNode decoded = cborMapper.readTree(cbor);
assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded timestamp must be integral");
assertEquals(1700000000L, decoded.path("Timestamp").asLong());
}
@Test
- void timestampListTagsEveryElementAsInteger() throws Exception {
- // Core regression guard for CloudWatch GetMetricData: every element of a
- // Timestamps list must be tagged (not just a scalar named "Timestamp").
- JsonNode node = jsonMapper.readTree("{\"Timestamps\": [1700000000, 1700000060]}");
+ void smithyRpcV2FractionalTimestampIsTaggedAsFloat() throws Exception {
+ // A non-integral timestamp is a valid tag(1) floating-point value (RFC 8949 3.4.2).
+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.5}");
+
+ byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
+
+ assertEquals(1, countTag1(cbor), "fractional Timestamp must still be tagged");
+ int tagIndex = indexOf(cbor, CBOR_TAG_1);
+ assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");
+ assertEquals(CBOR_MAJOR_FLOAT, majorType(cbor[tagIndex + 1]),
+ "fractional epoch seconds must be encoded as a CBOR float");
+ assertEquals(1700000000.5, cborMapper.readTree(cbor).path("Timestamp").asDouble());
+ }
+
+ @Test
+ void smithyRpcV2NonTimestampNumberIsNotTagged() throws Exception {
+ // A plain numeric field (e.g. CloudWatch Period) must remain an untagged integer.
+ JsonNode node = jsonMapper.readTree("{\"Period\": 60}");
byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
+ assertEquals(0, countTag1(cbor), "non-timestamp numbers must not carry a tag(1) byte");
+ assertFalse(contains(cbor, CBOR_TAG_1));
+ assertEquals(60L, cborMapper.readTree(cbor).path("Period").asLong());
+ }
+
+ @Test
+ void smithyRpcV2DecodedRequestTimestampIsNotScaled() throws Exception {
+ // The read side (plain bodyToJson, no legacyDialect) must leave a decoded
+ // timestamp-shaped field completely untouched - this is the exact regression
+ // that broke CloudWatch's GetMetricStatistics/GetMetricData: a real value
+ // wrongly divided by 1000 landed outside any query window.
+ byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(jsonMapper.readTree("{\"Timestamp\": 1700000000}"));
+
+ JsonNode decoded = cborMapper.readTree(cbor);
+
+ assertEquals(1700000000L, decoded.path("Timestamp").asLong());
+ }
+
+ // --- legacy application/x-amz-cbor-1.1 (nodeToLegacyCbor) - epoch milliseconds ---
+
+ @Test
+ void legacyScalarTimestampIsTaggedAsIntegerMillis() throws Exception {
+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000}");
+
+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);
+
+ // Exactly one tag(1) byte, and it is immediately followed by an unsigned integer
+ // (major type 0) — i.e. integer-ness is preserved, not coerced to a float.
+ assertEquals(1, countTag1(cbor), "scalar Timestamp must be tagged exactly once");
+ int tagIndex = indexOf(cbor, CBOR_TAG_1);
+ assertTrue(tagIndex >= 0, "tag(1) byte must be present");
+ assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");
+ assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),
+ "epoch millis must be encoded as a CBOR integer, not a float");
+
+ // 1700000000 epoch SECONDS (this codebase's internal convention) must be written
+ // under tag(1) as 1700000000000 epoch MILLISECONDS (the legacy dialect's real
+ // convention) — not the same number unchanged, which is exactly the
+ // pre-#2368-fix bug for this dialect.
+ JsonNode decoded = cborMapper.readTree(cbor);
+ assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded timestamp must be integral");
+ assertEquals(1700000000000L, decoded.path("Timestamp").asLong());
+ }
+
+ @Test
+ void legacyTimestampListTagsEveryElementAsIntegerMillis() throws Exception {
+ JsonNode node = jsonMapper.readTree("{\"Timestamps\": [1700000000, 1700000060]}");
+
+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);
+
assertEquals(2, countTag1(cbor), "each Timestamps element must be tagged with tag(1)");
- // Both tagged elements must be integers, not floats.
for (int i = 0; i < cbor.length; i++) {
if (cbor[i] == CBOR_TAG_1) {
assertTrue(i + 1 < cbor.length, "tag(1) byte at index " + i + " must not be the last byte in the output");
@@ -89,43 +171,81 @@ class AwsJsonCborSerializerTest {
JsonNode ts = decoded.path("Timestamps");
assertTrue(ts.isArray());
assertEquals(2, ts.size());
- assertEquals(1700000000L, ts.get(0).asLong());
- assertEquals(1700000060L, ts.get(1).asLong());
+ assertEquals(1700000000000L, ts.get(0).asLong());
+ assertEquals(1700000060000L, ts.get(1).asLong());
}
@Test
- void suffixedTimestampFieldIsTagged() throws Exception {
- // Future-proofing: the heuristic matches by "Timestamp" suffix, so AWS fields such
- // as StateUpdatedTimestamp are tagged even though Floci does not emit them today.
+ void legacySuffixedTimestampFieldIsTagged() throws Exception {
+ // Future-proofing: the heuristic matches by "Timestamp" suffix.
JsonNode node = jsonMapper.readTree("{\"StateUpdatedTimestamp\": 1700000000}");
- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);
assertEquals(1, countTag1(cbor), "a *Timestamp-suffixed field must be tagged");
- assertEquals(1700000000L, cborMapper.readTree(cbor).path("StateUpdatedTimestamp").asLong());
+ assertEquals(1700000000000L, cborMapper.readTree(cbor).path("StateUpdatedTimestamp").asLong());
}
@Test
- void fractionalTimestampIsTaggedAsFloat() throws Exception {
- // A non-integral timestamp is a valid tag(1) floating-point value (RFC 8949 3.4.2).
- JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.5}");
+ void legacyMillisecondPrecisionFractionalTimestampBecomesAnExactIntegerMillisValue() throws Exception {
+ // A fractional epoch-SECONDS value with real AWS's own millisecond-precision limit
+ // (3 decimal places) converts to an EXACT integer number of milliseconds - not a
+ // float - once written under tag(1). This is the realistic case (AWS timestamps
+ // never carry sub-millisecond precision) and the actual reproducer from
+ // floci-io/floci#2368: 1700000000.712s -> 1700000000712ms, exactly.
+ JsonNode node = jsonMapper.readTree("{\"Timestamp\": 1700000000.712}");
- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);
- assertEquals(1, countTag1(cbor), "fractional Timestamp must still be tagged");
int tagIndex = indexOf(cbor, CBOR_TAG_1);
+ assertTrue(tagIndex >= 0, "tag(1) byte must be present");
assertTrue(tagIndex + 1 < cbor.length, "tag(1) byte must not be the last byte in the output");
- assertEquals(CBOR_MAJOR_FLOAT, majorType(cbor[tagIndex + 1]),
- "fractional epoch seconds must be encoded as a CBOR float");
- assertEquals(1700000000.5, cborMapper.readTree(cbor).path("Timestamp").asDouble());
+ assertEquals(CBOR_MAJOR_UNSIGNED_INT, majorType(cbor[tagIndex + 1]),
+ "millisecond-precision epoch seconds must convert to an exact integer number "
+ + "of milliseconds, not a float");
+
+ JsonNode decoded = cborMapper.readTree(cbor);
+ assertTrue(decoded.path("Timestamp").isIntegralNumber(), "decoded millis value must be integral");
+ assertEquals(1700000000712L, decoded.path("Timestamp").asLong());
+ }
+
+ @Test
+ void legacyRoundTripsThroughEncodeAndDecodeBackToTheOriginalEpochSecondsValue() throws Exception {
+ // The actual end-to-end guarantee that matters: whatever
+ // normalizeLegacyCborTimestampsFromMillis decodes back out must equal what
+ // nodeToLegacyCbor was given, for both an integral and a millisecond-precision
+ // fractional epoch-seconds value - proving the two directions are genuine
+ // inverses of each other, not just independently plausible.
+ JsonNode integral = jsonMapper.readTree("{\"Timestamp\": 1700000000}");
+ JsonNode fractional = jsonMapper.readTree("{\"Timestamp\": 1700000000.712}");
+
+ JsonNode decodedIntegral = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(
+ cborMapper.readTree(AwsJsonCborController.nodeToLegacyCbor(integral)));
+ JsonNode decodedFractional = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(
+ cborMapper.readTree(AwsJsonCborController.nodeToLegacyCbor(fractional)));
+
+ assertEquals(1700000000, decodedIntegral.path("Timestamp").decimalValue().doubleValue());
+ assertEquals(1700000000.712, decodedFractional.path("Timestamp").decimalValue().doubleValue());
}
@Test
- void nonTimestampNumberIsNotTagged() throws Exception {
+ void legacyNonTimestampFieldIsNotConvertedByNormalization() {
+ // normalizeLegacyCborTimestampsFromMillis must use the identical field-name
+ // heuristic as the writer - a plain numeric field (e.g. CloudWatch Period) must
+ // pass through completely unchanged.
+ JsonNode node = jsonMapper.createObjectNode().put("Period", 60);
+
+ JsonNode normalized = AwsJsonCborController.normalizeLegacyCborTimestampsFromMillis(node);
+
+ assertEquals(60L, normalized.path("Period").asLong());
+ }
+
+ @Test
+ void legacyNonTimestampNumberIsNotTagged() throws Exception {
// A plain numeric field (e.g. CloudWatch Period) must remain an untagged integer.
JsonNode node = jsonMapper.readTree("{\"Period\": 60}");
- byte[] cbor = AwsJsonCborController.nodeToSmithyCbor(node);
+ byte[] cbor = AwsJsonCborController.nodeToLegacyCbor(node);
assertEquals(0, countTag1(cbor), "non-timestamp numbers must not carry a tag(1) byte");
assertFalse(contains(cbor, CBOR_TAG_1));