Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): deliver flow log records in the format the flow log declares (#4429)
lex004b15ca89508c
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java b/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.javaindex e5c81f0976..2ac8f9122f 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ThreadLocalRandom;import java.util.concurrent.TimeUnit;+import java.util.regex.Matcher;+import java.util.regex.Pattern;import java.util.zip.GZIPOutputStream;/**@@ -59,22 +61,45 @@ public class FlowLogService {/*** The format a flow log is created with when CreateFlowLogs omits LogFormat, which* DescribeFlowLogs reports for it thereafter. Version 2 fields in the order AWS documents.- *- * <p>Distinct from {@link #DEFAULT_HEADER}, which is the header line of a generated log file.*/public static final String DEFAULT_LOG_FORMAT ="${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} "+ "${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}";/**- * Every available flow log field, alphabetised, written as the header line of a generated log- * file. Not a version ordering, and not what {@link #DEFAULT_LOG_FORMAT} describes.+ * Every field a flow log record can carry, against the format version that introduced it.+ *+ * <p>The version a record reports is the highest of the versions its selected fields belong+ * to, which is how AWS numbers a format. A record of version 2 fields alone reports 2, and one+ * naming any version 5 field reports 5.+ */+ private static final Map<String, Integer> FIELD_VERSIONS = Map.ofEntries(+ Map.entry("version", 2), Map.entry("account-id", 2), Map.entry("interface-id", 2),+ Map.entry("srcaddr", 2), Map.entry("dstaddr", 2), Map.entry("srcport", 2),+ Map.entry("dstport", 2), Map.entry("protocol", 2), Map.entry("packets", 2),+ Map.entry("bytes", 2), Map.entry("start", 2), Map.entry("end", 2),+ Map.entry("action", 2), Map.entry("log-status", 2),+ Map.entry("vpc-id", 3), Map.entry("subnet-id", 3), Map.entry("instance-id", 3),+ Map.entry("tcp-flags", 3), Map.entry("type", 3), Map.entry("pkt-srcaddr", 3),+ Map.entry("pkt-dstaddr", 3),+ Map.entry("region", 4), Map.entry("az-id", 4), Map.entry("sublocation-type", 4),+ Map.entry("sublocation-id", 4),+ Map.entry("pkt-src-aws-service", 5), Map.entry("pkt-dst-aws-service", 5),+ Map.entry("flow-direction", 5), Map.entry("traffic-path", 5));++ /**+ * One ${field-name} token of a LogFormat.+ *+ * <p>Deliberately not restricted to the fields below. A token this emulator does not recognise+ * still names a column the caller asked for, and matching it keeps that column in place.+ */+ private static final Pattern FIELD_TOKEN = Pattern.compile("\\$\\{([^}\\s]+)\\}");++ /**+ * The fields a flow log created without a LogFormat delivers, read from the same constant the+ * describe reports rather than repeated, so the two cannot drift apart.*/- static final String DEFAULT_HEADER =- "account-id action az-id bytes dstaddr dstport end flow-direction instance-id "- + "interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service "- + "pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type "- + "subnet-id tcp-flags traffic-path type version vpc-id";+ static final List<String> DEFAULT_FIELDS = tokensOf(DEFAULT_LOG_FORMAT);private static final DateTimeFormatter PATH_FMT =DateTimeFormatter.ofPattern("yyyy/MM/dd/HH").withZone(ZoneOffset.UTC);@@ -222,9 +247,9 @@ public class FlowLogService {long endEpoch = now.getEpochSecond();long startEpoch = endEpoch - fl.getMaxAggregationInterval();+ List<String> fields = fieldsOf(fl);StringBuilder sb = new StringBuilder();- sb.append(fl.getLogFormat() != null && !fl.getLogFormat().isBlank()- ? customHeader(fl.getLogFormat()) : DEFAULT_HEADER).append('\n');+ sb.append(String.join(" ", fields)).append('\n');// Collect the private IPs of all sibling ENIs in this flow log's resource// so instances in the same VPC actually talk to EACH OTHER (not just to@@ -278,10 +303,10 @@ public class FlowLogService {int srcPortEph = 32768 + ThreadLocalRandom.current().nextInt(28000);String svc = endpointService.get(peer); // non-null only for endpoint peers// egress: eni -> peer- sb.append(record(fl, eni, eni.privateIp, peer, srcPortEph, dstPort,+ sb.append(record(fl, fields, eni, eni.privateIp, peer, srcPortEph, dstPort,"egress", startEpoch, endEpoch, svc)).append('\n');// ingress: peer -> eni (response)- sb.append(record(fl, eni, peer, eni.privateIp, dstPort, srcPortEph,+ sb.append(record(fl, fields, eni, peer, eni.privateIp, dstPort, srcPortEph,"ingress", startEpoch, endEpoch, svc)).append('\n');recordCount += 2;}@@ -309,8 +334,52 @@ public class FlowLogService {* pkt-src-aws-service (ingress) so the record looks like real* AWS-service traffic.*/- private String record(FlowLog fl, Eni eni, String src, String dst, int srcPort, int dstPort,- String direction, long start, long end, String svc) {+ /**+ * The fields this flow log delivers, taken from its LogFormat.+ *+ * <p>A token naming a field this emulator has no value for is kept rather than dropped, so the+ * record still has one column per token the caller asked for, carrying "-" as AWS does for a+ * field that does not apply.+ */+ static List<String> fieldsOf(FlowLog fl) {+ String format = fl.getLogFormat();+ if (format == null || format.isBlank()) {+ return DEFAULT_FIELDS;+ }+ List<String> fields = tokensOf(format);+ return fields.isEmpty() ? DEFAULT_FIELDS : fields;+ }++ /** The field names a LogFormat string names, in the order it names them. */+ private static List<String> tokensOf(String format) {+ List<String> fields = new ArrayList<>();+ Matcher matcher = FIELD_TOKEN.matcher(format);+ while (matcher.find()) {+ fields.add(matcher.group(1));+ }+ return List.copyOf(fields);+ }++ /**+ * The version AWS numbers a format by, which is the highest any of its fields belongs to.+ *+ * <p>{@link #FIELD_VERSIONS} is complete through version 5, so a field missing from it belongs+ * to a later version, and one this emulator has no value for either. Such a field counts as+ * the highest version the table does define rather than the lowest, which would report 2 for a+ * record that plainly is not a version 2 one.+ */+ private static String formatVersion(List<String> fields) {+ int highestKnown = FIELD_VERSIONS.values().stream().mapToInt(Integer::intValue).max().orElse(2);+ int version = 2;+ for (String field : fields) {+ version = Math.max(version, FIELD_VERSIONS.getOrDefault(field, highestKnown));+ }+ return String.valueOf(version);+ }++ private String record(FlowLog fl, List<String> fields, Eni eni, String src, String dst,+ int srcPort, int dstPort, String direction, long start, long end,+ String svc) {int bytes = 200 + ThreadLocalRandom.current().nextInt(40000);int packets = 1 + ThreadLocalRandom.current().nextInt(60);int protocol = 6; // TCP@@ -322,37 +391,45 @@ public class FlowLogService {// egress (eni->endpoint) => dst is the service; ingress (endpoint->eni) => src.String pktDstSvc = (svc != null && egress) ? svc : "-";String pktSrcSvc = (svc != null && !egress) ? svc : "-";- // Field order MUST match DEFAULT_HEADER exactly.- return String.join(" ",- fl.getAccountId(), // account-id- action, // action- azId, // az-id- String.valueOf(bytes), // bytes- dst, // dstaddr- String.valueOf(dstPort), // dstport- String.valueOf(end), // end- direction, // flow-direction- nz(eni.instanceId), // instance-id- eni.eniId, // interface-id- "OK", // log-status- String.valueOf(packets), // packets- pktDstSvc, // pkt-dst-aws-service- dst, // pkt-dstaddr- pktSrcSvc, // pkt-src-aws-service- src, // pkt-srcaddr- String.valueOf(protocol), // protocol- fl.getRegion(), // region- src, // srcaddr- String.valueOf(srcPort), // srcport- String.valueOf(start), // start- "-", // sublocation-id- "-", // sublocation-type- nz(eni.subnetId), // subnet-id- String.valueOf(tcpFlags), // tcp-flags- "egress".equals(direction) ? "1" : "-", // traffic-path- "IPv4", // type- "5", // version- nz(eni.vpcId)); // vpc-id+ Map<String, String> values = Map.ofEntries(+ Map.entry("version", formatVersion(fields)),+ Map.entry("account-id", nz(fl.getAccountId())),+ Map.entry("interface-id", nz(eni.eniId)),+ Map.entry("srcaddr", nz(src)),+ Map.entry("dstaddr", nz(dst)),+ Map.entry("srcport", String.valueOf(srcPort)),+ Map.entry("dstport", String.valueOf(dstPort)),+ Map.entry("protocol", String.valueOf(protocol)),+ Map.entry("packets", String.valueOf(packets)),+ Map.entry("bytes", String.valueOf(bytes)),+ Map.entry("start", String.valueOf(start)),+ Map.entry("end", String.valueOf(end)),+ Map.entry("action", action),+ Map.entry("log-status", "OK"),+ Map.entry("vpc-id", nz(eni.vpcId)),+ Map.entry("subnet-id", nz(eni.subnetId)),+ Map.entry("instance-id", nz(eni.instanceId)),+ Map.entry("tcp-flags", String.valueOf(tcpFlags)),+ Map.entry("type", "IPv4"),+ Map.entry("pkt-srcaddr", nz(src)),+ Map.entry("pkt-dstaddr", nz(dst)),+ Map.entry("region", nz(fl.getRegion())),+ Map.entry("az-id", nz(azId)),+ Map.entry("sublocation-type", "-"),+ Map.entry("sublocation-id", "-"),+ Map.entry("pkt-src-aws-service", pktSrcSvc),+ Map.entry("pkt-dst-aws-service", pktDstSvc),+ Map.entry("flow-direction", direction),+ Map.entry("traffic-path", egress ? "1" : "-"));++ StringBuilder row = new StringBuilder();+ for (String field : fields) {+ if (row.length() > 0) {+ row.append(' ');+ }+ row.append(values.getOrDefault(field, "-"));+ }+ return row.toString();}// ─── Inventory correlation ─────────────────────────────────────────────────@@ -539,11 +616,6 @@ public class FlowLogService {return code + "-az" + n;}- private static String customHeader(String logFormat) {- // AWS custom format uses ${field-name} tokens; convert to a space-delimited header.- return logFormat.replace("${", "").replace("}", "").trim();- }-private static String nz(String s) {return (s == null || s.isEmpty()) ? "-" : s;}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.javaindex fd9543dc10..049de6d875 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java@@ -2,16 +2,29 @@ package io.github.hectorvent.floci.services.ec2;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.ec2.model.FlowLog;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.Placement;+import io.github.hectorvent.floci.services.ec2.model.Reservation;import io.github.hectorvent.floci.services.s3.S3Service;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;+import java.io.ByteArrayInputStream;+import java.nio.charset.StandardCharsets;import java.util.List;+import java.util.Map;+import java.util.zip.GZIPInputStream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.atLeastOnce;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class FlowLogServiceTest {@@ -74,4 +87,149 @@ class FlowLogServiceTest {void deleteFlowLogsIgnoresUnknownIds() {assertTrue(flowLogService.deleteFlowLogs("us-east-1", List.of("fl-doesnotexist")).isEmpty());}++ /**+ * The delivered records follow the flow log's LogFormat.+ *+ * <p>record used to emit all 29 available fields in alphabetical order whatever the format+ * said, so a custom format produced a short header line above long rows and a reader following+ * the header took the wrong value for every column.+ */+ @Test+ void aCustomFormatDeliversThoseFieldsInThatOrder() throws Exception {+ List<String> lines = deliver("${srcaddr} ${dstaddr} ${action}");++ assertEquals("srcaddr dstaddr action", lines.get(0));+ for (String row : records(lines)) {+ assertEquals(3, row.split(" ").length, "row does not match the header: " + row);+ assertTrue(row.endsWith(" ACCEPT"), "action is not the last field: " + row);+ }+ }++ /** A flow log with no LogFormat delivers the version 2 fields, and reports version 2. */+ @Test+ void noCustomFormatDeliversTheVersion2Default() throws Exception {+ List<String> lines = deliver(null);++ assertEquals(String.join(" ", FlowLogService.DEFAULT_FIELDS), lines.get(0));+ for (String row : records(lines)) {+ String[] columns = row.split(" ");+ assertEquals(FlowLogService.DEFAULT_FIELDS.size(), columns.length,+ "row does not match the header: " + row);+ assertEquals("2", columns[0], "version is not the version 2 default: " + row);+ }+ }++ /** The version a record reports is the highest its fields belong to, not a fixed 5. */+ @Test+ void aVersion5FieldRaisesTheReportedVersion() throws Exception {+ List<String> lines = deliver("${version} ${flow-direction}");++ assertEquals("version flow-direction", lines.get(0));+ for (String row : records(lines)) {+ assertEquals("5", row.split(" ")[0], "version did not follow the fields: " + row);+ }+ }++ /**+ * A field this emulator carries no value for keeps its column and reads "-".+ *+ * <p>Dropping it would shift every later value one place left against the header the caller+ * asked for, which is worse than saying the field does not apply.+ */+ @Test+ void anUnsupportedFieldKeepsItsColumnAsADash() throws Exception {+ List<String> lines = deliver("${srcaddr} ${ecs-cluster-arn} ${action}");++ assertEquals("srcaddr ecs-cluster-arn action", lines.get(0));+ for (String row : records(lines)) {+ assertEquals("-", row.split(" ")[1], "unsupported field is not a dash: " + row);+ }+ }++ /**+ * A token spelled in a way this emulator does not recognise still holds its column.+ *+ * <p>The token pattern once matched lowercase names only, so a format naming ${DSTADDR} lost+ * that column from both the header and every row, delivering a narrower file than the caller+ * asked for.+ */+ @Test+ void anUnrecognisedTokenSpellingKeepsItsColumn() throws Exception {+ List<String> lines = deliver("${srcaddr} ${DSTADDR} ${action}");++ assertEquals("srcaddr DSTADDR action", lines.get(0));+ for (String row : records(lines)) {+ assertEquals(3, row.split(" ").length, "row does not match the header: " + row);+ assertEquals("-", row.split(" ")[1], "unrecognised token is not a dash: " + row);+ }+ }++ /**+ * A field outside the version table counts as the highest version that table defines.+ *+ * <p>The table is complete through version 5, so an unlisted field belongs to a later one.+ * Counting it as version 2 would label a record that plainly is not a version 2 record.+ */+ @Test+ void anUnlistedFieldDoesNotReportVersionTwo() throws Exception {+ List<String> lines = deliver("${version} ${ecs-cluster-arn}");++ assertEquals("version ecs-cluster-arn", lines.get(0));+ for (String row : records(lines)) {+ assertEquals("5", row.split(" ")[0], "version fell back to 2: " + row);+ }+ }++ /** The record lines of a delivered file, which must not be empty. */+ private static List<String> records(List<String> lines) {+ assertFalse(lines.size() < 2, "delivery carried a header and no records");+ return lines.subList(1, lines.size());+ }++ /** Deliver one file for a flow log with this LogFormat and return its lines. */+ private List<String> deliver(String logFormat) throws Exception {+ Ec2Service ec2Service = mock(Ec2Service.class);+ when(ec2Service.callerAccountId()).thenReturn("000000000000");+ when(ec2Service.describeInstances(any(), any(), any()))+ .thenReturn(List.of(reservation()));+ when(ec2Service.endpointNetworkInterfaces(any())).thenReturn(List.of());+ S3Service s3Service = mock(S3Service.class);+ FlowLogService service =+ new FlowLogService(ec2Service, s3Service, new InMemoryStorage<>());++ FlowLog fl = service.createFlowLog("us-east-1", "vpc-123", "VPC", "ALL", "s3",+ "arn:aws:s3:::flow-bucket", null, logFormat, 600);+ service.generateAndDeliver(fl);++ // createFlowLog delivers once on creation, so take the file from the explicit call.+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service, atLeastOnce()).putObject(eq("flow-bucket"), anyString(), body.capture(),+ anyString(), any(Map.class));+ return List.of(gunzip(body.getValue()).split("\n"));+ }++ private static Reservation reservation() {+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(instance("i-aaa", "10.0.0.4"), instance("i-bbb", "10.0.0.5")));+ return reservation;+ }++ private static Instance instance(String instanceId, String privateIp) {+ Instance instance = new Instance();+ instance.setInstanceId(instanceId);+ instance.setPrivateIpAddress(privateIp);+ instance.setVpcId("vpc-123");+ instance.setSubnetId("subnet-123");+ Placement placement = new Placement();+ placement.setAvailabilityZone("us-east-1a");+ instance.setPlacement(placement);+ return instance;+ }++ private static String gunzip(byte[] gz) throws Exception {+ try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz))) {+ return new String(in.readAllBytes(), StandardCharsets.UTF_8).trim();+ }+ }}