Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): deliver flow log records in the format the flow log declares (#4429)

lex004b15ca89508c
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java b/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java
index e5c81f0976..2ac8f9122f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/FlowLogService.java
@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
/**
@@ -59,22 +61,45 @@ public class FlowLogService {
/**
* The format a flow log is created with when CreateFlowLogs omits LogFormat, which
* DescribeFlowLogs reports for it thereafter. Version 2 fields in the order AWS documents.
- *
- * <p>Distinct from {@link #DEFAULT_HEADER}, which is the header line of a generated log file.
*/
public static final String DEFAULT_LOG_FORMAT =
"${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} "
+ "${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}";
/**
- * Every available flow log field, alphabetised, written as the header line of a generated log
- * file. Not a version ordering, and not what {@link #DEFAULT_LOG_FORMAT} describes.
+ * Every field a flow log record can carry, against the format version that introduced it.
+ *
+ * <p>The version a record reports is the highest of the versions its selected fields belong
+ * to, which is how AWS numbers a format. A record of version 2 fields alone reports 2, and one
+ * naming any version 5 field reports 5.
+ */
+ private static final Map<String, Integer> FIELD_VERSIONS = Map.ofEntries(
+ Map.entry("version", 2), Map.entry("account-id", 2), Map.entry("interface-id", 2),
+ Map.entry("srcaddr", 2), Map.entry("dstaddr", 2), Map.entry("srcport", 2),
+ Map.entry("dstport", 2), Map.entry("protocol", 2), Map.entry("packets", 2),
+ Map.entry("bytes", 2), Map.entry("start", 2), Map.entry("end", 2),
+ Map.entry("action", 2), Map.entry("log-status", 2),
+ Map.entry("vpc-id", 3), Map.entry("subnet-id", 3), Map.entry("instance-id", 3),
+ Map.entry("tcp-flags", 3), Map.entry("type", 3), Map.entry("pkt-srcaddr", 3),
+ Map.entry("pkt-dstaddr", 3),
+ Map.entry("region", 4), Map.entry("az-id", 4), Map.entry("sublocation-type", 4),
+ Map.entry("sublocation-id", 4),
+ Map.entry("pkt-src-aws-service", 5), Map.entry("pkt-dst-aws-service", 5),
+ Map.entry("flow-direction", 5), Map.entry("traffic-path", 5));
+
+ /**
+ * One ${field-name} token of a LogFormat.
+ *
+ * <p>Deliberately not restricted to the fields below. A token this emulator does not recognise
+ * still names a column the caller asked for, and matching it keeps that column in place.
+ */
+ private static final Pattern FIELD_TOKEN = Pattern.compile("\\$\\{([^}\\s]+)\\}");
+
+ /**
+ * The fields a flow log created without a LogFormat delivers, read from the same constant the
+ * describe reports rather than repeated, so the two cannot drift apart.
*/
- static final String DEFAULT_HEADER =
- "account-id action az-id bytes dstaddr dstport end flow-direction instance-id "
- + "interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service "
- + "pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type "
- + "subnet-id tcp-flags traffic-path type version vpc-id";
+ static final List<String> DEFAULT_FIELDS = tokensOf(DEFAULT_LOG_FORMAT);
private static final DateTimeFormatter PATH_FMT =
DateTimeFormatter.ofPattern("yyyy/MM/dd/HH").withZone(ZoneOffset.UTC);
@@ -222,9 +247,9 @@ public class FlowLogService {
long endEpoch = now.getEpochSecond();
long startEpoch = endEpoch - fl.getMaxAggregationInterval();
+ List<String> fields = fieldsOf(fl);
StringBuilder sb = new StringBuilder();
- sb.append(fl.getLogFormat() != null && !fl.getLogFormat().isBlank()
- ? customHeader(fl.getLogFormat()) : DEFAULT_HEADER).append('\n');
+ sb.append(String.join(" ", fields)).append('\n');
// Collect the private IPs of all sibling ENIs in this flow log's resource
// so instances in the same VPC actually talk to EACH OTHER (not just to
@@ -278,10 +303,10 @@ public class FlowLogService {
int srcPortEph = 32768 + ThreadLocalRandom.current().nextInt(28000);
String svc = endpointService.get(peer); // non-null only for endpoint peers
// egress: eni -> peer
- sb.append(record(fl, eni, eni.privateIp, peer, srcPortEph, dstPort,
+ sb.append(record(fl, fields, eni, eni.privateIp, peer, srcPortEph, dstPort,
"egress", startEpoch, endEpoch, svc)).append('\n');
// ingress: peer -> eni (response)
- sb.append(record(fl, eni, peer, eni.privateIp, dstPort, srcPortEph,
+ sb.append(record(fl, fields, eni, peer, eni.privateIp, dstPort, srcPortEph,
"ingress", startEpoch, endEpoch, svc)).append('\n');
recordCount += 2;
}
@@ -309,8 +334,52 @@ public class FlowLogService {
* pkt-src-aws-service (ingress) so the record looks like real
* AWS-service traffic.
*/
- private String record(FlowLog fl, Eni eni, String src, String dst, int srcPort, int dstPort,
- String direction, long start, long end, String svc) {
+ /**
+ * The fields this flow log delivers, taken from its LogFormat.
+ *
+ * <p>A token naming a field this emulator has no value for is kept rather than dropped, so the
+ * record still has one column per token the caller asked for, carrying "-" as AWS does for a
+ * field that does not apply.
+ */
+ static List<String> fieldsOf(FlowLog fl) {
+ String format = fl.getLogFormat();
+ if (format == null || format.isBlank()) {
+ return DEFAULT_FIELDS;
+ }
+ List<String> fields = tokensOf(format);
+ return fields.isEmpty() ? DEFAULT_FIELDS : fields;
+ }
+
+ /** The field names a LogFormat string names, in the order it names them. */
+ private static List<String> tokensOf(String format) {
+ List<String> fields = new ArrayList<>();
+ Matcher matcher = FIELD_TOKEN.matcher(format);
+ while (matcher.find()) {
+ fields.add(matcher.group(1));
+ }
+ return List.copyOf(fields);
+ }
+
+ /**
+ * The version AWS numbers a format by, which is the highest any of its fields belongs to.
+ *
+ * <p>{@link #FIELD_VERSIONS} is complete through version 5, so a field missing from it belongs
+ * to a later version, and one this emulator has no value for either. Such a field counts as
+ * the highest version the table does define rather than the lowest, which would report 2 for a
+ * record that plainly is not a version 2 one.
+ */
+ private static String formatVersion(List<String> fields) {
+ int highestKnown = FIELD_VERSIONS.values().stream().mapToInt(Integer::intValue).max().orElse(2);
+ int version = 2;
+ for (String field : fields) {
+ version = Math.max(version, FIELD_VERSIONS.getOrDefault(field, highestKnown));
+ }
+ return String.valueOf(version);
+ }
+
+ private String record(FlowLog fl, List<String> fields, Eni eni, String src, String dst,
+ int srcPort, int dstPort, String direction, long start, long end,
+ String svc) {
int bytes = 200 + ThreadLocalRandom.current().nextInt(40000);
int packets = 1 + ThreadLocalRandom.current().nextInt(60);
int protocol = 6; // TCP
@@ -322,37 +391,45 @@ public class FlowLogService {
// egress (eni->endpoint) => dst is the service; ingress (endpoint->eni) => src.
String pktDstSvc = (svc != null && egress) ? svc : "-";
String pktSrcSvc = (svc != null && !egress) ? svc : "-";
- // Field order MUST match DEFAULT_HEADER exactly.
- return String.join(" ",
- fl.getAccountId(), // account-id
- action, // action
- azId, // az-id
- String.valueOf(bytes), // bytes
- dst, // dstaddr
- String.valueOf(dstPort), // dstport
- String.valueOf(end), // end
- direction, // flow-direction
- nz(eni.instanceId), // instance-id
- eni.eniId, // interface-id
- "OK", // log-status
- String.valueOf(packets), // packets
- pktDstSvc, // pkt-dst-aws-service
- dst, // pkt-dstaddr
- pktSrcSvc, // pkt-src-aws-service
- src, // pkt-srcaddr
- String.valueOf(protocol), // protocol
- fl.getRegion(), // region
- src, // srcaddr
- String.valueOf(srcPort), // srcport
- String.valueOf(start), // start
- "-", // sublocation-id
- "-", // sublocation-type
- nz(eni.subnetId), // subnet-id
- String.valueOf(tcpFlags), // tcp-flags
- "egress".equals(direction) ? "1" : "-", // traffic-path
- "IPv4", // type
- "5", // version
- nz(eni.vpcId)); // vpc-id
+ Map<String, String> values = Map.ofEntries(
+ Map.entry("version", formatVersion(fields)),
+ Map.entry("account-id", nz(fl.getAccountId())),
+ Map.entry("interface-id", nz(eni.eniId)),
+ Map.entry("srcaddr", nz(src)),
+ Map.entry("dstaddr", nz(dst)),
+ Map.entry("srcport", String.valueOf(srcPort)),
+ Map.entry("dstport", String.valueOf(dstPort)),
+ Map.entry("protocol", String.valueOf(protocol)),
+ Map.entry("packets", String.valueOf(packets)),
+ Map.entry("bytes", String.valueOf(bytes)),
+ Map.entry("start", String.valueOf(start)),
+ Map.entry("end", String.valueOf(end)),
+ Map.entry("action", action),
+ Map.entry("log-status", "OK"),
+ Map.entry("vpc-id", nz(eni.vpcId)),
+ Map.entry("subnet-id", nz(eni.subnetId)),
+ Map.entry("instance-id", nz(eni.instanceId)),
+ Map.entry("tcp-flags", String.valueOf(tcpFlags)),
+ Map.entry("type", "IPv4"),
+ Map.entry("pkt-srcaddr", nz(src)),
+ Map.entry("pkt-dstaddr", nz(dst)),
+ Map.entry("region", nz(fl.getRegion())),
+ Map.entry("az-id", nz(azId)),
+ Map.entry("sublocation-type", "-"),
+ Map.entry("sublocation-id", "-"),
+ Map.entry("pkt-src-aws-service", pktSrcSvc),
+ Map.entry("pkt-dst-aws-service", pktDstSvc),
+ Map.entry("flow-direction", direction),
+ Map.entry("traffic-path", egress ? "1" : "-"));
+
+ StringBuilder row = new StringBuilder();
+ for (String field : fields) {
+ if (row.length() > 0) {
+ row.append(' ');
+ }
+ row.append(values.getOrDefault(field, "-"));
+ }
+ return row.toString();
}
// ─── Inventory correlation ─────────────────────────────────────────────────
@@ -539,11 +616,6 @@ public class FlowLogService {
return code + "-az" + n;
}
- private static String customHeader(String logFormat) {
- // AWS custom format uses ${field-name} tokens; convert to a space-delimited header.
- return logFormat.replace("${", "").replace("}", "").trim();
- }
-
private static String nz(String s) {
return (s == null || s.isEmpty()) ? "-" : s;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java
index fd9543dc10..049de6d875 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/FlowLogServiceTest.java
@@ -2,16 +2,29 @@ package io.github.hectorvent.floci.services.ec2;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.ec2.model.FlowLog;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.Placement;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
import io.github.hectorvent.floci.services.s3.S3Service;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class FlowLogServiceTest {
@@ -74,4 +87,149 @@ class FlowLogServiceTest {
void deleteFlowLogsIgnoresUnknownIds() {
assertTrue(flowLogService.deleteFlowLogs("us-east-1", List.of("fl-doesnotexist")).isEmpty());
}
+
+ /**
+ * The delivered records follow the flow log's LogFormat.
+ *
+ * <p>record used to emit all 29 available fields in alphabetical order whatever the format
+ * said, so a custom format produced a short header line above long rows and a reader following
+ * the header took the wrong value for every column.
+ */
+ @Test
+ void aCustomFormatDeliversThoseFieldsInThatOrder() throws Exception {
+ List<String> lines = deliver("${srcaddr} ${dstaddr} ${action}");
+
+ assertEquals("srcaddr dstaddr action", lines.get(0));
+ for (String row : records(lines)) {
+ assertEquals(3, row.split(" ").length, "row does not match the header: " + row);
+ assertTrue(row.endsWith(" ACCEPT"), "action is not the last field: " + row);
+ }
+ }
+
+ /** A flow log with no LogFormat delivers the version 2 fields, and reports version 2. */
+ @Test
+ void noCustomFormatDeliversTheVersion2Default() throws Exception {
+ List<String> lines = deliver(null);
+
+ assertEquals(String.join(" ", FlowLogService.DEFAULT_FIELDS), lines.get(0));
+ for (String row : records(lines)) {
+ String[] columns = row.split(" ");
+ assertEquals(FlowLogService.DEFAULT_FIELDS.size(), columns.length,
+ "row does not match the header: " + row);
+ assertEquals("2", columns[0], "version is not the version 2 default: " + row);
+ }
+ }
+
+ /** The version a record reports is the highest its fields belong to, not a fixed 5. */
+ @Test
+ void aVersion5FieldRaisesTheReportedVersion() throws Exception {
+ List<String> lines = deliver("${version} ${flow-direction}");
+
+ assertEquals("version flow-direction", lines.get(0));
+ for (String row : records(lines)) {
+ assertEquals("5", row.split(" ")[0], "version did not follow the fields: " + row);
+ }
+ }
+
+ /**
+ * A field this emulator carries no value for keeps its column and reads "-".
+ *
+ * <p>Dropping it would shift every later value one place left against the header the caller
+ * asked for, which is worse than saying the field does not apply.
+ */
+ @Test
+ void anUnsupportedFieldKeepsItsColumnAsADash() throws Exception {
+ List<String> lines = deliver("${srcaddr} ${ecs-cluster-arn} ${action}");
+
+ assertEquals("srcaddr ecs-cluster-arn action", lines.get(0));
+ for (String row : records(lines)) {
+ assertEquals("-", row.split(" ")[1], "unsupported field is not a dash: " + row);
+ }
+ }
+
+ /**
+ * A token spelled in a way this emulator does not recognise still holds its column.
+ *
+ * <p>The token pattern once matched lowercase names only, so a format naming ${DSTADDR} lost
+ * that column from both the header and every row, delivering a narrower file than the caller
+ * asked for.
+ */
+ @Test
+ void anUnrecognisedTokenSpellingKeepsItsColumn() throws Exception {
+ List<String> lines = deliver("${srcaddr} ${DSTADDR} ${action}");
+
+ assertEquals("srcaddr DSTADDR action", lines.get(0));
+ for (String row : records(lines)) {
+ assertEquals(3, row.split(" ").length, "row does not match the header: " + row);
+ assertEquals("-", row.split(" ")[1], "unrecognised token is not a dash: " + row);
+ }
+ }
+
+ /**
+ * A field outside the version table counts as the highest version that table defines.
+ *
+ * <p>The table is complete through version 5, so an unlisted field belongs to a later one.
+ * Counting it as version 2 would label a record that plainly is not a version 2 record.
+ */
+ @Test
+ void anUnlistedFieldDoesNotReportVersionTwo() throws Exception {
+ List<String> lines = deliver("${version} ${ecs-cluster-arn}");
+
+ assertEquals("version ecs-cluster-arn", lines.get(0));
+ for (String row : records(lines)) {
+ assertEquals("5", row.split(" ")[0], "version fell back to 2: " + row);
+ }
+ }
+
+ /** The record lines of a delivered file, which must not be empty. */
+ private static List<String> records(List<String> lines) {
+ assertFalse(lines.size() < 2, "delivery carried a header and no records");
+ return lines.subList(1, lines.size());
+ }
+
+ /** Deliver one file for a flow log with this LogFormat and return its lines. */
+ private List<String> deliver(String logFormat) throws Exception {
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ when(ec2Service.callerAccountId()).thenReturn("000000000000");
+ when(ec2Service.describeInstances(any(), any(), any()))
+ .thenReturn(List.of(reservation()));
+ when(ec2Service.endpointNetworkInterfaces(any())).thenReturn(List.of());
+ S3Service s3Service = mock(S3Service.class);
+ FlowLogService service =
+ new FlowLogService(ec2Service, s3Service, new InMemoryStorage<>());
+
+ FlowLog fl = service.createFlowLog("us-east-1", "vpc-123", "VPC", "ALL", "s3",
+ "arn:aws:s3:::flow-bucket", null, logFormat, 600);
+ service.generateAndDeliver(fl);
+
+ // createFlowLog delivers once on creation, so take the file from the explicit call.
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, atLeastOnce()).putObject(eq("flow-bucket"), anyString(), body.capture(),
+ anyString(), any(Map.class));
+ return List.of(gunzip(body.getValue()).split("\n"));
+ }
+
+ private static Reservation reservation() {
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(instance("i-aaa", "10.0.0.4"), instance("i-bbb", "10.0.0.5")));
+ return reservation;
+ }
+
+ private static Instance instance(String instanceId, String privateIp) {
+ Instance instance = new Instance();
+ instance.setInstanceId(instanceId);
+ instance.setPrivateIpAddress(privateIp);
+ instance.setVpcId("vpc-123");
+ instance.setSubnetId("subnet-123");
+ Placement placement = new Placement();
+ placement.setAvailabilityZone("us-east-1a");
+ instance.setPlacement(placement);
+ return instance;
+ }
+
+ private static String gunzip(byte[] gz) throws Exception {
+ try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz))) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8).trim();
+ }
+ }
}