Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ses): scan raw sends for the EICAR test file and keep it out of the mailbox store (#4156)

Kenshin Okinaka235ee91a76b4
diff --git a/docs/services/ses.md b/docs/services/ses.md
index e2bb5c9267..26ded81726 100644
--- a/docs/services/ses.md
+++ b/docs/services/ses.md
@@ -225,7 +225,9 @@ Messages are stored locally by Floci and can be persisted when SES storage is ba
Alongside the LocalStack fields, each captured message carries a
`ReturnPath` holding the resolved envelope sender described under
-[SMTP Relay](#smtp-relay).
+[SMTP Relay](#smtp-relay). A message the content scan rejected carries
+`RejectReason` and none of its content: no `Subject`, `Body`, `Headers`,
+`ReplyToAddresses` or `RawData`, in neither the Simple nor the raw shape.
## Examples
@@ -396,7 +398,7 @@ A successful send without a simulator-address recipient emits only the `Send` ev
A recipient on the [account-level suppression list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-suppression-list.html) produces a `Bounce` with `bounceSubType: OnAccountSuppressionList` or a `Complaint` with `complaintSubType: OnAccountSuppressionList`, following the stored reason, and never a `Delivery`. Events are split by cause: a message that reaches both `bounce@simulator` and a suppressed address publishes two `Bounce` events, each listing only its own recipients, with `mail.destination` carrying the full envelope on both. These shapes were verified against real SES on 2026-09-21.
-`Reject` is emitted the way AWS documents it: a message carrying the [EICAR test file](https://www.eicar.org/download-anti-malware-testfile/) in its subject, a header or a body is accepted (the send returns a `MessageId`) and then rejected with a `Reject` event whose `reason` is `Bad content`. The message is not relayed, the stored record keeps only its source, its envelope and `RejectReason`, and its `Send` and `Reject` events carry the envelope but no subject or headers in `mail`, where SES would include them, so the signature reaches neither the mailbox store nor an event destination. The test string itself is deliberately not reproduced here.
+`Reject` is emitted the way AWS documents it: a message carrying the [EICAR test file](https://www.eicar.org/download-anti-malware-testfile/) in its subject, a header, any text body or MIME part, including a base64 attachment or a forwarded message, is accepted (the send returns a `MessageId`) and then rejected with a `Reject` event whose `reason` is `Bad content`. The message is not relayed, the stored record keeps only its source, its envelope and `RejectReason`, and its `Send` and `Reject` events carry the envelope but no subject or headers in `mail`, where SES would include them, so the scanned content reaches neither the mailbox store nor an event destination. What the request itself supplied, its envelope and its tags, is kept. The test string itself is deliberately not reproduced here.
Account-level VDM (Virtual Deliverability Manager) attributes are stored per region. `PutAccountVdmAttributes` sets `VdmEnabled` (opt-in, defaults `DISABLED`) plus the optional `DashboardAttributes.EngagementMetrics` and `GuardianAttributes.OptimizedSharedDelivery`. `GetAccount` omits `VdmAttributes` until VDM has been configured for the region, then returns `VdmEnabled`, adding the `DashboardAttributes`/`GuardianAttributes` sub-objects only while `VdmEnabled` is `ENABLED`. Floci stores the settings but does not run VDM analytics.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesContentScan.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesContentScan.java
index f04c6f932c..a0f2214d00 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesContentScan.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesContentScan.java
@@ -1,6 +1,22 @@
package io.github.hectorvent.floci.services.ses;
+import org.apache.james.mime4j.codec.DecodeMonitor;
+import org.apache.james.mime4j.codec.DecoderUtil;
+import org.apache.james.mime4j.dom.Body;
+import org.apache.james.mime4j.dom.Entity;
+import org.apache.james.mime4j.dom.Header;
+import org.apache.james.mime4j.dom.Message;
+import org.apache.james.mime4j.dom.Multipart;
+import org.apache.james.mime4j.dom.SingleBody;
+import org.apache.james.mime4j.stream.Field;
+import org.jboss.logging.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
import java.util.Arrays;
+import java.util.Deque;
/**
* The content scan SES runs on every accepted message. Real SES scans for viruses and rejects the
@@ -9,10 +25,15 @@ import java.util.Arrays;
*
* <p>The EICAR string is a signature that endpoint protection quarantines on sight, so it is never
* written out verbatim: it is assembled from fragments at class initialisation, must never be
- * logged or persisted, and tests obtain it from {@link #signature()} rather than spelling it out.
+ * logged, and is exposed to tests only through {@link #signature()} so no test source carries it
+ * either. A message that trips the scan is likewise never persisted with its body.
*/
final class SesContentScan {
+ private static final Logger LOG = Logger.getLogger(SesContentScan.class);
+
+ // The fragments only mean something once joined, which keeps this source file below the
+ // detection threshold of the scanners the joined string is designed to trigger.
private static final String[] FRAGMENTS = {
"X5O!P%@AP[4\\PZX54(P^)7CC)7}$",
"EICAR-STANDARD-",
@@ -20,6 +41,10 @@ final class SesContentScan {
"FILE!$H+H*"
};
private static final String SIGNATURE_TEXT = String.join("", FRAGMENTS);
+ private static final byte[] SIGNATURE = SIGNATURE_TEXT.getBytes(StandardCharsets.US_ASCII);
+ // Decoded parts are read as a stream in chunks of this size with a rolling overlap, so the
+ // scan itself never needs a whole part in one array, whatever the parser buffers.
+ static final int CHUNK_SIZE = 64 * 1024;
private SesContentScan() {}
@@ -42,4 +67,125 @@ final class SesContentScan {
}
return false;
}
+
+ /** The outcome of scanning a raw message. */
+ enum Result {
+ CLEAN,
+ REJECTED,
+ /** The parser could not read the message, so only the wire bytes were checked. */
+ UNREADABLE
+ }
+
+ /** Scans a raw message from its wire bytes and the parse of them, {@code null} when it failed. */
+ static Result scan(byte[] mime, Message parsed) {
+ if (mime == null || mime.length == 0) {
+ return Result.CLEAN;
+ }
+ // The wire bytes are scanned first: a lenient MIME parse can swallow a headerless payload
+ // as malformed header lines, and a plain copy is found either way. An encoded copy the
+ // parser cannot reach is one no mail client would decode either, so it is not looked for.
+ if (indexOf(mime, SIGNATURE) >= 0) {
+ return Result.REJECTED;
+ }
+ if (parsed == null) {
+ return Result.UNREADABLE;
+ }
+ try {
+ return entityContains(parsed) ? Result.REJECTED : Result.CLEAN;
+ } catch (IOException | RuntimeException e) {
+ LOG.warnv("SES content scan could not walk the parsed message, only the wire bytes were checked: {0}",
+ e.getClass().getName());
+ return Result.UNREADABLE;
+ }
+ }
+
+ /** Scans a raw message, parsing it first; the parse is shared with the send path elsewhere. */
+ static Result scan(byte[] mime) {
+ if (mime == null || mime.length == 0) {
+ return Result.CLEAN;
+ }
+ return scan(mime, SmtpRelay.parseMime(mime));
+ }
+
+ // An explicit worklist rather than recursion: a message can nest forwarded messages and
+ // multiparts as deep as its size allows, and the stack must not be the limit.
+ private static boolean entityContains(Entity root) throws IOException {
+ Deque<Entity> pending = new ArrayDeque<>();
+ pending.push(root);
+ while (!pending.isEmpty()) {
+ Entity entity = pending.pop();
+ if (headerContains(entity.getHeader())) {
+ return true;
+ }
+ Body body = entity.getBody();
+ // A forwarded message (message/rfc822) is a Message body, neither multipart nor a leaf.
+ if (body instanceof Message embedded) {
+ pending.push(embedded);
+ } else if (body instanceof Multipart multipart) {
+ for (Entity part : multipart.getBodyParts()) {
+ pending.push(part);
+ }
+ } else if (body instanceof SingleBody single) {
+ try (InputStream in = single.getInputStream()) {
+ if (streamContains(in, SIGNATURE)) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // Header values are scanned decoded: an RFC 2047 encoded word puts the signature behind an
+ // encoding the wire scan cannot see through.
+ private static boolean headerContains(Header header) {
+ if (header == null) {
+ return false;
+ }
+ for (Field field : header.getFields()) {
+ String decoded = DecoderUtil.decodeEncodedWords(field.getBody(), DecodeMonitor.SILENT);
+ if (decoded != null && decoded.contains(SIGNATURE_TEXT)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Searches the whole stream for {@code needle} while holding at most one chunk plus the
+ * {@code needle.length - 1} bytes carried over from the previous chunk, so a match that
+ * straddles a chunk boundary is still found.
+ */
+ static boolean streamContains(InputStream in, byte[] needle) throws IOException {
+ int overlap = needle.length - 1;
+ byte[] window = new byte[overlap + CHUNK_SIZE];
+ int carried = 0;
+ int read;
+ while ((read = in.read(window, carried, CHUNK_SIZE)) > 0) {
+ int filled = carried + read;
+ if (indexOf(window, filled, needle) >= 0) {
+ return true;
+ }
+ carried = Math.min(overlap, filled);
+ System.arraycopy(window, filled - carried, window, 0, carried);
+ }
+ return false;
+ }
+
+ private static int indexOf(byte[] haystack, byte[] needle) {
+ return indexOf(haystack, haystack.length, needle);
+ }
+
+ private static int indexOf(byte[] haystack, int length, byte[] needle) {
+ outer:
+ for (int i = 0; i <= length - needle.length; i++) {
+ for (int j = 0; j < needle.length; j++) {
+ if (haystack[i + j] != needle[j]) {
+ continue outer;
+ }
+ }
+ return i;
+ }
+ return -1;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesInspectionController.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesInspectionController.java
index 439050202a..66160ed7fa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesInspectionController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesInspectionController.java
@@ -62,24 +62,16 @@ public class SesInspectionController {
node.put("ReturnPath", email.getReturnPath());
}
- if (email.isRaw()) {
+ if (email.getRejectReason() != null) {
+ // A record the content scan rejected holds nothing the scan looked at, so neither
+ // the raw nor the Simple content shape applies: only the envelope is shown.
+ putDestination(node, email);
+ } else if (email.isRaw()) {
// LocalStack returns RawData for raw emails, without
// Destination / Subject / Body fields.
node.put("RawData", email.getRawData());
} else {
- ObjectNode destination = node.putObject("Destination");
- if (email.getToAddresses() != null && !email.getToAddresses().isEmpty()) {
- ArrayNode toArr = destination.putArray("ToAddresses");
- email.getToAddresses().forEach(toArr::add);
- }
- if (email.getCcAddresses() != null && !email.getCcAddresses().isEmpty()) {
- ArrayNode ccArr = destination.putArray("CcAddresses");
- email.getCcAddresses().forEach(ccArr::add);
- }
- if (email.getBccAddresses() != null && !email.getBccAddresses().isEmpty()) {
- ArrayNode bccArr = destination.putArray("BccAddresses");
- email.getBccAddresses().forEach(bccArr::add);
- }
+ putDestination(node, email);
if (email.getReplyToAddresses() != null && !email.getReplyToAddresses().isEmpty()) {
ArrayNode replyTo = node.putArray("ReplyToAddresses");
@@ -122,6 +114,22 @@ public class SesInspectionController {
return Response.ok(result).build();
}
+ private static void putDestination(ObjectNode node, SentEmail email) {
+ ObjectNode destination = node.putObject("Destination");
+ if (email.getToAddresses() != null && !email.getToAddresses().isEmpty()) {
+ ArrayNode toArr = destination.putArray("ToAddresses");
+ email.getToAddresses().forEach(toArr::add);
+ }
+ if (email.getCcAddresses() != null && !email.getCcAddresses().isEmpty()) {
+ ArrayNode ccArr = destination.putArray("CcAddresses");
+ email.getCcAddresses().forEach(ccArr::add);
+ }
+ if (email.getBccAddresses() != null && !email.getBccAddresses().isEmpty()) {
+ ArrayNode bccArr = destination.putArray("BccAddresses");
+ email.getBccAddresses().forEach(bccArr::add);
+ }
+ }
+
@DELETE
public Response clearEmails() {
sentEmailService.clear();
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesService.java
index a8f1a56584..8a33294ab8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesService.java
@@ -316,7 +316,8 @@ public class SesService {
// The MIME headers are always parsed: X-SES-CONFIGURATION-SET can name the configuration
// set that decides whether events are published at all, so the configuration set cannot be
// resolved before the message has been read.
- SmtpRelay.RawMessageHeaders headers = SmtpRelay.parseRawHeaders(rawMessage);
+ SmtpRelay.ParsedRawMessage parsed = SmtpRelay.parseRawMessage(rawMessage);
+ SmtpRelay.RawMessageHeaders headers = parsed.headers();
// AWS accepts the configuration set either as a request field or as the
// X-SES-CONFIGURATION-SET header on the message itself; the request field wins.
String requestedConfigSet = firstNonBlank(configurationSetName, headers.configurationSet());
@@ -364,27 +365,48 @@ public class SesService {
.forEach(suppressedReasons::putIfAbsent);
String messageId = UUID.randomUUID().toString();
+ // The scan walks the decoded MIME, so a base64 attachment does not hide the content, and a
+ // message the parser cannot read is refused outright rather than stored and relayed unseen.
+ SesContentScan.Result scan = SesContentScan.scan(parsed.bytes(), parsed.message());
+ if (scan == SesContentScan.Result.UNREADABLE) {
+ throw new AwsException("InvalidParameterValue", "Raw message could not be parsed.", 400);
+ }
+ boolean rejected = scan == SesContentScan.Result.REJECTED;
// AWS routes bounces to the Return-Path carried by the message, falling back to the
- // request's return path and then the sender.
- String effectiveReturnPath = firstNonBlank(headers.returnPath(), returnPath, effectiveSource);
+ // request's return path and then the sender. A rejected message keeps nothing read off its
+ // own headers, and that header's value reaches here unparsed, so it is dropped; an envelope
+ // read off the headers is a list of parsed addresses and carries no message text.
+ String effectiveReturnPath = rejected
+ ? firstNonBlank(returnPath, effectiveSource)
+ : firstNonBlank(headers.returnPath(), returnPath, effectiveSource);
SentEmail email = new SentEmail(messageId, region, effectiveSource, effectiveDestinations, rawMessage);
email.setReturnPath(effectiveReturnPath);
+ if (rejected) {
+ email.discardContent(SesRecipientEvents.CONTENT_REJECT_REASON);
+ }
sentEmailService.record(region, messageId, email);
List<String> relayedDestinations = filterUnsuppressed(effectiveDestinations, suppressedReasons);
- if (!relayedDestinations.isEmpty()) {
+ if (!rejected && !relayedDestinations.isEmpty()) {
smtpRelay.relayRaw(new SmtpRelay.RawRelayMessage(effectiveSource, effectiveReturnPath,
relayedDestinations, rawMessage, messageId));
} else {
- LOG.infov("SES raw email accepted but not relayed (all recipients suppressed): messageId={0}",
- messageId);
+ LOG.infov("SES raw email accepted but not relayed ({0}): messageId={1}",
+ rejected ? "content rejected" : "all recipients suppressed", messageId);
}
- LOG.infov("SES raw email sent: from={0}, messageId={1}", effectiveSource, messageId);
+ if (!rejected) {
+ LOG.infov("SES raw email sent: from={0}, messageId={1}", effectiveSource, messageId);
+ }
+ // The recipient lists are read off the message, so a rejected one publishes none of them
+ // either; mail.destination still carries the envelope.
+ List<String> publishedTo = rejected ? List.of() : headers.to();
+ List<String> publishedCc = rejected ? List.of() : headers.cc();
+ List<String> publishedBcc = rejected ? List.of() : headers.bcc();
publishSendEvents(effectiveConfigSet, messageId, effectiveSource,
- headers.subject(), headers.to(), headers.cc(), headers.bcc(),
+ headers.subject(), publishedTo, publishedCc, publishedBcc,
effectiveDestinations,
- suppressedReasons, false, effectiveTags, List.of(), region);
+ suppressedReasons, rejected, rejected ? requestTags(emailTags) : effectiveTags, List.of(), region);
return messageId;
}
@@ -1483,6 +1505,12 @@ public class SesService {
return texts;
}
+ // Tags read off the raw message's own header are scanned content; only the request's tags,
+ // which the controller validated, are published for a rejected message.
+ private static List<MessageTag> requestTags(List<MessageTag> emailTags) {
+ return emailTags == null ? List.of() : emailTags;
+ }
+
private static int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SmtpRelay.java b/src/main/java/io/github/hectorvent/floci/services/ses/SmtpRelay.java
index 9826ab19a1..2e5d3b4e5f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SmtpRelay.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SmtpRelay.java
@@ -606,13 +606,39 @@ public class SmtpRelay {
* or unparseable.
*/
public static RawMessageHeaders parseRawHeaders(String rawMessage) {
+ return parseRawMessage(rawMessage).headers();
+ }
+
+ /**
+ * The decoded bytes of a raw message, the parsed message ({@code null} when the parser cannot
+ * read it) and the headers read off it, from one parse that the send path shares between
+ * header extraction and the content scan.
+ */
+ public record ParsedRawMessage(byte[] bytes, Message message, RawMessageHeaders headers) {}
+
+ public static ParsedRawMessage parseRawMessage(String rawMessage) {
if (rawMessage == null || rawMessage.isBlank()) {
- return RawMessageHeaders.empty();
+ return new ParsedRawMessage(new byte[0], null, RawMessageHeaders.empty());
+ }
+ byte[] mimeBytes = tryBase64Decode(rawMessage);
+ Message message = parseMime(mimeBytes);
+ return new ParsedRawMessage(mimeBytes, message,
+ message == null ? RawMessageHeaders.empty() : headersOf(message));
+ }
+
+ /** Parses MIME bytes, or returns {@code null} for anything the parser cannot read. */
+ static Message parseMime(byte[] mimeBytes) {
+ try {
+ return new DefaultMessageBuilder().parseMessage(new ByteArrayInputStream(mimeBytes));
+ } catch (Exception | StackOverflowError ignored) {
+ // The caller treats an unparseable message as such; there is nothing to diagnose here,
+ // and mime4j nests a reader per forwarded message, so a hostile depth overflows too.
+ return null;
}
+ }
+
+ private static RawMessageHeaders headersOf(Message message) {
try {
- byte[] mimeBytes = tryBase64Decode(rawMessage);
- var builder = new DefaultMessageBuilder();
- var message = builder.parseMessage(new ByteArrayInputStream(mimeBytes));
String subject = message.getSubject() != null ? message.getSubject() : "";
List<String> from = message.getFrom() != null
? toMailboxAddresses(message.getFrom()) : List.of();
@@ -630,7 +656,8 @@ public class SmtpRelay {
returnPath == null ? "" : returnPath,
configurationSet == null ? "" : configurationSet,
messageTags);
- } catch (Exception e) {
+ } catch (Exception ignored) {
+ // A message whose addresses cannot be read is treated like one without headers.
return RawMessageHeaders.empty();
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesContentScanTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesContentScanTest.java
index f6f15d6984..b1aa96d9b5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesContentScanTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesContentScanTest.java
@@ -1,7 +1,14 @@
package io.github.hectorvent.floci.services.ses;
+import io.github.hectorvent.floci.services.ses.SesContentScan.Result;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -28,4 +35,199 @@ class SesContentScanTest {
void signatureInsideATextBodyIsFlagged() {
assertTrue(SesContentScan.containsTestVirus(null, "prefix " + SIGNATURE + " suffix"));
}
+
+ @Test
+ void signatureInsideAPlainMimeBodyIsFlagged() {
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n\r\n" + SIGNATURE + "\r\n";
+
+ assertEquals(Result.REJECTED, SesContentScan.scan(mime.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ @Test
+ void signatureInsideAQuotedPrintableAttachmentIsFoundOnlyByDecoding() {
+ // Quoted-printable with escaped characters: neither the plain signature nor its base64
+ // spelling is on the wire, so only the decoded part walk can find it.
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: text/plain\r\n\r\nsee attachment\r\n"
+ + "--b\r\nContent-Type: application/octet-stream; name=\"eicar.com\"\r\n"
+ + "Content-Transfer-Encoding: quoted-printable\r\n"
+ + "Content-Disposition: attachment; filename=\"eicar.com\"\r\n\r\n"
+ + quotedPrintable(SIGNATURE) + "\r\n--b--\r\n";
+ byte[] bytes = mime.getBytes(StandardCharsets.UTF_8);
+
+ assertFalse(new String(bytes, StandardCharsets.ISO_8859_1).contains(SIGNATURE),
+ "the fixture must only carry the signature in encoded form");
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void signatureStraddlingAChunkBoundaryInsideALargeAttachmentIsFlagged() {
+ // The decoded part is longer than one scan chunk and the signature starts 30 bytes
+ // before the boundary, so only the carried-over overlap can complete the match. Quoted-
+ // printable keeps the signature off the wire, so the match has to come from the stream.
+ String decoded = "a".repeat(SesContentScan.CHUNK_SIZE - 30) + SIGNATURE + "a".repeat(4096);
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: application/octet-stream\r\n"
+ + "Content-Transfer-Encoding: quoted-printable\r\n\r\n"
+ + quotedPrintable(decoded) + "\r\n--b--\r\n";
+ byte[] bytes = mime.getBytes(StandardCharsets.UTF_8);
+
+ assertFalse(new String(bytes, StandardCharsets.ISO_8859_1).contains(SIGNATURE),
+ "the fixture must only carry the signature in encoded form");
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void streamScanFindsASignatureStraddlingTheChunkBoundary() throws IOException {
+ // A ByteArrayInputStream hands back exactly the requested chunk, so the signature that
+ // starts 30 bytes before CHUNK_SIZE is split across two reads and only the carried-over
+ // overlap can complete the match. The MIME test above cannot pin this down because the
+ // decoder's own read sizes do not line up with the scan chunk.
+ byte[] signature = SIGNATURE.getBytes(StandardCharsets.US_ASCII);
+ int prefix = SesContentScan.CHUNK_SIZE - 30;
+ byte[] data = new byte[prefix + signature.length + 10];
+ Arrays.fill(data, (byte) 'a');
+ System.arraycopy(signature, 0, data, prefix, signature.length);
+
+ assertTrue(SesContentScan.streamContains(new ByteArrayInputStream(data), signature));
+ assertFalse(SesContentScan.streamContains(new ByteArrayInputStream(new byte[prefix + 200]), signature));
+ }
+
+ @Test
+ void signatureInsideAForwardedMessageIsFlagged() {
+ // message/rfc822 parts surface as an embedded Message body, not a leaf; the signature sits
+ // in a quoted-printable attachment of the forwarded message, two levels down, so only the
+ // walk into the embedded message can find it.
+ String inner = "From: [email protected]\r\nTo: [email protected]\r\nSubject: fwd\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"i\"\r\n\r\n"
+ + "--i\r\nContent-Type: text/plain\r\n\r\nforwarded\r\n"
+ + "--i\r\nContent-Type: application/octet-stream\r\n"
+ + "Content-Transfer-Encoding: quoted-printable\r\n\r\n"
+ + quotedPrintable(SIGNATURE) + "\r\n--i--\r\n";
+ String outer = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"o\"\r\n\r\n"
+ + "--o\r\nContent-Type: text/plain\r\n\r\nsee forwarded message\r\n"
+ + "--o\r\nContent-Type: message/rfc822\r\n\r\n" + inner + "\r\n--o--\r\n";
+ byte[] bytes = outer.getBytes(StandardCharsets.UTF_8);
+
+ assertFalse(new String(bytes, StandardCharsets.ISO_8859_1).contains(SIGNATURE),
+ "the fixture must only carry the signature in encoded form");
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void signatureInsideAnEncodedWordSubjectIsFoundOnlyByDecodingTheHeader() {
+ // An RFC 2047 encoded word carries at most 75 characters, so the signature spans two words
+ // and each restarts the base64 alignment: neither the plain nor the base64 wire scan sees it.
+ byte[] signature = SIGNATURE.getBytes(StandardCharsets.US_ASCII);
+ String first = Base64.getEncoder().encodeToString(Arrays.copyOfRange(signature, 0, 45));
+ String second = Base64.getEncoder().encodeToString(Arrays.copyOfRange(signature, 45, signature.length));
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\n"
+ + "Subject: =?UTF-8?B?" + first + "?=\r\n =?UTF-8?B?" + second + "?=\r\n\r\nclean\r\n";
+ byte[] bytes = mime.getBytes(StandardCharsets.UTF_8);
+
+ assertFalse(new String(bytes, StandardCharsets.ISO_8859_1).contains(SIGNATURE),
+ "the fixture must only carry the signature in encoded form");
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void base64SpellingNothingWouldDecodeIsNotFlagged() {
+ // Neither a header value nor a text body without a transfer encoding is decoded by a mail
+ // client, so the base64 spelling of the signature in either is plain text, not the file.
+ String encoded = Base64.getEncoder().encodeToString(SIGNATURE.getBytes(StandardCharsets.US_ASCII));
+ String mime = "From: [email protected]\r\nX-Probe: " + encoded + "\r\n\r\n" + encoded + "\r\n";
+
+ assertEquals(Result.CLEAN, SesContentScan.scan(mime.getBytes(StandardCharsets.US_ASCII)));
+ }
+
+ @Test
+ void signatureInsideABase64AttachmentIsFoundByDecodingThePart() {
+ String encoded = Base64.getMimeEncoder().encodeToString(SIGNATURE.getBytes(StandardCharsets.US_ASCII));
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: base64\r\n\r\n"
+ + encoded + "\r\n--b--\r\n";
+
+ assertEquals(Result.REJECTED, SesContentScan.scan(mime.getBytes(StandardCharsets.US_ASCII)));
+ }
+
+ @Test
+ void thousandsOfForwardingLayersAreWalked() {
+ byte[] bytes = nestedForwards(3_000);
+
+ assertFalse(new String(bytes, StandardCharsets.ISO_8859_1).contains(SIGNATURE));
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void nestingBeyondTheParserIsRefusedNotCrashed() {
+ // mime4j nests a reader per forwarded message and overflows the stack at this depth; the
+ // scan must say so rather than throw an Error or call the message clean.
+ assertEquals(Result.UNREADABLE, SesContentScan.scan(nestedForwards(100_000)));
+ }
+
+ // message/rfc822 layers around a quoted-printable copy of the signature, so nothing on the
+ // wire matches and only a walk to the innermost part can find it.
+ private static byte[] nestedForwards(int depth) {
+ StringBuilder mime = new StringBuilder();
+ for (int i = 0; i < depth; i++) {
+ mime.append("Content-Type: message/rfc822\r\n\r\n");
+ }
+ mime.append("Content-Type: text/plain\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n")
+ .append(quotedPrintable(SIGNATURE)).append("\r\n");
+ return mime.toString().getBytes(StandardCharsets.US_ASCII);
+ }
+
+ @Test
+ void base64OfCleanContentIsNotFlagged() {
+ byte[] blob = Base64.getMimeEncoder().encode("nothing to see here, just a long enough clean payload".getBytes(StandardCharsets.US_ASCII));
+
+ assertEquals(Result.CLEAN, SesContentScan.scan(blob));
+ }
+
+ @Test
+ void cleanMimeWithAnAttachmentIsNotFlagged() {
+ String encoded = Base64.getEncoder().encodeToString("just a file".getBytes(StandardCharsets.US_ASCII));
+ String mime = "From: [email protected]\r\nTo: [email protected]\r\nSubject: x\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: text/plain\r\n\r\nhello\r\n"
+ + "--b\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: base64\r\n\r\n"
+ + encoded + "\r\n--b--\r\n";
+
+ assertEquals(Result.CLEAN, SesContentScan.scan(mime.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ @Test
+ void bytesThatAreNotMimeFallBackToARawScan() {
+ byte[] signature = SIGNATURE.getBytes(StandardCharsets.US_ASCII);
+ byte[] bytes = new byte[signature.length + 2];
+ bytes[0] = 0;
+ bytes[1] = 1;
+ System.arraycopy(signature, 0, bytes, 2, signature.length);
+
+ assertEquals(Result.REJECTED, SesContentScan.scan(bytes));
+ }
+
+ @Test
+ void emptyInputIsNotFlagged() {
+ assertEquals(Result.CLEAN, SesContentScan.scan(new byte[0]));
+ assertEquals(Result.CLEAN, SesContentScan.scan((byte[]) null));
+ }
+
+ // Escapes the characters of the signature that quoted-printable may encode, and soft-wraps,
+ // so the wire carries no plain copy of the signature.
+ private static String quotedPrintable(String decoded) {
+ StringBuilder out = new StringBuilder();
+ for (int i = 0; i < decoded.length(); i += 60) {
+ if (i > 0) {
+ out.append("=\r\n");
+ }
+ out.append(decoded.substring(i, Math.min(decoded.length(), i + 60))
+ .replace("!", "=21").replace("$", "=24"));
+ }
+ return out.toString();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
index 3bbbceff44..809726e689 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
@@ -1650,9 +1650,90 @@ class SesEventPublishingV2IntegrationTest {
}
}
+ @Test
+ @Order(34)
+ void eicarAttachment_isAcceptedThenRejected() throws Exception {
+ // The signature comes from the scanner itself so it never appears in test source.
+ String encoded = Base64.getEncoder().encodeToString(
+ SesContentScan.signature().getBytes(StandardCharsets.US_ASCII));
+ String mime = "From: " + SENDER + "\r\nTo: [email protected]\r\nSubject: scan\r\n"
+ + "X-SES-MESSAGE-TAGS: probe=leak\r\n"
+ + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: text/plain\r\n\r\nsee attachment\r\n"
+ + "--b\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: base64\r\n\r\n"
+ + encoded + "\r\n--b--\r\n";
+ String rawB64 = Base64.getEncoder().encodeToString(
+ mime.getBytes(StandardCharsets.UTF_8));
+ drainQueue();
+ String messageId = given().contentType("application/json").header("Authorization", SES_AUTH)
+ .body("""
+ {"FromEmailAddress": "%s",
+ "Destination": {"ToAddresses": ["[email protected]"]},
+ "ConfigurationSetName": "%s",
+ "Content": {"Raw": {"Data": "%s"}}}
+ """.formatted(SENDER, CS, rawB64))
+ .when().post("/v2/email/outbound-emails").then().statusCode(200)
+ .extract().jsonPath().getString("MessageId");
+
+ List<JsonNode> events = receiveSesEvents(2);
+ assertEquals(2, events.size(), "expected Send and Reject, and no Delivery");
+ assertTrue(events.stream().anyMatch(e -> "Send".equals(e.path("eventType").asText())));
+ JsonNode reject = events.stream()
+ .filter(e -> "Reject".equals(e.path("eventType").asText()))
+ .findFirst().orElseThrow();
+ assertEquals("Bad content", reject.path("reject").path("reason").asText());
+ // Nothing read off the message is published: not the subject the parser found, not the tags
+ // from its X-SES-MESSAGE-TAGS header, not the recipient lists from its To and Cc headers.
+ // Only the From the request supplied stays, and mail.destination carries the envelope.
+ for (JsonNode event : events) {
+ JsonNode mail = event.path("mail");
+ String type = event.path("eventType").asText();
+ assertTrue(mail.path("commonHeaders").path("subject").isMissingNode(),
+ type + " must not carry the subject read off the message");
+ assertTrue(mail.path("tags").path("probe").isMissingNode(),
+ type + " must not carry the raw header tag");
+ assertEquals(0, mail.path("commonHeaders").path("to").size(),
+ type + " must not carry the recipients read off the message");
+ for (JsonNode header : mail.path("headers")) {
+ assertEquals("From", header.path("name").asText(),
+ type + " must not carry header " + header.path("name").asText());
+ }
+ assertEquals("[email protected]", mail.path("destination").path(0).asText());
+ }
+
+ // The request's own tags, validated by the controller, stay on both events.
+ drainQueue();
+ given().contentType("application/json").header("Authorization", SES_AUTH)
+ .body("""
+ {"FromEmailAddress": "%s",
+ "Destination": {"ToAddresses": ["[email protected]"]},
+ "ConfigurationSetName": "%s",
+ "EmailTags": [{"Name": "kept", "Value": "yes"}],
+ "Content": {"Raw": {"Data": "%s"}}}
+ """.formatted(SENDER, CS, rawB64))
+ .when().post("/v2/email/outbound-emails").then().statusCode(200);
+ List<JsonNode> tagged = receiveSesEvents(2);
+ assertEquals(2, tagged.size(), "expected Send and Reject");
+ for (JsonNode event : tagged) {
+ assertEquals("yes", event.path("mail").path("tags").path("kept").path(0).asText(),
+ event.path("eventType").asText() + " must keep the request tag");
+ assertTrue(event.path("mail").path("tags").path("probe").isMissingNode());
+ }
+
+ // The stored record keeps the marker and its envelope but no copy of the content, so it
+ // is listed in neither the raw shape (no RawData) nor the Simple one (no Subject or Body).
+ given().header("Authorization", SES_AUTH)
+ .when().get("/_aws/ses?id=" + messageId).then().statusCode(200)
+ .body("messages[0].RejectReason", equalTo("Bad content"))
+ .body("messages[0]", hasKey("Destination"))
+ .body("messages[0]", not(hasKey("RawData")))
+ .body("messages[0]", not(hasKey("Subject")))
+ .body("messages[0]", not(hasKey("Body")));
+ }
+
@Test
@Order(35)
- void eicarInSimpleBody_isAcceptedThenRejectedAndStoredWithoutItsContent() throws Exception {
+ void eicarInSimpleBody_isAcceptedThenRejectedAndStoredWithoutTheBody() throws Exception {
drainQueue();
String messageId = given().contentType("application/json").header("Authorization", SES_AUTH)
.body("""
@@ -1671,21 +1752,20 @@ class SesEventPublishingV2IntegrationTest {
assertTrue(events.stream().anyMatch(e -> "Reject".equals(e.path("eventType").asText())));
assertTrue(events.stream().noneMatch(e -> "Delivery".equals(e.path("eventType").asText())));
- // The record keeps the reason and the envelope, but none of the scanned content.
given().header("Authorization", SES_AUTH)
.when().get("/_aws/ses?id=" + messageId).then().statusCode(200)
.body("messages[0].RejectReason", equalTo("Bad content"))
- .body("messages[0].Subject", nullValue())
- .body("messages[0].Body.text_part", nullValue())
- .body("messages[0].Body.html_part", nullValue())
- .body("messages[0]", not(hasKey("Headers")));
+ .body("messages[0]", hasKey("Destination"))
+ .body("messages[0]", not(hasKey("Subject")))
+ .body("messages[0]", not(hasKey("Body")));
}
@Test
@Order(36)
void eicarInSimpleSubjectOrHeader_isRejectedLikeARawSend() throws Exception {
- // The signature can sit in the subject or in a caller-supplied header name or value as
- // well as in a body, so the simple path scans all of them, and the record keeps none.
+ // A raw send is scanned as a whole, header block included, so the simple path has to
+ // cover the subject and the caller's header names and values as well or the two paths
+ // disagree; and the stored record keeps none of them.
String signature = SesContentScan.signature().replace("\\", "\\\\");
for (String content : new String[] {
"\"Subject\": {\"Data\": \"" + signature + "\"}, \"Body\": {\"Text\": {\"Data\": \"clean\"}}",
@@ -1723,7 +1803,8 @@ class SesEventPublishingV2IntegrationTest {
given().header("Authorization", SES_AUTH)
.when().get("/_aws/ses?id=" + messageId).then().statusCode(200)
.body("messages[0].RejectReason", equalTo("Bad content"))
- .body("messages[0].Subject", nullValue())
+ .body("messages[0]", not(hasKey("Subject")))
+ .body("messages[0]", not(hasKey("Body")))
.body("messages[0]", not(hasKey("Headers")));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesServiceSmtpTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesServiceSmtpTest.java
index 60611e3a52..f4df69dcc0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesServiceSmtpTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesServiceSmtpTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.ses;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.ses.model.ConfigurationSet;
import io.github.hectorvent.floci.services.ses.model.SentEmail;
@@ -14,6 +15,7 @@ import org.mockito.ArgumentCaptor;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
@ExtendWith(MockitoExtension.class)
@@ -111,6 +113,35 @@ class SesServiceSmtpTest {
assertEquals("[email protected]", storedEmail(messageId).getReturnPath());
}
+ @Test
+ void sendRawEmail_rejectedMessage_keepsNoReturnPathFromItsHeaders() {
+ // The scan trips on the subject; the Return-Path header, read off the same message, must
+ // not survive on the record either, so the request's return path is used.
+ String raw = "From: [email protected]\r\nTo: [email protected]\r\nReturn-Path: <[email protected]>\r\n"
+ + "Subject: " + SesContentScan.signature() + "\r\n\r\nbody\r\n";
+
+ String messageId = service.sendRawEmail("[email protected]", List.of("[email protected]"), raw,
+ "[email protected]", null, List.of(), null, "us-east-1");
+
+ assertEquals("[email protected]", storedEmail(messageId).getReturnPath());
+ assertEquals("Bad content", storedEmail(messageId).getRejectReason());
+ verify(smtpRelay, never()).relayRaw(any());
+ }
+
+ @Test
+ void sendRawEmail_unreadableMessage_isRefusedNotStoredNotRelayed() {
+ // Nested past what mime4j can parse: the content cannot be scanned, so the send is refused
+ // rather than recorded and relayed unseen.
+ String raw = "Content-Type: message/rfc822\r\n\r\n".repeat(100_000) + "Subject: deep\r\n\r\nx\r\n";
+
+ AwsException e = assertThrows(AwsException.class, () -> service.sendRawEmail("[email protected]",
+ List.of("[email protected]"), raw, null, null, List.of(), null, "us-east-1"));
+
+ assertEquals(400, e.getHttpStatus());
+ assertTrue(sentEmails.listAll().isEmpty());
+ verify(smtpRelay, never()).relayRaw(any());
+ }
+
@Test
void sendRawEmail_callsRelayRaw() {
String messageId = service.sendRawEmail("[email protected]",
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SmtpRelayTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SmtpRelayTest.java
index a9ef9f03a3..470e9004e9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SmtpRelayTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SmtpRelayTest.java
@@ -389,6 +389,18 @@ class SmtpRelayTest {
assertEquals(List.of("[email protected]"), h.bcc());
}
+ @Test
+ void parseRawHeaders_nestedBeyondTheParser_returnsEmptyRecord() {
+ // mime4j nests a reader per forwarded message and overflows the stack at this depth; the
+ // header pre-parse must answer like any other unparseable message, not kill the request.
+ String raw = "Content-Type: message/rfc822\r\n\r\n".repeat(100_000) + "Subject: deep\r\n\r\nx\r\n";
+
+ SmtpRelay.ParsedRawMessage parsed = SmtpRelay.parseRawMessage(raw);
+ assertNull(parsed.message());
+ assertEquals("", parsed.headers().subject());
+ assertTrue(parsed.headers().to().isEmpty());
+ }
+
@Test
void parseRawHeaders_blank_returnsEmptyRecord() {
SmtpRelay.RawMessageHeaders h = SmtpRelay.parseRawHeaders(null);