Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
//-----------------------------------------------------------------------
// <copyright file="DatabaseMessageStore.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.SmtpService.Handlers;
using System.Buffers;
using System.Globalization;
using System.IO.Compression;
using System.Net.Mail;
using System.Text.RegularExpressions;
using AliasServerDb;
using AliasVault.Cryptography;
using Microsoft.EntityFrameworkCore;
using MimeKit;
using MimeKit.IO;
using NUglify;
using SmtpServer;
using SmtpServer.Mail;
using SmtpServer.Protocol;
using SmtpServer.Storage;
/// <summary>
/// Database message store.
/// </summary>
/// <param name="logger">ILogger instance.</param>
/// <param name="config">Config instance.</param>
/// <param name="dbContextFactory">IDbContextFactory instance.</param>
public class DatabaseMessageStore(ILogger<DatabaseMessageStore> logger, Config config, IAliasServerDbContextFactory dbContextFactory) : MessageStore
{
/// <summary>
/// Maximum accepted raw message size in bytes: 10 MB of content plus headroom for MIME encoding overhead.
/// </summary>
public const int MaxEmailSizeInBytes = (int)(10 * 1024 * 1024 * 1.4);
/// <summary>
/// Attachment bodies smaller than this are left inline in the message source. Detaching them would trade a
/// download saving too small to notice for an extra round trip whenever the user opens the attachment.
/// </summary>
private const int DetachedPartMinimumSizeInBytes = 64 * 1024;
/// <summary>
/// Header stamped on a detached part carrying the index its body is stored and requested under. Clients
/// read it from the part headers, so the index never depends on how a given MIME parser orders attachments.
/// </summary>
private const string DetachedPartIndexHeader = "X-AliasVault-Part";
/// <summary>
/// Header stamped on a detached part carrying the decoded size of the attachment, so clients can show the
/// file size in the attachment list without downloading the body first.
/// </summary>
private const string DetachedPartLengthHeader = "X-AliasVault-Detached-Length";
/// <summary>
/// Override the SaveAsync method to save the email into the database.
/// </summary>
/// <param name="context">ISessionContext instance.</param>
/// <param name="transaction">IMessageTransaction instance.</param>
/// <param name="buffer">Buffer which contains the email contents.</param>
/// <param name="cancellationToken">CancellationToken instance.</param>
/// <returns>SmtpResponse.</returns>
public override async Task<SmtpResponse> SaveAsync(ISessionContext context, IMessageTransaction transaction, ReadOnlySequence<byte> buffer, CancellationToken cancellationToken)
{
try
{
if (buffer.Length > MaxEmailSizeInBytes)
{
return SmtpResponse.SizeLimitExceeded;
}
var message = await LoadMessageFromBuffer(buffer, cancellationToken);
// Detach the large attachments once, up front: this mutates the message into the skeleton that every
// recipient's copy is serialized from, so it must not run again inside the per-recipient loop.
var detachedParts = DetachLargeAttachments(message);
// Retrieve all addresses from the SMTP transaction which should contain all recipients for this mail instance.
var allAddresses = transaction.To
.Distinct()
.ToList();
// Limit list to 15 addresses maximum to prevent mailbomb/spam abuse.
var toAddresses = allAddresses.Take(15).ToList();
var toAddressesCount = toAddresses.Count;
var toAddressesFailCount = 0;
foreach (var toAddress in toAddresses)
{
// Process the email for each recipient separately.
var process = await ProcessEmailForRecipient(message, detachedParts, toAddress);
if (!process)
{
toAddressesFailCount++;
}
// If all recipients failed, return error to sender.
if (toAddressesFailCount == toAddressesCount)
{
// No valid recipients given.
logger.LogDebug("No valid recipients in email, returning error to sender.");
return SmtpResponse.NoValidRecipientsGiven;
}
}
return SmtpResponse.Ok;
}
catch (Exception ex)
{
logger.LogError(ex, "Error saving email into database.");
return SmtpResponse.MailboxUnavailable;
}
}
/// <summary>
/// Load the email message from the buffer.
/// </summary>
/// <param name="buffer">Buffer which contains the email contents.</param>
/// <param name="cancellationToken">CancellationToken instance.</param>
/// <returns>MimeMessage.</returns>
private static async Task<MimeMessage> LoadMessageFromBuffer(ReadOnlySequence<byte> buffer, CancellationToken cancellationToken)
{
await using var stream = new MemoryStream();
var position = buffer.GetPosition(0);
while (buffer.TryGet(ref position, out var memory))
{
stream.Write(memory.Span);
}
stream.Position = 0;
return await MimeMessage.LoadAsync(stream, cancellationToken);
}
/// <summary>
/// Convert MimeMessage to Email database object.
/// </summary>
/// <param name="message">MimeMessage object.</param>
/// <param name="detachedParts">The attachment bodies detached from the message source at ingest.</param>
/// <param name="toAddress">The recipient for this mail.</param>
/// <returns>Email object.</returns>
private static Email ConvertMimeMessageToEmail(MimeMessage message, IReadOnlyCollection<DetachedPart> detachedParts, MailAddress toAddress)
{
var from = string.Empty;
try
{
from = message.From.FirstOrDefault()?.ToString() ?? string.Empty;
}
catch
{
// Do nothing.
}
string fromLocal;
string fromDomain;
// Try to extract from address firstly from "from" in the mail.
try
{
MailAddress fromAddress = new MailAddress(message.From.FirstOrDefault()?.ToString() ?? string.Empty);
fromLocal = fromAddress.User;
fromDomain = fromAddress.Host;
}
catch
{
// If this fails, then simply use a blank value.
fromLocal = string.Empty;
fromDomain = string.Empty;
}
// Create email object. The gzipped raw source is the single authoritative copy of the body content:
// clients derive the html/plain bodies and attachments by parsing it after decrypt+gunzip.
var email = new Email
{
From = from,
FromLocal = fromLocal.ToLower(),
FromDomain = fromDomain.ToLower(),
To = toAddress.Address.ToLower(),
ToLocal = toAddress.User.ToLower(),
ToDomain = toAddress.Host.ToLower(),
Subject = message.Subject ?? string.Empty,
MessageSourceBytes = CompressMessageSource(message),
AttachmentCount = message.Attachments.Count(),
Date = message.Date.DateTime.ToUniversalTime(),
DateSystem = DateTime.UtcNow,
Visible = true,
};
// Extract a preview of the email message body to be used in the email listing preview in the UI.
email.MessagePreview = ExtractMessagePreview(message.TextBody, message.HtmlBody);
// Add the detached parts to the email.
foreach (var part in detachedParts)
{
email.Parts.Add(new EmailPart { PartIndex = part.PartIndex, Bytes = part.Bytes });
}
return email;
}
/// <summary>
/// Serialize the full RFC 822 message and gzip-compress it.
/// </summary>
/// <param name="message">MimeMessage object.</param>
/// <returns>Gzip-compressed raw message source.</returns>
private static byte[] CompressMessageSource(MimeMessage message)
{
using var output = new MemoryStream();
using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true))
{
message.WriteTo(gzip);
}
return output.ToArray();
}
/// <summary>
/// Gzip-compress a byte array.
/// </summary>
/// <param name="bytes">The bytes to compress.</param>
/// <returns>Gzip-compressed bytes.</returns>
private static byte[] Compress(byte[] bytes)
{
using var output = new MemoryStream();
using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true))
{
gzip.Write(bytes, 0, bytes.Length);
}
return output.ToArray();
}
/// <summary>
/// Move the body of every sizeable attachment out of the message and into a separately stored part, leaving
/// the message a valid MIME skeleton: the attachment keeps its headers (and therefore its filename and MIME
/// type, which stay encrypted along with the rest of the source) but carries an empty body plus the two
/// X-AliasVault-* headers that tell clients where to fetch it and how large it is.
///
/// The captured body is the transfer-encoded form exactly as it appeared in the source, so splicing it back in
/// is a plain byte insert in case client wants to fully reconstruct the message source.
/// </summary>
/// <param name="message">MimeMessage to detach the attachments from. Mutated in place.</param>
/// <returns>The detached parts, in the order they were stamped.</returns>
private static List<DetachedPart> DetachLargeAttachments(MimeMessage message)
{
var detached = new List<DetachedPart>();
foreach (var part in message.BodyParts.OfType<MimePart>())
{
if (!part.IsAttachment || part.Content is null)
{
continue;
}
using var encoded = new MemoryStream();
part.Content.WriteTo(encoded);
if (encoded.Length < DetachedPartMinimumSizeInBytes)
{
continue;
}
using var measuring = new MeasuringStream();
part.Content.DecodeTo(measuring);
var partIndex = detached.Count;
part.Headers[DetachedPartIndexHeader] = partIndex.ToString(CultureInfo.InvariantCulture);
part.Headers[DetachedPartLengthHeader] = measuring.Length.ToString(CultureInfo.InvariantCulture);
var encoding = part.Content.Encoding;
part.Content = new MimeContent(new MemoryStream([], false), encoding);
detached.Add(new DetachedPart(partIndex, Compress(encoded.ToArray())));
}
return detached;
}
/// <summary>
/// Extracts a preview of the email message body to be used in the email listing preview in the UI.
/// This so the client does not need to load the full email body when rendering a list.
/// </summary>
/// <param name="messagePlain">The parsed plain text body of the message, if any.</param>
/// <param name="messageHtml">The parsed HTML body of the message, if any.</param>
/// <returns>Email preview as string.</returns>
private static string ExtractMessagePreview(string? messagePlain, string? messageHtml)
{
var messagePreview = string.Empty;
const int maxPreviewLength = 180;
try
{
if (messagePlain != null && !string.IsNullOrEmpty(messagePlain) && messagePlain.Length > 3)
{
// Decode HTML entities (e.g., ' -> ', & -> &)
string plainToPlainText = System.Net.WebUtility.HtmlDecode(messagePlain);
// Replace any newline characters with a space
plainToPlainText = Regex.Replace(plainToPlainText, @"\t|\n|\r", " ", RegexOptions.NonBacktracking);
// Remove all "-" or "=" characters if there are 3 or more in a row
plainToPlainText = Regex.Replace(plainToPlainText, @"-{3,}|\={3,}", string.Empty, RegexOptions.NonBacktracking);
// Remove control characters while preserving printable Unicode such as accented letters and emoji.
plainToPlainText = Regex.Replace(plainToPlainText, @"\p{Cc}", string.Empty, RegexOptions.NonBacktracking);
// Replace multiple spaces with a single space
plainToPlainText = Regex.Replace(plainToPlainText, @"\s+", " ", RegexOptions.NonBacktracking);
// Trim start and end of string
plainToPlainText = plainToPlainText.Trim();
messagePreview = plainToPlainText.Length > maxPreviewLength
? plainToPlainText.Substring(0, maxPreviewLength)
: plainToPlainText;
}
else if (messageHtml != null)
{
string htmlToPlainText = Uglify.HtmlToText(messageHtml).ToString();
// Decode HTML entities (e.g., ' -> ', & -> &)
htmlToPlainText = System.Net.WebUtility.HtmlDecode(htmlToPlainText);
// Replace any newline characters with a space
htmlToPlainText = Regex.Replace(htmlToPlainText, @"\t|\n|\r", string.Empty, RegexOptions.NonBacktracking);
// Remove all "-" or "=" characters if there are 3 or more in a row
htmlToPlainText = Regex.Replace(htmlToPlainText, @"-{3,}|\={3,}", string.Empty, RegexOptions.NonBacktracking);
// Remove control characters while preserving printable Unicode such as accented letters and emoji.
htmlToPlainText = Regex.Replace(htmlToPlainText, @"\p{Cc}", string.Empty, RegexOptions.NonBacktracking);
// Replace multiple spaces with a single space
htmlToPlainText = Regex.Replace(htmlToPlainText, @"\s+", " ", RegexOptions.NonBacktracking);
// Trim start and end of string
htmlToPlainText = htmlToPlainText.Trim();
messagePreview =
htmlToPlainText.Length > maxPreviewLength ? htmlToPlainText.Substring(0, maxPreviewLength) : htmlToPlainText;
}
}
catch
{
// Extracting useful words from email failed. Skip the step, do nothing.
}
return messagePreview;
}
/// <summary>
/// Process email for recipient separately.
/// </summary>
/// <param name="message">MimeMessage.</param>
/// <param name="detachedParts">The attachment bodies detached from the message source at ingest.</param>
/// <param name="toAddress">ToAddress.</param>
/// <returns>True if success or silent skip, false if SmtpResponse.NoValidRecipientsGiven should be triggered.</returns>
private async Task<bool> ProcessEmailForRecipient(MimeMessage message, IReadOnlyCollection<DetachedPart> detachedParts, IMailbox? toAddress)
{
// Check if toAddress domain is allowed.
if (toAddress is null ||
string.IsNullOrWhiteSpace(toAddress.Host) ||
!config.AllowedToDomains.Contains(toAddress.Host.Trim().ToLowerInvariant()))
{
// ToAddress domain is not allowed.
logger.LogInformation(
"Rejected email: email for {ToAddress} is not allowed. Domain not in allowed domain list.",
toAddress?.User + "@" + toAddress?.Host);
return false;
}
// Check if the local part of the toAddress is a known alias (claimed by a user)
await using var dbContext = await dbContextFactory.CreateDbContextAsync(CancellationToken.None);
var toAddressLocal = toAddress.User.ToLowerInvariant();
var toAddressDomain = toAddress.Host.ToLowerInvariant();
var emailClaim = await dbContext.EmailClaims
.FirstOrDefaultAsync(
x =>
x.AddressLocal == toAddressLocal &&
x.AddressDomain == toAddressDomain,
CancellationToken.None);
if (emailClaim is null)
{
// Email address has no user claim with corresponding encryption key, so we cannot process it.
logger.LogInformation(
"Rejected email: email for {ToAddress} is not allowed. No user claim on this ToAddress.",
toAddress.User + "@" + toAddress.Host);
return false;
}
if (emailClaim.VaultManifestId is not Guid ownerManifestId)
{
// The claim is orphaned: the manifest that owned it no longer exists (owner deleted account or vault).
logger.LogInformation(
"Rejected email: email for {ToAddress} is claimed but its owning vault no longer exists. The owner has most likely deleted their account.",
toAddress.User + "@" + toAddress.Host);
return false;
}
if (emailClaim.State == EmailClaimState.Removed)
{
// Email claim is disabled, so we cannot process this email.
logger.LogInformation(
"Rejected email: email for {ToAddress} is claimed but is disabled which means the user has deleted the email alias.",
toAddress.User + "@" + toAddress.Host);
return false;
}
if (emailClaim.State == EmailClaimState.Paused)
{
logger.LogInformation(
"Rejected email: email for {ToAddress} is claimed but the vault that carries it has the alias switched off.",
toAddress.User + "@" + toAddress.Host);
return false;
}
// Resolve the owning manifest's primary delivery key.
var deliveryKeys = await dbContext.VaultManifestDeliveryKeys.Where(x => x.VaultManifestId == ownerManifestId && x.IsPrimary).ToListAsync(CancellationToken.None);
if (deliveryKeys.Count == 0)
{
// The owning manifest has no published primary delivery key, so we cannot process this email.
logger.LogWarning(
"Rejected email: email for {ToAddress} cannot be processed. No primary delivery encryption key found for manifest {ManifestId}.",
toAddress.User + "@" + toAddress.Host,
ownerManifestId);
return false;
}
// Resolve the group that owns the manifest, only used to increment its EmailsReceived counter.
var recipientGroupIds = await dbContext.VaultManifests.Where(m => m.ManifestId == ownerManifestId).Select(m => m.OwnerGroupId).ToListAsync(CancellationToken.None);
var insertedId = await InsertEmailIntoDatabase(message, detachedParts, new MailAddress(toAddress.AsAddress()), deliveryKeys, recipientGroupIds, emailClaim.Id, emailClaim.AnonymizedSenderCounted);
logger.LogDebug("Email for {ToAddress} successfully saved into database with ID {InsertedId}.", toAddress.User + "@" + toAddress.Host, insertedId);
return true;
}
/// <summary>
/// Insert email into database.
/// </summary>
/// <param name="message">MimeMessage to save into database.</param>
/// <param name="detachedParts">The attachment bodies detached from the message source at ingest.</param>
/// <param name="toAddress">The recipient for this mail.</param>
/// <param name="deliveryKeys">The delivery keys of every manifest claiming this alias; each gets its own wrap of the email's symmetric key.</param>
/// <param name="recipientGroupIds">The groups that own the manifests this alias is claimed by.</param>
/// <param name="emailClaimId">The claim on the recipient alias, latched when its anonymized sender bucket is counted.</param>
/// <param name="senderAlreadyCounted">Whether the sender is already counted for anonymized sender usage detection.</param>
private async Task<int> InsertEmailIntoDatabase(MimeMessage message, IReadOnlyCollection<DetachedPart> detachedParts, MailAddress toAddress, IReadOnlyCollection<VaultManifestDeliveryKey> deliveryKeys, IReadOnlyCollection<Guid> recipientGroupIds, Guid emailClaimId, bool senderAlreadyCounted)
{
await using var dbContext = await dbContextFactory.CreateDbContextAsync();
var newEmail = ConvertMimeMessageToEmail(message, detachedParts, toAddress);
var senderHost = newEmail.FromDomain;
newEmail = EmailEncryption.EncryptEmail(newEmail, deliveryKeys);
// Insert the email into the database.
dbContext.Emails.Add(newEmail);
await dbContext.SaveChangesAsync();
await IncrementEmailsReceived(dbContext, recipientGroupIds);
await RecordAnonymizedSenderBucket(dbContext, emailClaimId, senderAlreadyCounted, senderHost, recipientGroupIds);
return newEmail.Id;
}
/// <summary>
/// Increment the EmailsReceived counter for every recipient group.
/// </summary>
/// <param name="dbContext">The context the email was inserted with.</param>
/// <param name="recipientGroupIds">The groups that own the manifests this alias is claimed by.</param>
private async Task IncrementEmailsReceived(AliasServerDbContext dbContext, IReadOnlyCollection<Guid> recipientGroupIds)
{
try
{
var groupIds = recipientGroupIds.ToArray();
await dbContext.Database.ExecuteSqlAsync($"UPDATE \"Groups\" SET \"EmailsReceived\" = \"EmailsReceived\" + 1 WHERE \"Id\" = ANY({groupIds})");
}
catch (Exception ex)
{
logger.LogWarning(ex, "Could not increment the EmailsReceived counter for the recipient groups.");
}
}
/// <summary>
/// Record the sender host in the sender bucket for anonymized sender usage detection.
/// </summary>
/// <param name="dbContext">The context the email was inserted with.</param>
/// <param name="emailClaimId">The claim on the recipient alias.</param>
/// <param name="senderAlreadyCounted">Whether the sender is already counted for anonymized sender usage detection.</param>
/// <param name="senderHost">The plaintext sender host, captured before encryption.</param>
/// <param name="recipientGroupIds">The groups that own the manifests this alias is claimed by.</param>
private async Task RecordAnonymizedSenderBucket(AliasServerDbContext dbContext, Guid emailClaimId, bool senderAlreadyCounted, string senderHost, IReadOnlyCollection<Guid> recipientGroupIds)
{
if (senderAlreadyCounted || string.IsNullOrWhiteSpace(config.AbuseMetricsSalt) || string.IsNullOrWhiteSpace(senderHost))
{
return;
}
try
{
// Set the sender already counted flag for the email claim.
var latched = await dbContext.Database.ExecuteSqlAsync($"UPDATE \"EmailClaims\" SET \"AnonymizedSenderCounted\" = true WHERE \"Id\" = {emailClaimId} AND \"AnonymizedSenderCounted\" = false");
if (latched == 0)
{
return;
}
// Compute the bucket index for the sender host and increment the count for the recipient groups.
var position = AnonymizedSenderBucket.Compute(config.AbuseMetricsSalt, senderHost) + 1;
var groupIds = recipientGroupIds.ToArray();
await dbContext.Database.ExecuteSqlAsync(
$"""
UPDATE "Groups"
SET "AnonymizedEmailAliasSenderCounts"[{position}] = COALESCE("AnonymizedEmailAliasSenderCounts"[{position}], 0) + 1
WHERE "Id" = ANY({groupIds}) AND array_length("AnonymizedEmailAliasSenderCounts", 1) >= {position}
""");
}
catch (Exception ex)
{
// Could not record the anonymized sender bucket for email claim. The email itself was stored and delivered normally.
logger.LogWarning(ex, "Could not record the anonymized sender bucket for email claim {EmailClaimId}. The email itself was stored and delivered normally.", emailClaimId);
}
}
/// <summary>
/// An attachment body lifted out of the message source at ingest, ready to be stored as an EmailPart row
/// for every recipient of the message.
/// </summary>
/// <param name="PartIndex">The index stamped on the part in the source as the X-AliasVault-Part header.</param>
/// <param name="Bytes">The transfer-encoded part body, gzip-compressed but not yet encrypted.</param>
private sealed record DetachedPart(int PartIndex, byte[] Bytes);
}