Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update FaviconExtractor logic and extend tests (#2383)

Leendert de Borst7d27ee57b66b
diff --git a/apps/server/Tests/AliasVault.UnitTests/Utilities/FaviconExtractorNetworkTests.cs b/apps/server/Tests/AliasVault.UnitTests/Utilities/FaviconExtractorNetworkTests.cs
index c693f843f..582f29733 100644
--- a/apps/server/Tests/AliasVault.UnitTests/Utilities/FaviconExtractorNetworkTests.cs
+++ b/apps/server/Tests/AliasVault.UnitTests/Utilities/FaviconExtractorNetworkTests.cs
@@ -28,6 +28,9 @@ public class FaviconExtractorNetworkTests
"https://gmail.com", // Redirects to another host; icon path resolves against the final URL.
"https://www.edreams.es", // Page behind a CDN that blocks the page request itself.
"https://www.infojobs.net", // Page declaring a charset .NET cannot construct.
+ "https://mudblazor.com/getting-started/installation", // Blazor app: page under a sub-path with <base href="/"> and a relative icon href.
+ "https://www.instagram.com/accounts/login/", // Heavy page (600KB+) whose preferred icon is WebP.
+ "https://www.epicgames.com/id/login", // Cloudflare-fronted; admits HTTP/1.1 but challenges HTTP/2 requests.
];
/// <summary>
diff --git a/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs b/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs
index 02ac87168..af4819098 100644
--- a/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs
+++ b/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs
@@ -28,7 +28,7 @@ public static class FaviconExtractor
private const int MaxSizeBytes = 20 * 1024; // 20KB max size; images above this are resized/re-encoded.
private const int MaxResponseBytes = 5 * 1024 * 1024; // 5MB cap per response body, measured after decompression.
private const int MaxDecodedPixels = 2048 * 2048; // Pixel budget per image; see IsWithinDecodePixelBudget.
- private const int MaxFaviconCandidates = 10; // Distinct favicon URLs fetched per page; see TryExtractFaviconFromNodes.
+ private const int MaxFaviconCandidates = 10; // Distinct favicon URLs fetched per page; see TryFetchFaviconCandidatesAsync.
private static readonly TimeSpan ExtractionDeadline = TimeSpan.FromSeconds(5); // Wall-clock budget for one full extraction.
private static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(3); // Per-request budget, so one slow host still leaves room for another candidate.
private static readonly int[] ResizeWidths = [96, 64, 48, 32];
@@ -286,6 +286,74 @@ public static class FaviconExtractor
return (long)info.Width * info.Height <= MaxDecodedPixels;
}
+ /// <summary>
+ /// Lists the favicon URLs a page declares, absolute and in order of preference, ending with the
+ /// origin's default /favicon.ico. Relative hrefs resolve against the page's base URL as a browser would.
+ /// </summary>
+ /// <param name="htmlBytes">The raw page bytes.</param>
+ /// <param name="documentUri">The URL the page was served from.</param>
+ /// <returns>The distinct candidate URLs.</returns>
+ internal static IReadOnlyList<string> GetFaviconCandidateUrls(byte[] htmlBytes, Uri documentUri)
+ {
+ HtmlDocument htmlDoc = new();
+ using (var htmlStream = new MemoryStream(htmlBytes))
+ {
+ htmlDoc.Load(htmlStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
+ }
+
+ var baseUri = ResolveDocumentBaseUri(htmlDoc, documentUri);
+
+ // Icon link queries, in order of preference.
+ string[] queries =
+ [
+ "//link[@rel='icon' and @type='image/svg+xml']",
+ "//link[@rel='icon' and @sizes='96x96']",
+ "//link[@rel='icon' and @sizes='128x128']",
+ "//link[@rel='icon' and @sizes='48x48']",
+ "//link[@rel='icon' and @sizes='32x32']",
+ "//link[@rel='icon' and @sizes='192x192']",
+ "//link[@rel='apple-touch-icon' or @rel='apple-touch-icon-precomposed']",
+ "//link[@rel='icon' or @rel='shortcut icon']",
+ ];
+
+ var candidates = new List<string>();
+ var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+ foreach (var query in queries)
+ {
+ var nodes = htmlDoc.DocumentNode.SelectNodes(query);
+ if (nodes == null)
+ {
+ continue;
+ }
+
+ foreach (var node in nodes)
+ {
+ var href = node.Attributes["href"]?.DeEntitizeValue?.Trim();
+
+ // Skip empty or unparseable hrefs.
+ if (string.IsNullOrEmpty(href) || !Uri.TryCreate(baseUri, href, out var resolved))
+ {
+ continue;
+ }
+
+ if (seenUrls.Add(resolved.AbsoluteUri))
+ {
+ candidates.Add(resolved.AbsoluteUri);
+ }
+ }
+ }
+
+ // Try conventional location as fallback.
+ var defaultFavicon = $"{documentUri.GetLeftPart(UriPartial.Authority)}/favicon.ico";
+ if (seenUrls.Add(defaultFavicon))
+ {
+ candidates.Add(defaultFavicon);
+ }
+
+ return candidates;
+ }
+
/// <summary>
/// Opens the TCP connection for an outgoing request and resolve a hostname only once.
/// </summary>
@@ -360,94 +428,44 @@ public static class FaviconExtractor
// Resolve relative hrefs against the URL we actually ended up on after possible redirect.
var finalUri = response.RequestMessage?.RequestUri ?? uri;
- var faviconNodes = await GetFaviconNodesFromHtml(response, finalUri, cancellationToken);
- return await TryExtractFaviconFromNodes(faviconNodes, client, finalUri, cancellationToken);
+ var htmlBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken);
+ var candidates = GetFaviconCandidateUrls(htmlBytes, finalUri);
+ return await TryFetchFaviconCandidatesAsync(client, candidates, cancellationToken);
}
/// <summary>
- /// Gets the favicon nodes from the HTML.
+ /// Resolves the URL relative links in a page resolve against: the first base href when
+ /// present and usable, otherwise the page URL.
/// </summary>
- /// <param name="response">The response to get the favicon nodes from.</param>
- /// <param name="uri">The URI to get the favicon nodes from.</param>
- /// <param name="cancellationToken">Token maxing the overall extraction.</param>
- /// <returns>The favicon nodes.</returns>
- private static async Task<HtmlNodeCollection[]> GetFaviconNodesFromHtml(HttpResponseMessage response, Uri uri, CancellationToken cancellationToken)
+ /// <param name="htmlDoc">The parsed page.</param>
+ /// <param name="documentUri">The URL the page was served from.</param>
+ /// <returns>The base URL for relative links.</returns>
+ private static Uri ResolveDocumentBaseUri(HtmlDocument htmlDoc, Uri documentUri)
{
- var htmlBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken);
- HtmlDocument htmlDoc = new();
- using (var htmlStream = new MemoryStream(htmlBytes))
+ var baseHref = htmlDoc.DocumentNode.SelectSingleNode("//base[@href]")?.Attributes["href"]?.DeEntitizeValue?.Trim();
+ if (!string.IsNullOrEmpty(baseHref) && Uri.TryCreate(documentUri, baseHref, out var baseUri) && AllowedSchemes.Contains(baseUri.Scheme))
{
- htmlDoc.Load(htmlStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
+ return baseUri;
}
- var defaultFavicon = new HtmlNode(HtmlNodeType.Element, htmlDoc, 0);
- defaultFavicon.Attributes.Add("href", $"{uri.GetLeftPart(UriPartial.Authority)}/favicon.ico");
-
- // Get the favicon nodes from the HTML, in order of preference.
- HtmlNodeCollection?[] nodeArray =
- [
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @type='image/svg+xml']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @sizes='96x96']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @sizes='128x128']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @sizes='48x48']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @sizes='32x32']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' and @sizes='192x192']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='apple-touch-icon' or @rel='apple-touch-icon-precomposed']"),
- htmlDoc.DocumentNode.SelectNodes("//link[@rel='icon' or @rel='shortcut icon']"),
- new(htmlDoc.DocumentNode) { defaultFavicon },
- ];
-
- // Filter node array to only return non-null values and cast to non-nullable array
- return nodeArray.Where(x => x != null).Cast<HtmlNodeCollection>().ToArray();
+ return documentUri;
}
/// <summary>
- /// Tries to extract the favicon from the nodes.
+ /// Fetches the candidate URLs in order and returns the first usable icon.
/// </summary>
- /// <param name="faviconNodes">The favicon nodes.</param>
/// <param name="client">The HTTP client.</param>
- /// <param name="baseUri">The base URI.</param>
+ /// <param name="candidates">The candidate URLs, best first.</param>
/// <param name="cancellationToken">Token maxing the overall extraction.</param>
- /// <returns>The favicon bytes.</returns>
- private static async Task<byte[]?> TryExtractFaviconFromNodes(HtmlNodeCollection[] faviconNodes, HttpClient client, Uri baseUri, CancellationToken cancellationToken)
+ /// <returns>The favicon bytes, or null if no candidate yielded a usable icon.</returns>
+ private static async Task<byte[]?> TryFetchFaviconCandidatesAsync(HttpClient client, IReadOnlyList<string> candidates, CancellationToken cancellationToken)
{
- var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-
- foreach (var nodeCollection in faviconNodes)
+ foreach (var candidate in candidates.Take(MaxFaviconCandidates))
{
- if (nodeCollection == null || nodeCollection.Count == 0)
- {
- continue;
- }
-
- foreach (var node in nodeCollection)
+ var faviconBytes = await FetchAndProcessFaviconAsync(client, candidate, cancellationToken);
+ if (faviconBytes != null)
{
- var faviconUrl = node.GetAttributeValue("href", string.Empty);
- if (string.IsNullOrEmpty(faviconUrl))
- {
- continue;
- }
-
- if (!Uri.IsWellFormedUriString(faviconUrl, UriKind.Absolute))
- {
- faviconUrl = new Uri(baseUri, faviconUrl).ToString();
- }
-
- if (!seenUrls.Add(faviconUrl))
- {
- continue;
- }
-
- if (seenUrls.Count > MaxFaviconCandidates)
- {
- return null;
- }
-
- var faviconBytes = await FetchAndProcessFaviconAsync(client, faviconUrl, cancellationToken);
- if (faviconBytes != null)
- {
- return faviconBytes;
- }
+ return faviconBytes;
}
}
@@ -535,9 +553,9 @@ public static class FaviconExtractor
var handler = new SocketsHttpHandler
{
AllowAutoRedirect = false, // Handle redirects manually
- UseCookies = true, // Enable cookie handling for session management
- CookieContainer = new System.Net.CookieContainer(),
- AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate | System.Net.DecompressionMethods.Brotli,
+ UseCookies = true, // Enable cookie handling for session management
+ CookieContainer = new CookieContainer(),
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
ConnectCallback = ConnectToValidatedAddressAsync,
};
@@ -627,8 +645,8 @@ public static class FaviconExtractor
private static async Task<HttpResponseMessage?> FollowRedirectsAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
{
var currentUri = uri;
- int redirectCount = 0;
- const int maxRedirects = 5;
+ var redirectCount = 0;
+ var maxRedirects = 5;
while (redirectCount < maxRedirects)
{