Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix: Handle invalid favicon URLs gracefully in metadata extraction (#2361)

Abimael Martell608604b8ba5f
diff --git a/apps/api/src/__tests__/snips/metadata-concat.test.ts b/apps/api/src/__tests__/snips/metadata-concat.test.ts
index 13e0334eb..251c35a5a 100644
--- a/apps/api/src/__tests__/snips/metadata-concat.test.ts
+++ b/apps/api/src/__tests__/snips/metadata-concat.test.ts
@@ -41,4 +41,66 @@ describe("Metadata concatenation", () => {
expect(Array.isArray(metadata.keywords)).toBe(true);
expect(metadata.keywords).toEqual(["first keyword", "second keyword"]);
});
+
+ it("should handle invalid favicon URLs gracefully", async () => {
+ const html = `
+ <html>
+ <head>
+ <link rel="icon" href="//#DOMAIN#/favicon.ico">
+ <title>Test Page</title>
+ </head>
+ <body></body>
+ </html>
+ `;
+
+ const mockLogger = {
+ warn: jest.fn(),
+ error: jest.fn(),
+ debug: jest.fn(),
+ };
+
+ const meta: any = {
+ url: "https://example.com",
+ rewrittenUrl: "https://example.com",
+ id: "test-id",
+ logger: mockLogger,
+ };
+
+ const metadata = await extractMetadata(meta, html);
+
+ expect(metadata.favicon).toBeUndefined();
+ expect(mockLogger.debug).toHaveBeenCalledWith(
+ "Failed to resolve favicon URL",
+ expect.objectContaining({
+ favicon: expect.stringContaining("#DOMAIN#"),
+ error: expect.any(Error),
+ }),
+ );
+ });
+
+ it("should successfully resolve valid favicon URLs", async () => {
+ const html = `
+ <html>
+ <head>
+ <link rel="icon" href="/favicon.ico">
+ <title>Test Page</title>
+ </head>
+ <body></body>
+ </html>
+ `;
+
+ const meta: any = {
+ url: "https://example.com",
+ id: "test-id",
+ logger: {
+ warn: jest.fn(),
+ error: jest.fn(),
+ debug: jest.fn(),
+ },
+ };
+
+ const metadata = await extractMetadata(meta, html);
+
+ expect(metadata.favicon).toBe("https://example.com/favicon.ico");
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
index 1d4bc253f..2a94af883 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -7,16 +7,24 @@ async function extractMetadataRust(
meta: Meta,
html: string,
): Promise<Partial<Document["metadata"]>> {
- const fromRust = await _extractMetadata(html);
+ const { favicon: _favicon, ...fromRust } = await _extractMetadata(html);
+
+ let favicon: string | undefined = undefined;
+
+ if (_favicon) {
+ try {
+ favicon = new URL(_favicon, meta.rewrittenUrl ?? meta.url).href;
+ } catch (error) {
+ meta.logger.debug("Failed to resolve favicon URL", {
+ favicon: _favicon,
+ error,
+ });
+ }
+ }
return {
...fromRust,
- ...(fromRust.favicon
- ? {
- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)
- .href,
- }
- : {}),
+ favicon,
scrapeId: meta.id,
};
}