Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix: Handle invalid favicon URLs gracefully in metadata extraction (#2361)
Abimael Martell608604b8ba5f
diff --git a/apps/api/src/__tests__/snips/metadata-concat.test.ts b/apps/api/src/__tests__/snips/metadata-concat.test.tsindex 13e0334eb..251c35a5a 100644--- a/apps/api/src/__tests__/snips/metadata-concat.test.ts+++ b/apps/api/src/__tests__/snips/metadata-concat.test.ts@@ -41,4 +41,66 @@ describe("Metadata concatenation", () => {expect(Array.isArray(metadata.keywords)).toBe(true);expect(metadata.keywords).toEqual(["first keyword", "second keyword"]);});++ it("should handle invalid favicon URLs gracefully", async () => {+ const html = `+ <html>+ <head>+ <link rel="icon" href="//#DOMAIN#/favicon.ico">+ <title>Test Page</title>+ </head>+ <body></body>+ </html>+ `;++ const mockLogger = {+ warn: jest.fn(),+ error: jest.fn(),+ debug: jest.fn(),+ };++ const meta: any = {+ url: "https://example.com",+ rewrittenUrl: "https://example.com",+ id: "test-id",+ logger: mockLogger,+ };++ const metadata = await extractMetadata(meta, html);++ expect(metadata.favicon).toBeUndefined();+ expect(mockLogger.debug).toHaveBeenCalledWith(+ "Failed to resolve favicon URL",+ expect.objectContaining({+ favicon: expect.stringContaining("#DOMAIN#"),+ error: expect.any(Error),+ }),+ );+ });++ it("should successfully resolve valid favicon URLs", async () => {+ const html = `+ <html>+ <head>+ <link rel="icon" href="/favicon.ico">+ <title>Test Page</title>+ </head>+ <body></body>+ </html>+ `;++ const meta: any = {+ url: "https://example.com",+ id: "test-id",+ logger: {+ warn: jest.fn(),+ error: jest.fn(),+ debug: jest.fn(),+ },+ };++ const metadata = await extractMetadata(meta, html);++ expect(metadata.favicon).toBe("https://example.com/favicon.ico");+ });});diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex 1d4bc253f..2a94af883 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -7,16 +7,24 @@ async function extractMetadataRust(meta: Meta,html: string,): Promise<Partial<Document["metadata"]>> {- const fromRust = await _extractMetadata(html);+ const { favicon: _favicon, ...fromRust } = await _extractMetadata(html);++ let favicon: string | undefined = undefined;++ if (_favicon) {+ try {+ favicon = new URL(_favicon, meta.rewrittenUrl ?? meta.url).href;+ } catch (error) {+ meta.logger.debug("Failed to resolve favicon URL", {+ favicon: _favicon,+ error,+ });+ }+ }return {...fromRust,- ...(fromRust.favicon- ? {- favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)- .href,- }- : {}),+ favicon,scrapeId: meta.id,};}