Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/wikipedia): emit og:image meta tag from Enterprise API article.image.content_url
firecrawl-spring[bot]03d53c8f6daa
diff --git a/apps/api/src/__tests__/snips/v2/wikipedia.test.ts b/apps/api/src/__tests__/snips/v2/wikipedia.test.tsindex dc9f24f53..a0accdc1d 100644--- a/apps/api/src/__tests__/snips/v2/wikipedia.test.ts+++ b/apps/api/src/__tests__/snips/v2/wikipedia.test.ts@@ -276,6 +276,27 @@ describeIf(HAS_WIKIPEDIA && !process.env.TEST_SUITE_SELF_HOSTED)();});+ describe("og:image metadata", () => {+ it.concurrent(+ "includes og:image in metadata and rawHtml for articles with an image",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/Military",+ formats: ["rawHtml"],+ },+ identity,+ );++ expect(response.metadata.statusCode).toBe(200);+ expect(response.metadata.ogImage).toBeTruthy();+ expect(response.metadata.ogImage).toMatch(/^https?:\/\//);+ expect(response.rawHtml).toContain('property="og:image"');+ },+ scrapeTimeout,+ );+ });+describe("error handling", () => {it.concurrent("fails gracefully for non-existent Wikipedia article",diff --git a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.tsindex 77ff5a117..1f6a17e7b 100644--- a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts@@ -221,6 +221,11 @@ interface WikimediaArticle {date_modified: string;categories?: { name: string; url: string }[];has_parts?: { name: string }[];+ image?: {+ content_url: string;+ width?: number;+ height?: number;+ };}export async function scrapeURLWithWikipedia(@@ -328,6 +333,7 @@ export async function scrapeURLWithWikipedia(<meta property="og:title" content="${escapeHtml(article.name)}"><meta property="og:description" content="${escapeHtml(article.abstract ?? "")}"><meta property="og:url" content="${escapeHtml(article.url ?? urlToScrape)}">+ ${article.image?.content_url ? `<meta property="og:image" content="${escapeHtml(article.image.content_url)}">` : ""}<meta name="article:modified_time" content="${article.date_modified ?? ""}"><meta name="language" content="${article.in_language?.name ?? lang}"></head>