Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/scrapeURL): add special support for YouTube watch pages (#2157)
Gergő Móriczebe2d8730876
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 53db121fb..e413fbbc4 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1297,6 +1297,25 @@ describe("Scrape tests", () => {scrapeTimeout * 5,);});++ describe("YouTube (f-e dependant)", () => {+ it.concurrent(+ "scrapes YouTube videos and transcripts",+ async () => {+ const response = await scrape(+ {+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.markdown).toContain("Rick Astley");+ expect(response.markdown).toContain("Never gonna let you down");+ },+ scrapeTimeout,+ );+ });}describe("URL rewriting", () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex dc3ec0e54..683dea09e 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -907,6 +907,7 @@ export type Document = {cacheState?: "hit" | "miss";cachedAt?: string;creditsUsed?: number;+ postprocessorsUsed?: string[];// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex dc67932cf..998321b7e 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -766,6 +766,7 @@ export type Document = {cacheState?: "hit" | "miss";cachedAt?: string;creditsUsed?: number;+ postprocessorsUsed?: string[];// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex d0391d77e..edb088d02 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -101,6 +101,7 @@ const successSchema = z.object({docUrl: z.string().optional(),usedMobileProxy: z.boolean().optional(),+ youtubeTranscriptContent: z.any().optional(),});export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex b764a5b50..116768348 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -31,6 +31,7 @@ import { getInnerJson } from "@mendable/firecrawl-rs";import { hasFormatOfType } from "../../../../lib/format-utils";import { Action } from "../../../../controllers/v1/types";import { AbortManagerThrownError } from "../../lib/abortManager";+import { youtubePostprocessor } from "../../postprocessors/youtube";// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the@@ -247,6 +248,12 @@ export async function scrapeURLWithFireEngineChromeCDP(!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,+ ...(youtubePostprocessor.shouldRun(+ meta,+ new URL(meta.rewrittenUrl ?? meta.url),+ )+ ? { blockMedia: false }+ : {}),};let response = await performFireEngineScrape(@@ -314,6 +321,7 @@ export async function scrapeURLWithFireEngineChromeCDP(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ youtubeTranscriptContent: response.youtubeTranscriptContent,};}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 876205831..743f744ed 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -50,7 +50,7 @@ export type FireEngineScrapeRequestChromeCDP = {engine: "chrome-cdp";skipTlsVerification?: boolean;actions?: Action[];- blockMedia?: true; // cannot be false+ blockMedia?: boolean;mobile?: boolean;disableSmartWaitCache?: boolean;};@@ -151,6 +151,7 @@ const successSchema = z.object({docUrl: z.string().optional(),usedMobileProxy: z.boolean().optional(),+ youtubeTranscriptContent: z.any().optional(),});type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 246e27a7e..affea25d2 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -127,6 +127,9 @@ export type EngineScrapeResult = {contentType?: string;+ youtubeTranscriptContent?: any;+ postprocessorsUsed?: string[];+proxyUsed: "basic" | "stealth";};diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 95eb1d0ae..cfe622261 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -82,6 +82,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {}: undefined,contentType: document.metadata.contentType,+ postprocessorsUsed: document.metadata.postprocessorsUsed,});} catch (error) {meta.logger.error("Failed to save document to index", {@@ -365,6 +366,8 @@ export async function scrapeURLWithIndex(created_at: new Date(data[0].created_at),},+ postprocessorsUsed: doc.postprocessorsUsed,+proxyUsed: doc.proxyUsed ?? "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c9f6e8a8c..73819e577 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -65,6 +65,7 @@ import {} from "./lib/abortManager";import { ScrapeJobTimeoutError } from "../../lib/error";import { htmlTransform } from "./lib/removeUnwantedElements";+import { postprocessors } from "./postprocessors";export type ScrapeUrlResponse =| {@@ -612,34 +613,64 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw new NoEnginesLeftError(fallbackList.map(x => x.engine));}+ let engineResult: EngineScrapeResult = result.result;++ for (const postprocessor of postprocessors) {+ if (+ postprocessor.shouldRun(+ meta,+ new URL(engineResult.url),+ engineResult.postprocessorsUsed,+ )+ ) {+ meta.logger.info("Running postprocessor " + postprocessor.name);+ try {+ engineResult = await postprocessor.run(+ {+ ...meta,+ logger: meta.logger.child({+ method: "postprocessors/" + postprocessor.name,+ }),+ },+ engineResult,+ );+ } catch (error) {+ meta.logger.warn("Failed to run postprocessor " + postprocessor.name, {+ error,+ });+ }+ }+ }+let document: Document = {- markdown: result.result.markdown,- rawHtml: result.result.html,- screenshot: result.result.screenshot,- actions: result.result.actions,+ markdown: engineResult.markdown,+ rawHtml: engineResult.html,+ screenshot: engineResult.screenshot,+ actions: engineResult.actions,metadata: {sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,- url: result.result.url,- statusCode: result.result.statusCode,- error: result.result.error,- numPages: result.result.pdfMetadata?.numPages,- ...(result.result.pdfMetadata?.title- ? { title: result.result.pdfMetadata.title }+ url: engineResult.url,+ statusCode: engineResult.statusCode,+ error: engineResult.error,+ numPages: engineResult.pdfMetadata?.numPages,+ ...(engineResult.pdfMetadata?.title+ ? { title: engineResult.pdfMetadata.title }: {}),- contentType: result.result.contentType,+ contentType: engineResult.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",...(fallbackList.find(x =>["index", "index;documents"].includes(x.engine),)- ? result.result.cacheInfo+ ? engineResult.cacheInfo? {cacheState: "hit",- cachedAt: result.result.cacheInfo.created_at.toISOString(),+ cachedAt: engineResult.cacheInfo.created_at.toISOString(),}: {cacheState: "miss",}: {}),+ postprocessorsUsed: engineResult.postprocessorsUsed,},};diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/index.ts b/apps/api/src/scraper/scrapeURL/postprocessors/index.tsnew file mode 100644index 000000000..e199b2e41--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/postprocessors/index.ts@@ -0,0 +1,14 @@+import { Meta } from "..";+import { EngineScrapeResult } from "../engines";+import { youtubePostprocessor } from "./youtube";++export interface Postprocessor {+ name: string;+ shouldRun: (meta: Meta, url: URL, postProcessorsUsed?: string[]) => boolean;+ run: (+ meta: Meta,+ engineResult: EngineScrapeResult,+ ) => Promise<EngineScrapeResult>;+}++export const postprocessors: Postprocessor[] = [youtubePostprocessor];diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.tsnew file mode 100644index 000000000..2cea65530--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts@@ -0,0 +1,104 @@+import type { Meta } from "..";+import type { Postprocessor } from ".";+import type { EngineScrapeResult } from "../engines";++export const youtubePostprocessor: Postprocessor = {+ name: "youtube",+ shouldRun: (_meta: Meta, url: URL, postProcessorsUsed?: string[]) => {+ if (postProcessorsUsed?.includes("youtube")) {+ return false;+ }++ if (+ url.hostname.endsWith(".youtube.com") ||+ url.hostname === "youtube.com"+ ) {+ return url.pathname === "/watch" && !!url.searchParams.get("v");+ } else if (url.hostname === "youtu.be") {+ return url.pathname !== "/";+ } else {+ return false;+ }+ },+ run: async (meta: Meta, engineResult: EngineScrapeResult) => {+ let initialData;+ try {+ initialData = JSON.parse(+ engineResult.html+ .split("var ytInitialPlayerResponse = ")[1]+ .split(";var meta =")[0],+ );+ } catch (e) {+ meta.logger.warn("Failed to parse YouTube initial data");+ return engineResult;+ }++ const largestThumbnail =+ initialData.videoDetails.thumbnail.thumbnails.slice(-1)[0];+ const lengthSeconds = parseFloat(initialData.videoDetails.lengthSeconds);+ const lengthTrueSeconds = lengthSeconds % 60;+ const lengthMinutes = Math.floor(lengthSeconds / 60) % 60;+ const lengthHours = Math.floor(lengthSeconds / 3600);++ const endscreen = (+ initialData.endscreen?.endscreenRenderer?.elements || []+ ).filter(x => x.endscreenElementRenderer?.style === "VIDEO");++ let preferredCaptionMarkdown = "";++ if (engineResult.youtubeTranscriptContent) {+ const initialSegments =+ engineResult.youtubeTranscriptContent?.actions?.[0]+ ?.updateEngagementPanelAction?.content?.transcriptRenderer?.content+ ?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer+ ?.initialSegments ?? [];+ const transcriptText = (+ Array.isArray(initialSegments) ? initialSegments : []+ )+ .map(x => x?.transcriptSegmentRenderer?.snippet?.runs?.[0]?.text)+ .filter(Boolean)+ .join(" ");+ preferredCaptionMarkdown = `## Transcript++${transcriptText}+`;+ }++ const markdown = `\++# [${initialData.videoDetails.title}](${initialData.microformat.playerMicroformatRenderer.canonicalUrl})++**Visibility**: ${initialData.videoDetails.isPrivate ? "Private" : initialData.microformat.playerMicroformatRenderer.isUnlisted ? "Unlisted" : "Public"}+**Uploaded by**: [${initialData.videoDetails.author}](${initialData.microformat.playerMicroformatRenderer.ownerProfileUrl})+**Uploaded at**: ${initialData.microformat.playerMicroformatRenderer.uploadDate}+**Published at**: ${initialData.microformat.playerMicroformatRenderer.publishDate}+**Length**: ${lengthHours > 0 ? `${lengthHours.toString().padStart(2, "0")}:` : ""}${lengthMinutes.toString().padStart(2, "0")}:${lengthTrueSeconds.toString().padStart(2, "0")}+**Views**: ${initialData.videoDetails.viewCount}+**Likes**: ${initialData.microformat.playerMicroformatRenderer.likeCount}+**Category**: ${initialData.microformat.playerMicroformatRenderer.category}++## Description++\`\`\`+${initialData.videoDetails.shortDescription}+\`\`\`++${preferredCaptionMarkdown ? preferredCaptionMarkdown + "\n\n" : ""}\+${+ endscreen.length > 0+ ? `## Endscreen++${endscreen.map(element => `- [${element.endscreenElementRenderer.title.simpleText}](${new URL(element.endscreenElementRenderer.endpoint.commandMetadata.webCommandMetadata.url, engineResult.url).toString()})`).join("\n")}`+ : ""+}`;++ return {+ ...engineResult,+ markdown,+ postprocessorsUsed: [+ ...(engineResult.postprocessorsUsed ?? []),+ "youtube",+ ],+ };+ },+};diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex e45b24a51..c09dd1266 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -1,6 +1,6 @@import { parseMarkdown } from "../../../lib/html-to-markdown";import { Meta } from "..";-import { Document } from "../../../controllers/v1/types";+import { Document } from "../../../controllers/v2/types";import { htmlTransform } from "../lib/removeUnwantedElements";import { extractLinks } from "../lib/extractLinks";import { extractImages } from "../lib/extractImages";@@ -72,6 +72,10 @@ async function deriveMarkdownFromHTML();}+ if (document.metadata.postprocessorsUsed?.includes("youtube")) {+ return document;+ }+if (document.metadata.contentType?.includes("application/json")) {if (document.rawHtml === undefined) {throw new Error(diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex b41814a4f..b9fa1e8f3 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -150,6 +150,7 @@ export async function saveIndexToGCS(screenshot?: string;pdfMetadata?: PdfMetadata;contentType?: string;+ postprocessorsUsed?: string[];},): Promise<void> {try {