Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/scrapeURL): add special support for YouTube watch pages (#2157)

Gergő Móriczebe2d8730876
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 53db121fb..e413fbbc4 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -1297,6 +1297,25 @@ describe("Scrape tests", () => {
scrapeTimeout * 5,
);
});
+
+ describe("YouTube (f-e dependant)", () => {
+ it.concurrent(
+ "scrapes YouTube videos and transcripts",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toContain("Rick Astley");
+ expect(response.markdown).toContain("Never gonna let you down");
+ },
+ scrapeTimeout,
+ );
+ });
}
describe("URL rewriting", () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index dc3ec0e54..683dea09e 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -907,6 +907,7 @@ export type Document = {
cacheState?: "hit" | "miss";
cachedAt?: string;
creditsUsed?: number;
+ postprocessorsUsed?: string[];
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index dc67932cf..998321b7e 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -766,6 +766,7 @@ export type Document = {
cacheState?: "hit" | "miss";
cachedAt?: string;
creditsUsed?: number;
+ postprocessorsUsed?: string[];
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index d0391d77e..edb088d02 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -101,6 +101,7 @@ const successSchema = z.object({
docUrl: z.string().optional(),
usedMobileProxy: z.boolean().optional(),
+ youtubeTranscriptContent: z.any().optional(),
});
export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index b764a5b50..116768348 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -31,6 +31,7 @@ import { getInnerJson } from "@mendable/firecrawl-rs";
import { hasFormatOfType } from "../../../../lib/format-utils";
import { Action } from "../../../../controllers/v1/types";
import { AbortManagerThrownError } from "../../lib/abortManager";
+import { youtubePostprocessor } from "../../postprocessors/youtube";
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
@@ -247,6 +248,12 @@ export async function scrapeURLWithFireEngineChromeCDP(
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ ...(youtubePostprocessor.shouldRun(
+ meta,
+ new URL(meta.rewrittenUrl ?? meta.url),
+ )
+ ? { blockMedia: false }
+ : {}),
};
let response = await performFireEngineScrape(
@@ -314,6 +321,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
: {}),
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ youtubeTranscriptContent: response.youtubeTranscriptContent,
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 876205831..743f744ed 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -50,7 +50,7 @@ export type FireEngineScrapeRequestChromeCDP = {
engine: "chrome-cdp";
skipTlsVerification?: boolean;
actions?: Action[];
- blockMedia?: true; // cannot be false
+ blockMedia?: boolean;
mobile?: boolean;
disableSmartWaitCache?: boolean;
};
@@ -151,6 +151,7 @@ const successSchema = z.object({
docUrl: z.string().optional(),
usedMobileProxy: z.boolean().optional(),
+ youtubeTranscriptContent: z.any().optional(),
});
type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 246e27a7e..affea25d2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -127,6 +127,9 @@ export type EngineScrapeResult = {
contentType?: string;
+ youtubeTranscriptContent?: any;
+ postprocessorsUsed?: string[];
+
proxyUsed: "basic" | "stealth";
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 95eb1d0ae..cfe622261 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -82,6 +82,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
}
: undefined,
contentType: document.metadata.contentType,
+ postprocessorsUsed: document.metadata.postprocessorsUsed,
});
} catch (error) {
meta.logger.error("Failed to save document to index", {
@@ -365,6 +366,8 @@ export async function scrapeURLWithIndex(
created_at: new Date(data[0].created_at),
},
+ postprocessorsUsed: doc.postprocessorsUsed,
+
proxyUsed: doc.proxyUsed ?? "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index c9f6e8a8c..73819e577 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -65,6 +65,7 @@ import {
} from "./lib/abortManager";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { htmlTransform } from "./lib/removeUnwantedElements";
+import { postprocessors } from "./postprocessors";
export type ScrapeUrlResponse =
| {
@@ -612,34 +613,64 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
throw new NoEnginesLeftError(fallbackList.map(x => x.engine));
}
+ let engineResult: EngineScrapeResult = result.result;
+
+ for (const postprocessor of postprocessors) {
+ if (
+ postprocessor.shouldRun(
+ meta,
+ new URL(engineResult.url),
+ engineResult.postprocessorsUsed,
+ )
+ ) {
+ meta.logger.info("Running postprocessor " + postprocessor.name);
+ try {
+ engineResult = await postprocessor.run(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "postprocessors/" + postprocessor.name,
+ }),
+ },
+ engineResult,
+ );
+ } catch (error) {
+ meta.logger.warn("Failed to run postprocessor " + postprocessor.name, {
+ error,
+ });
+ }
+ }
+ }
+
let document: Document = {
- markdown: result.result.markdown,
- rawHtml: result.result.html,
- screenshot: result.result.screenshot,
- actions: result.result.actions,
+ markdown: engineResult.markdown,
+ rawHtml: engineResult.html,
+ screenshot: engineResult.screenshot,
+ actions: engineResult.actions,
metadata: {
sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,
- url: result.result.url,
- statusCode: result.result.statusCode,
- error: result.result.error,
- numPages: result.result.pdfMetadata?.numPages,
- ...(result.result.pdfMetadata?.title
- ? { title: result.result.pdfMetadata.title }
+ url: engineResult.url,
+ statusCode: engineResult.statusCode,
+ error: engineResult.error,
+ numPages: engineResult.pdfMetadata?.numPages,
+ ...(engineResult.pdfMetadata?.title
+ ? { title: engineResult.pdfMetadata.title }
: {}),
- contentType: result.result.contentType,
+ contentType: engineResult.contentType,
proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
...(fallbackList.find(x =>
["index", "index;documents"].includes(x.engine),
)
- ? result.result.cacheInfo
+ ? engineResult.cacheInfo
? {
cacheState: "hit",
- cachedAt: result.result.cacheInfo.created_at.toISOString(),
+ cachedAt: engineResult.cacheInfo.created_at.toISOString(),
}
: {
cacheState: "miss",
}
: {}),
+ postprocessorsUsed: engineResult.postprocessorsUsed,
},
};
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/index.ts b/apps/api/src/scraper/scrapeURL/postprocessors/index.ts
new file mode 100644
index 000000000..e199b2e41
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/index.ts
@@ -0,0 +1,14 @@
+import { Meta } from "..";
+import { EngineScrapeResult } from "../engines";
+import { youtubePostprocessor } from "./youtube";
+
+export interface Postprocessor {
+ name: string;
+ shouldRun: (meta: Meta, url: URL, postProcessorsUsed?: string[]) => boolean;
+ run: (
+ meta: Meta,
+ engineResult: EngineScrapeResult,
+ ) => Promise<EngineScrapeResult>;
+}
+
+export const postprocessors: Postprocessor[] = [youtubePostprocessor];
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
new file mode 100644
index 000000000..2cea65530
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
@@ -0,0 +1,104 @@
+import type { Meta } from "..";
+import type { Postprocessor } from ".";
+import type { EngineScrapeResult } from "../engines";
+
+export const youtubePostprocessor: Postprocessor = {
+ name: "youtube",
+ shouldRun: (_meta: Meta, url: URL, postProcessorsUsed?: string[]) => {
+ if (postProcessorsUsed?.includes("youtube")) {
+ return false;
+ }
+
+ if (
+ url.hostname.endsWith(".youtube.com") ||
+ url.hostname === "youtube.com"
+ ) {
+ return url.pathname === "/watch" && !!url.searchParams.get("v");
+ } else if (url.hostname === "youtu.be") {
+ return url.pathname !== "/";
+ } else {
+ return false;
+ }
+ },
+ run: async (meta: Meta, engineResult: EngineScrapeResult) => {
+ let initialData;
+ try {
+ initialData = JSON.parse(
+ engineResult.html
+ .split("var ytInitialPlayerResponse = ")[1]
+ .split(";var meta =")[0],
+ );
+ } catch (e) {
+ meta.logger.warn("Failed to parse YouTube initial data");
+ return engineResult;
+ }
+
+ const largestThumbnail =
+ initialData.videoDetails.thumbnail.thumbnails.slice(-1)[0];
+ const lengthSeconds = parseFloat(initialData.videoDetails.lengthSeconds);
+ const lengthTrueSeconds = lengthSeconds % 60;
+ const lengthMinutes = Math.floor(lengthSeconds / 60) % 60;
+ const lengthHours = Math.floor(lengthSeconds / 3600);
+
+ const endscreen = (
+ initialData.endscreen?.endscreenRenderer?.elements || []
+ ).filter(x => x.endscreenElementRenderer?.style === "VIDEO");
+
+ let preferredCaptionMarkdown = "";
+
+ if (engineResult.youtubeTranscriptContent) {
+ const initialSegments =
+ engineResult.youtubeTranscriptContent?.actions?.[0]
+ ?.updateEngagementPanelAction?.content?.transcriptRenderer?.content
+ ?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer
+ ?.initialSegments ?? [];
+ const transcriptText = (
+ Array.isArray(initialSegments) ? initialSegments : []
+ )
+ .map(x => x?.transcriptSegmentRenderer?.snippet?.runs?.[0]?.text)
+ .filter(Boolean)
+ .join(" ");
+ preferredCaptionMarkdown = `## Transcript
+
+${transcriptText}
+`;
+ }
+
+ const markdown = `\
+![Thumbnail (${largestThumbnail.width}x${largestThumbnail.height})](${largestThumbnail.url})
+# [${initialData.videoDetails.title}](${initialData.microformat.playerMicroformatRenderer.canonicalUrl})
+
+**Visibility**: ${initialData.videoDetails.isPrivate ? "Private" : initialData.microformat.playerMicroformatRenderer.isUnlisted ? "Unlisted" : "Public"}
+**Uploaded by**: [${initialData.videoDetails.author}](${initialData.microformat.playerMicroformatRenderer.ownerProfileUrl})
+**Uploaded at**: ${initialData.microformat.playerMicroformatRenderer.uploadDate}
+**Published at**: ${initialData.microformat.playerMicroformatRenderer.publishDate}
+**Length**: ${lengthHours > 0 ? `${lengthHours.toString().padStart(2, "0")}:` : ""}${lengthMinutes.toString().padStart(2, "0")}:${lengthTrueSeconds.toString().padStart(2, "0")}
+**Views**: ${initialData.videoDetails.viewCount}
+**Likes**: ${initialData.microformat.playerMicroformatRenderer.likeCount}
+**Category**: ${initialData.microformat.playerMicroformatRenderer.category}
+
+## Description
+
+\`\`\`
+${initialData.videoDetails.shortDescription}
+\`\`\`
+
+${preferredCaptionMarkdown ? preferredCaptionMarkdown + "\n\n" : ""}\
+${
+ endscreen.length > 0
+ ? `## Endscreen
+
+${endscreen.map(element => `- [${element.endscreenElementRenderer.title.simpleText}](${new URL(element.endscreenElementRenderer.endpoint.commandMetadata.webCommandMetadata.url, engineResult.url).toString()})`).join("\n")}`
+ : ""
+}`;
+
+ return {
+ ...engineResult,
+ markdown,
+ postprocessorsUsed: [
+ ...(engineResult.postprocessorsUsed ?? []),
+ "youtube",
+ ],
+ };
+ },
+};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index e45b24a51..c09dd1266 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -1,6 +1,6 @@
import { parseMarkdown } from "../../../lib/html-to-markdown";
import { Meta } from "..";
-import { Document } from "../../../controllers/v1/types";
+import { Document } from "../../../controllers/v2/types";
import { htmlTransform } from "../lib/removeUnwantedElements";
import { extractLinks } from "../lib/extractLinks";
import { extractImages } from "../lib/extractImages";
@@ -72,6 +72,10 @@ async function deriveMarkdownFromHTML(
);
}
+ if (document.metadata.postprocessorsUsed?.includes("youtube")) {
+ return document;
+ }
+
if (document.metadata.contentType?.includes("application/json")) {
if (document.rawHtml === undefined) {
throw new Error(
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index b41814a4f..b9fa1e8f3 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -150,6 +150,7 @@ export async function saveIndexToGCS(
screenshot?: string;
pdfMetadata?: PdfMetadata;
contentType?: string;
+ postprocessorsUsed?: string[];
},
): Promise<void> {
try {