Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/scrape/youtube): use new metadata method

mogery2823806e92f7
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 6712d9614..f7468b9ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -266,6 +266,10 @@ export async function scrapeURLWithFireEngineChromeCDP(
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
const hasAudio = hasFormatOfType(meta.options.formats, "audio");
const onlyAudio = !!hasAudio && (meta.options.formats?.length ?? 0) === 1;
+ const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(
+ meta,
+ new URL(meta.rewrittenUrl ?? meta.url),
+ );
const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;
const effectiveWait =
meta.options.waitFor != null && meta.options.waitFor !== 0
@@ -318,7 +322,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
},
]
: []),
- ...(hasAudio
+ ...(hasAudio || shouldRunYoutubePostprocessor
? ([
{
type: "getCookies",
@@ -335,10 +339,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
const shouldAllowMedia =
hasFormatOfType(meta.options.formats, "branding") ||
- youtubePostprocessor.shouldRun(
- meta,
- new URL(meta.rewrittenUrl ?? meta.url),
- );
+ shouldRunYoutubePostprocessor;
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
@@ -364,7 +365,9 @@ export async function scrapeURLWithFireEngineChromeCDP(
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
...(shouldAllowMedia ? { blockMedia: false } : {}),
- ...(onlyAudio ? { skipYouTubeTranscript: true } : {}),
+ ...(onlyAudio || shouldRunYoutubePostprocessor
+ ? { skipYouTubeTranscript: true }
+ : {}),
persistentStorage: meta.options.profile
? {
uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,
@@ -465,7 +468,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
timezone: response.timezone,
- ...(hasAudio ? { audioCookies } : {}),
+ ...(hasAudio || shouldRunYoutubePostprocessor ? { audioCookies } : {}),
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
index 2cea65530..3a9ead810 100644
--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
@@ -1,7 +1,133 @@
-import type { Meta } from "..";
+import { config } from "../../../config";
+import type { BrowserCookie, Meta } from "..";
import type { Postprocessor } from ".";
import type { EngineScrapeResult } from "../engines";
+type YouTubeMetadataResponse = {
+ thumbnail_image: {
+ url: string;
+ width?: number | null;
+ height?: number | null;
+ };
+ title: string;
+ visibility?: string | null;
+ uploaded_by?: {
+ name?: string | null;
+ url?: string | null;
+ } | null;
+ uploaded_at?: string | null;
+ published_at?: string | null;
+ length?: string | null;
+ views?: number | null;
+ likes?: number | null;
+ category?: string | null;
+ description?: string | null;
+ transcript?: string | null;
+};
+
+type YouTubeMetadataRequest = {
+ url: string;
+ transcript_language: string;
+ cookies?: BrowserCookie[];
+};
+
+function getTranscriptLanguage(meta: Meta): string {
+ const requestedLanguage = meta.options.location?.languages?.[0];
+ return requestedLanguage?.split(/[-_]/)[0]?.toLowerCase() || "en";
+}
+
+function formatValue(value: string | number | null | undefined): string {
+ return value === null || value === undefined ? "" : String(value);
+}
+
+function formatUploadedBy(metadata: YouTubeMetadataResponse): string {
+ const name = metadata.uploaded_by?.name ?? "";
+ const url = metadata.uploaded_by?.url;
+
+ if (name && url) {
+ return `[${name}](${url})`;
+ }
+
+ return name || url || "";
+}
+
+function buildMarkdown(
+ metadata: YouTubeMetadataResponse,
+ sourceUrl: string,
+): string {
+ const thumbnailDimensions =
+ metadata.thumbnail_image.width && metadata.thumbnail_image.height
+ ? ` (${metadata.thumbnail_image.width}x${metadata.thumbnail_image.height})`
+ : "";
+ const sections = [
+ `![Thumbnail${thumbnailDimensions}](${metadata.thumbnail_image.url})
+# [${metadata.title}](${sourceUrl})
+
+**Visibility**: ${formatValue(metadata.visibility)}
+**Uploaded by**: ${formatUploadedBy(metadata)}
+**Uploaded at**: ${formatValue(metadata.uploaded_at)}
+**Published at**: ${formatValue(metadata.published_at)}
+**Length**: ${formatValue(metadata.length)}
+**Views**: ${formatValue(metadata.views)}
+**Likes**: ${formatValue(metadata.likes)}
+**Category**: ${formatValue(metadata.category)}`,
+ `## Description
+
+\`\`\`
+${formatValue(metadata.description)}
+\`\`\``,
+ ];
+
+ if (metadata.transcript) {
+ sections.push(`## Transcript
+
+${metadata.transcript}`);
+ }
+
+ return sections.join("\n\n");
+}
+
+async function getYouTubeMetadata(
+ meta: Meta,
+ engineResult: EngineScrapeResult,
+): Promise<YouTubeMetadataResponse> {
+ const cookies = meta.audioCookies ?? engineResult.audioCookies;
+ const requestBody: YouTubeMetadataRequest = {
+ url: engineResult.url,
+ transcript_language: getTranscriptLanguage(meta),
+ ...(cookies && cookies.length > 0 ? { cookies } : {}),
+ };
+
+ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/metadata`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(requestBody),
+ });
+
+ if (!response.ok) {
+ const error = await response
+ .json()
+ .catch(() => ({ detail: "Unknown error" }));
+ throw new Error(`YouTube metadata extraction failed: ${error.detail}`);
+ }
+
+ const data = (await response
+ .json()
+ .catch(() => null)) as YouTubeMetadataResponse | null;
+ if (
+ !data ||
+ typeof data.title !== "string" ||
+ !data.thumbnail_image ||
+ typeof data.thumbnail_image.url !== "string"
+ ) {
+ throw new Error(
+ "YouTube metadata extraction failed: avgrab service returned an invalid response",
+ );
+ }
+
+ return data;
+}
+
export const youtubePostprocessor: Postprocessor = {
name: "youtube",
shouldRun: (_meta: Meta, url: URL, postProcessorsUsed?: string[]) => {
@@ -21,76 +147,17 @@ export const youtubePostprocessor: Postprocessor = {
}
},
run: async (meta: Meta, engineResult: EngineScrapeResult) => {
- let initialData;
- try {
- initialData = JSON.parse(
- engineResult.html
- .split("var ytInitialPlayerResponse = ")[1]
- .split(";var meta =")[0],
- );
- } catch (e) {
- meta.logger.warn("Failed to parse YouTube initial data");
+ if (meta.options.lockdown) {
return engineResult;
}
- const largestThumbnail =
- initialData.videoDetails.thumbnail.thumbnails.slice(-1)[0];
- const lengthSeconds = parseFloat(initialData.videoDetails.lengthSeconds);
- const lengthTrueSeconds = lengthSeconds % 60;
- const lengthMinutes = Math.floor(lengthSeconds / 60) % 60;
- const lengthHours = Math.floor(lengthSeconds / 3600);
-
- const endscreen = (
- initialData.endscreen?.endscreenRenderer?.elements || []
- ).filter(x => x.endscreenElementRenderer?.style === "VIDEO");
-
- let preferredCaptionMarkdown = "";
-
- if (engineResult.youtubeTranscriptContent) {
- const initialSegments =
- engineResult.youtubeTranscriptContent?.actions?.[0]
- ?.updateEngagementPanelAction?.content?.transcriptRenderer?.content
- ?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer
- ?.initialSegments ?? [];
- const transcriptText = (
- Array.isArray(initialSegments) ? initialSegments : []
- )
- .map(x => x?.transcriptSegmentRenderer?.snippet?.runs?.[0]?.text)
- .filter(Boolean)
- .join(" ");
- preferredCaptionMarkdown = `## Transcript
-
-${transcriptText}
-`;
+ if (!config.AVGRAB_SERVICE_URL) {
+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
+ return engineResult;
}
- const markdown = `\
-![Thumbnail (${largestThumbnail.width}x${largestThumbnail.height})](${largestThumbnail.url})
-# [${initialData.videoDetails.title}](${initialData.microformat.playerMicroformatRenderer.canonicalUrl})
-
-**Visibility**: ${initialData.videoDetails.isPrivate ? "Private" : initialData.microformat.playerMicroformatRenderer.isUnlisted ? "Unlisted" : "Public"}
-**Uploaded by**: [${initialData.videoDetails.author}](${initialData.microformat.playerMicroformatRenderer.ownerProfileUrl})
-**Uploaded at**: ${initialData.microformat.playerMicroformatRenderer.uploadDate}
-**Published at**: ${initialData.microformat.playerMicroformatRenderer.publishDate}
-**Length**: ${lengthHours > 0 ? `${lengthHours.toString().padStart(2, "0")}:` : ""}${lengthMinutes.toString().padStart(2, "0")}:${lengthTrueSeconds.toString().padStart(2, "0")}
-**Views**: ${initialData.videoDetails.viewCount}
-**Likes**: ${initialData.microformat.playerMicroformatRenderer.likeCount}
-**Category**: ${initialData.microformat.playerMicroformatRenderer.category}
-
-## Description
-
-\`\`\`
-${initialData.videoDetails.shortDescription}
-\`\`\`
-
-${preferredCaptionMarkdown ? preferredCaptionMarkdown + "\n\n" : ""}\
-${
- endscreen.length > 0
- ? `## Endscreen
-
-${endscreen.map(element => `- [${element.endscreenElementRenderer.title.simpleText}](${new URL(element.endscreenElementRenderer.endpoint.commandMetadata.webCommandMetadata.url, engineResult.url).toString()})`).join("\n")}`
- : ""
-}`;
+ const metadata = await getYouTubeMetadata(meta, engineResult);
+ const markdown = buildMarkdown(metadata, engineResult.url);
return {
...engineResult,