Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/scrape/youtube): use new metadata method
mogery2823806e92f7
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 6712d9614..f7468b9ac 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -266,6 +266,10 @@ export async function scrapeURLWithFireEngineChromeCDP(const hasBranding = hasFormatOfType(meta.options.formats, "branding");const hasAudio = hasFormatOfType(meta.options.formats, "audio");const onlyAudio = !!hasAudio && (meta.options.formats?.length ?? 0) === 1;+ const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(+ meta,+ new URL(meta.rewrittenUrl ?? meta.url),+ );const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;const effectiveWait =meta.options.waitFor != null && meta.options.waitFor !== 0@@ -318,7 +322,7 @@ export async function scrapeURLWithFireEngineChromeCDP(},]: []),- ...(hasAudio+ ...(hasAudio || shouldRunYoutubePostprocessor? ([{type: "getCookies",@@ -335,10 +339,7 @@ export async function scrapeURLWithFireEngineChromeCDP(const shouldAllowMedia =hasFormatOfType(meta.options.formats, "branding") ||- youtubePostprocessor.shouldRun(- meta,- new URL(meta.rewrittenUrl ?? meta.url),- );+ shouldRunYoutubePostprocessor;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {@@ -364,7 +365,9 @@ export async function scrapeURLWithFireEngineChromeCDP(meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,...(shouldAllowMedia ? { blockMedia: false } : {}),- ...(onlyAudio ? { skipYouTubeTranscript: true } : {}),+ ...(onlyAudio || shouldRunYoutubePostprocessor+ ? { skipYouTubeTranscript: true }+ : {}),persistentStorage: meta.options.profile? {uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,@@ -465,7 +468,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,timezone: response.timezone,- ...(hasAudio ? { audioCookies } : {}),+ ...(hasAudio || shouldRunYoutubePostprocessor ? { audioCookies } : {}),};});}diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.tsindex 2cea65530..3a9ead810 100644--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts@@ -1,7 +1,133 @@-import type { Meta } from "..";+import { config } from "../../../config";+import type { BrowserCookie, Meta } from "..";import type { Postprocessor } from ".";import type { EngineScrapeResult } from "../engines";+type YouTubeMetadataResponse = {+ thumbnail_image: {+ url: string;+ width?: number | null;+ height?: number | null;+ };+ title: string;+ visibility?: string | null;+ uploaded_by?: {+ name?: string | null;+ url?: string | null;+ } | null;+ uploaded_at?: string | null;+ published_at?: string | null;+ length?: string | null;+ views?: number | null;+ likes?: number | null;+ category?: string | null;+ description?: string | null;+ transcript?: string | null;+};++type YouTubeMetadataRequest = {+ url: string;+ transcript_language: string;+ cookies?: BrowserCookie[];+};++function getTranscriptLanguage(meta: Meta): string {+ const requestedLanguage = meta.options.location?.languages?.[0];+ return requestedLanguage?.split(/[-_]/)[0]?.toLowerCase() || "en";+}++function formatValue(value: string | number | null | undefined): string {+ return value === null || value === undefined ? "" : String(value);+}++function formatUploadedBy(metadata: YouTubeMetadataResponse): string {+ const name = metadata.uploaded_by?.name ?? "";+ const url = metadata.uploaded_by?.url;++ if (name && url) {+ return `[${name}](${url})`;+ }++ return name || url || "";+}++function buildMarkdown(+ metadata: YouTubeMetadataResponse,+ sourceUrl: string,+): string {+ const thumbnailDimensions =+ metadata.thumbnail_image.width && metadata.thumbnail_image.height+ ? ` (${metadata.thumbnail_image.width}x${metadata.thumbnail_image.height})`+ : "";+ const sections = [+ `+# [${metadata.title}](${sourceUrl})++**Visibility**: ${formatValue(metadata.visibility)}+**Uploaded by**: ${formatUploadedBy(metadata)}+**Uploaded at**: ${formatValue(metadata.uploaded_at)}+**Published at**: ${formatValue(metadata.published_at)}+**Length**: ${formatValue(metadata.length)}+**Views**: ${formatValue(metadata.views)}+**Likes**: ${formatValue(metadata.likes)}+**Category**: ${formatValue(metadata.category)}`,+ `## Description++\`\`\`+${formatValue(metadata.description)}+\`\`\``,+ ];++ if (metadata.transcript) {+ sections.push(`## Transcript++${metadata.transcript}`);+ }++ return sections.join("\n\n");+}++async function getYouTubeMetadata(+ meta: Meta,+ engineResult: EngineScrapeResult,+): Promise<YouTubeMetadataResponse> {+ const cookies = meta.audioCookies ?? engineResult.audioCookies;+ const requestBody: YouTubeMetadataRequest = {+ url: engineResult.url,+ transcript_language: getTranscriptLanguage(meta),+ ...(cookies && cookies.length > 0 ? { cookies } : {}),+ };++ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/metadata`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(requestBody),+ });++ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ throw new Error(`YouTube metadata extraction failed: ${error.detail}`);+ }++ const data = (await response+ .json()+ .catch(() => null)) as YouTubeMetadataResponse | null;+ if (+ !data ||+ typeof data.title !== "string" ||+ !data.thumbnail_image ||+ typeof data.thumbnail_image.url !== "string"+ ) {+ throw new Error(+ "YouTube metadata extraction failed: avgrab service returned an invalid response",+ );+ }++ return data;+}+export const youtubePostprocessor: Postprocessor = {name: "youtube",shouldRun: (_meta: Meta, url: URL, postProcessorsUsed?: string[]) => {@@ -21,76 +147,17 @@ export const youtubePostprocessor: Postprocessor = {}},run: async (meta: Meta, engineResult: EngineScrapeResult) => {- let initialData;- try {- initialData = JSON.parse(- engineResult.html- .split("var ytInitialPlayerResponse = ")[1]- .split(";var meta =")[0],- );- } catch (e) {- meta.logger.warn("Failed to parse YouTube initial data");+ if (meta.options.lockdown) {return engineResult;}- const largestThumbnail =- initialData.videoDetails.thumbnail.thumbnails.slice(-1)[0];- const lengthSeconds = parseFloat(initialData.videoDetails.lengthSeconds);- const lengthTrueSeconds = lengthSeconds % 60;- const lengthMinutes = Math.floor(lengthSeconds / 60) % 60;- const lengthHours = Math.floor(lengthSeconds / 3600);-- const endscreen = (- initialData.endscreen?.endscreenRenderer?.elements || []- ).filter(x => x.endscreenElementRenderer?.style === "VIDEO");-- let preferredCaptionMarkdown = "";-- if (engineResult.youtubeTranscriptContent) {- const initialSegments =- engineResult.youtubeTranscriptContent?.actions?.[0]- ?.updateEngagementPanelAction?.content?.transcriptRenderer?.content- ?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer- ?.initialSegments ?? [];- const transcriptText = (- Array.isArray(initialSegments) ? initialSegments : []- )- .map(x => x?.transcriptSegmentRenderer?.snippet?.runs?.[0]?.text)- .filter(Boolean)- .join(" ");- preferredCaptionMarkdown = `## Transcript--${transcriptText}-`;+ if (!config.AVGRAB_SERVICE_URL) {+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");+ return engineResult;}- const markdown = `\--# [${initialData.videoDetails.title}](${initialData.microformat.playerMicroformatRenderer.canonicalUrl})--**Visibility**: ${initialData.videoDetails.isPrivate ? "Private" : initialData.microformat.playerMicroformatRenderer.isUnlisted ? "Unlisted" : "Public"}-**Uploaded by**: [${initialData.videoDetails.author}](${initialData.microformat.playerMicroformatRenderer.ownerProfileUrl})-**Uploaded at**: ${initialData.microformat.playerMicroformatRenderer.uploadDate}-**Published at**: ${initialData.microformat.playerMicroformatRenderer.publishDate}-**Length**: ${lengthHours > 0 ? `${lengthHours.toString().padStart(2, "0")}:` : ""}${lengthMinutes.toString().padStart(2, "0")}:${lengthTrueSeconds.toString().padStart(2, "0")}-**Views**: ${initialData.videoDetails.viewCount}-**Likes**: ${initialData.microformat.playerMicroformatRenderer.likeCount}-**Category**: ${initialData.microformat.playerMicroformatRenderer.category}--## Description--\`\`\`-${initialData.videoDetails.shortDescription}-\`\`\`--${preferredCaptionMarkdown ? preferredCaptionMarkdown + "\n\n" : ""}\-${- endscreen.length > 0- ? `## Endscreen--${endscreen.map(element => `- [${element.endscreenElementRenderer.title.simpleText}](${new URL(element.endscreenElementRenderer.endpoint.commandMetadata.webCommandMetadata.url, engineResult.url).toString()})`).join("\n")}`- : ""-}`;+ const metadata = await getYouTubeMetadata(meta, engineResult);+ const markdown = buildMarkdown(metadata, engineResult.url);return {...engineResult,