Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): support YouTube live postprocessing
mogery181c341a0e80
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.tsnew file mode 100644index 000000000..9e70415b4--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts@@ -0,0 +1,48 @@+import { youtubePostprocessor } from "../youtube";++describe("youtubePostprocessor.shouldRun", () => {+ const meta = {} as any;++ it("runs for YouTube live video URLs", () => {+ expect(+ youtubePostprocessor.shouldRun(+ meta,+ new URL("https://www.youtube.com/live/H4fUJQCIV5E"),+ ),+ ).toBe(true);+ });++ it("keeps existing YouTube video URL support", () => {+ expect(+ youtubePostprocessor.shouldRun(+ meta,+ new URL("https://www.youtube.com/watch?v=H4fUJQCIV5E"),+ ),+ ).toBe(true);+ expect(+ youtubePostprocessor.shouldRun(+ meta,+ new URL("https://youtu.be/H4fUJQCIV5E"),+ ),+ ).toBe(true);+ });++ it("does not run for non-video YouTube paths or already processed URLs", () => {+ expect(+ youtubePostprocessor.shouldRun(meta, new URL("https://www.youtube.com/")),+ ).toBe(false);+ expect(+ youtubePostprocessor.shouldRun(+ meta,+ new URL("https://www.youtube.com/live/"),+ ),+ ).toBe(false);+ expect(+ youtubePostprocessor.shouldRun(+ meta,+ new URL("https://www.youtube.com/live/H4fUJQCIV5E"),+ ["youtube"],+ ),+ ).toBe(false);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.tsindex 3a9ead810..e05ffe442 100644--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts@@ -51,6 +51,15 @@ function formatUploadedBy(metadata: YouTubeMetadataResponse): string {return name || url || "";}+function isYouTubeVideoPath(url: URL): boolean {+ if (url.pathname === "/watch" && !!url.searchParams.get("v")) {+ return true;+ }++ const pathParts = url.pathname.split("/").filter(Boolean);+ return pathParts.length === 2 && pathParts[0] === "live";+}+function buildMarkdown(metadata: YouTubeMetadataResponse,sourceUrl: string,@@ -139,7 +148,7 @@ export const youtubePostprocessor: Postprocessor = {url.hostname.endsWith(".youtube.com") ||url.hostname === "youtube.com") {- return url.pathname === "/watch" && !!url.searchParams.get("v");+ return isYouTubeVideoPath(url);} else if (url.hostname === "youtu.be") {return url.pathname !== "/";} else {