Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): support YouTube live postprocessing

mogery181c341a0e80
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts
new file mode 100644
index 000000000..9e70415b4
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts
@@ -0,0 +1,48 @@
+import { youtubePostprocessor } from "../youtube";
+
+describe("youtubePostprocessor.shouldRun", () => {
+ const meta = {} as any;
+
+ it("runs for YouTube live video URLs", () => {
+ expect(
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL("https://www.youtube.com/live/H4fUJQCIV5E"),
+ ),
+ ).toBe(true);
+ });
+
+ it("keeps existing YouTube video URL support", () => {
+ expect(
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL("https://www.youtube.com/watch?v=H4fUJQCIV5E"),
+ ),
+ ).toBe(true);
+ expect(
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL("https://youtu.be/H4fUJQCIV5E"),
+ ),
+ ).toBe(true);
+ });
+
+ it("does not run for non-video YouTube paths or already processed URLs", () => {
+ expect(
+ youtubePostprocessor.shouldRun(meta, new URL("https://www.youtube.com/")),
+ ).toBe(false);
+ expect(
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL("https://www.youtube.com/live/"),
+ ),
+ ).toBe(false);
+ expect(
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL("https://www.youtube.com/live/H4fUJQCIV5E"),
+ ["youtube"],
+ ),
+ ).toBe(false);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
index 3a9ead810..e05ffe442 100644
--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
@@ -51,6 +51,15 @@ function formatUploadedBy(metadata: YouTubeMetadataResponse): string {
return name || url || "";
}
+function isYouTubeVideoPath(url: URL): boolean {
+ if (url.pathname === "/watch" && !!url.searchParams.get("v")) {
+ return true;
+ }
+
+ const pathParts = url.pathname.split("/").filter(Boolean);
+ return pathParts.length === 2 && pathParts[0] === "live";
+}
+
function buildMarkdown(
metadata: YouTubeMetadataResponse,
sourceUrl: string,
@@ -139,7 +148,7 @@ export const youtubePostprocessor: Postprocessor = {
url.hostname.endsWith(".youtube.com") ||
url.hostname === "youtube.com"
) {
- return url.pathname === "/watch" && !!url.searchParams.get("v");
+ return isYouTubeVideoPath(url);
} else if (url.hostname === "youtu.be") {
return url.pathname !== "/";
} else {