Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add generic video discovery results (#3764)

Abimael Martell87cb6d72b930
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 22ac101ba..1c62a4c8f 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1197,6 +1197,7 @@ export type Document = {
screenshot?: string;
audio?: string;
video?: string;
+ videos?: VideoItem[];
extract?: any;
json?: any;
summary?: string;
@@ -1300,6 +1301,22 @@ export type Document = {
};
};
+export type VideoItem = {
+ url: string;
+ sourceURL: string;
+ source: string;
+ kind?: string;
+ provider?: string;
+ title?: string;
+ thumbnail?: string;
+ description?: string;
+ duration?: string;
+ mimeType?: string;
+ width?: number;
+ height?: number;
+ metadata?: Record<string, unknown>;
+};
+
export type ErrorResponse = {
success: false;
code?: ErrorCodes;
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
index c7d847767..0a93a024e 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
@@ -1,4 +1,4 @@
-import { fetchVideo } from "../video";
+import { fetchVideo, resetVideoTransformerCacheForTests } from "../video";
import { config } from "../../../../config";
describe("fetchVideo", () => {
@@ -8,6 +8,7 @@ describe("fetchVideo", () => {
afterEach(() => {
global.fetch = originalFetch;
config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;
+ resetVideoTransformerCacheForTests();
jest.clearAllMocks();
});
@@ -16,7 +17,7 @@ describe("fetchVideo", () => {
if (url.endsWith("/supported-urls")) {
return {
ok: true,
- json: async () => ({ regex: "https://example\\.com/video" }),
+ json: async () => ({ regex: "https://www\\.youtube\\.com/watch" }),
};
}
@@ -31,12 +32,63 @@ describe("fetchVideo", () => {
return fetchSpy;
}
+ it("returns generic videos without downloading or requiring a supported provider URL", async () => {
+ const video = {
+ url: "https://cdn.example.com/product.mp4",
+ sourceURL: "https://example.com/product",
+ source: "metadata",
+ kind: "file",
+ provider: "example.com",
+ title: "Product clip",
+ thumbnail: "https://cdn.example.com/poster.jpg",
+ mimeType: "video/mp4",
+ };
+ const fetchSpy = jest.fn(async (url: string) => {
+ if (url.endsWith("/videos")) {
+ return {
+ ok: true,
+ json: async () => ({ videos: [video] }),
+ };
+ }
+
+ throw new Error(`Unexpected request: ${url}`);
+ });
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+
+ const meta: any = {
+ url: "https://example.com/product",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = { rawHtml: "<html></html>" };
+
+ await fetchVideo(meta, document);
+
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ expect(fetchSpy).toHaveBeenCalledWith(
+ "https://avgrab.example/videos",
+ expect.objectContaining({
+ method: "POST",
+ body: JSON.stringify({
+ url: "https://example.com/product",
+ html: "<html></html>",
+ }),
+ }),
+ );
+ expect(document.videos).toEqual([video]);
+ expect(document.video).toBeUndefined();
+ });
+
it("does not issue any fetch when lockdown is true, even if video format is requested", async () => {
const fetchSpy = jest.fn();
global.fetch = fetchSpy as any;
const meta: any = {
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
options: {
lockdown: true,
formats: [{ type: "video" }],
@@ -57,7 +109,7 @@ describe("fetchVideo", () => {
global.fetch = fetchSpy as any;
const meta: any = {
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
options: {
lockdown: false,
formats: [{ type: "markdown" }],
@@ -109,7 +161,7 @@ describe("fetchVideo", () => {
];
const meta: any = {
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
audioCookies: cookies,
options: {
lockdown: false,
@@ -128,17 +180,98 @@ describe("fetchVideo", () => {
const body = downloadCall![1]?.body;
expect(typeof body).toBe("string");
expect(JSON.parse(body as string)).toEqual({
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
cookies,
});
expect(document.video).toBe("https://storage.example/video.mp4");
});
+ it("skips generic discovery for YouTube URLs and preserves legacy video output", async () => {
+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {
+ if (url.endsWith("/videos")) {
+ throw new Error("Generic discovery should not run for YouTube URLs");
+ }
+ if (url.endsWith("/supported-urls")) {
+ return {
+ ok: true,
+ json: async () => ({ regex: "https://www\\.youtube\\.com/watch" }),
+ };
+ }
+
+ return {
+ ok: true,
+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),
+ };
+ });
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+
+ const meta: any = {
+ url: "https://www.youtube.com/watch?v=abc123",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchVideo(meta, document);
+
+ expect(fetchSpy).toHaveBeenCalledTimes(2);
+ expect(fetchSpy).not.toHaveBeenCalledWith(
+ "https://avgrab.example/videos",
+ expect.anything(),
+ );
+ expect(document.videos).toBeUndefined();
+ expect(document.video).toBe("https://storage.example/video.mp4");
+ });
+
+ it("falls back to legacy download when generic discovery request fails", async () => {
+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {
+ if (url.endsWith("/videos")) {
+ throw new Error("connection reset");
+ }
+ if (url.endsWith("/supported-urls")) {
+ return {
+ ok: true,
+ json: async () => ({ regex: "https://example\\.com/video" }),
+ };
+ }
+
+ return {
+ ok: true,
+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),
+ };
+ });
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+
+ const meta: any = {
+ url: "https://example.com/video",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchVideo(meta, document);
+
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "Generic video discovery failed",
+ { detail: "connection reset" },
+ );
+ expect(document.videos).toBeUndefined();
+ expect(document.video).toBe("https://storage.example/video.mp4");
+ });
+
it("omits cookies from the avgrab request when no video cookies are available", async () => {
const fetchSpy = mockSuccessfulAvgrab();
const meta: any = {
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
options: {
lockdown: false,
formats: [{ type: "video" }],
@@ -156,11 +289,11 @@ describe("fetchVideo", () => {
const body = downloadCall![1]?.body;
expect(typeof body).toBe("string");
expect(JSON.parse(body as string)).toEqual({
- url: "https://example.com/video",
+ url: "https://www.youtube.com/watch?v=abc123",
});
});
- it("rejects unsupported URLs", async () => {
+ it("leaves video fields empty when no generic videos are found and the URL is not supported by legacy download", async () => {
mockSuccessfulAvgrab();
const meta: any = {
@@ -173,6 +306,8 @@ describe("fetchVideo", () => {
};
const document: any = {};
- await expect(fetchVideo(meta, document)).rejects.toThrow(/video/i);
+ await expect(fetchVideo(meta, document)).resolves.toBe(document);
+ expect(document.video).toBeUndefined();
+ expect(document.videos).toBeUndefined();
});
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 44481a40c..1a9b35d82 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -500,7 +500,14 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasVideo = hasFormatOfType(meta.options.formats, "video");
if (!hasVideo && document.video !== undefined) {
delete document.video;
- } else if (hasVideo && document.video === undefined) {
+ }
+ if (!hasVideo && document.videos !== undefined) {
+ delete document.videos;
+ } else if (
+ hasVideo &&
+ document.video === undefined &&
+ document.videos === undefined
+ ) {
meta.logger.warn(
"Request had format: video, but there was no video field in the result.",
);
diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.ts
index 5a61b59e0..63880558c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts
@@ -1,13 +1,17 @@
import { Meta } from "..";
-import { Document } from "../../../controllers/v2/types";
+import { Document, VideoItem } from "../../../controllers/v2/types";
import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
-import { VideoUnsupportedUrlError } from "../error";
let cachedUrlRegex: RegExp | null = null;
let cacheTimestamp = 0;
const CACHE_TTL_MS = 5 * 60 * 1000;
+export function resetVideoTransformerCacheForTests() {
+ cachedUrlRegex = null;
+ cacheTimestamp = 0;
+}
+
async function getSupportedUrlRegex(): Promise<RegExp> {
if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
return cachedUrlRegex;
@@ -34,31 +38,125 @@ async function getSupportedUrlRegex(): Promise<RegExp> {
return cachedUrlRegex;
}
-export async function fetchVideo(
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function optionalString(value: unknown): string | undefined {
+ return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function optionalNumber(value: unknown): number | undefined {
+ return typeof value === "number" && Number.isFinite(value)
+ ? value
+ : undefined;
+}
+
+function normalizeVideoItem(value: unknown): VideoItem | null {
+ if (!isRecord(value)) {
+ return null;
+ }
+
+ const url = optionalString(value.url);
+ const sourceURL = optionalString(value.sourceURL);
+ const source = optionalString(value.source);
+ if (!url || !sourceURL || !source) {
+ return null;
+ }
+
+ return {
+ url,
+ sourceURL,
+ source,
+ kind: optionalString(value.kind),
+ provider: optionalString(value.provider),
+ title: optionalString(value.title),
+ thumbnail: optionalString(value.thumbnail),
+ description: optionalString(value.description),
+ duration: optionalString(value.duration),
+ mimeType: optionalString(value.mimeType),
+ width: optionalNumber(value.width),
+ height: optionalNumber(value.height),
+ metadata: isRecord(value.metadata) ? value.metadata : undefined,
+ };
+}
+
+async function fetchGenericVideos(
meta: Meta,
document: Document,
-): Promise<Document> {
- if (!hasFormatOfType(meta.options.formats, "video")) {
- return document;
+): Promise<VideoItem[]> {
+ const requestBody = {
+ url: meta.rewrittenUrl ?? meta.url,
+ ...(document.rawHtml || document.html
+ ? { html: document.rawHtml ?? document.html }
+ : {}),
+ };
+
+ let response: Response;
+ try {
+ response = await fetch(`${config.AVGRAB_SERVICE_URL}/videos`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(requestBody),
+ });
+ } catch (error) {
+ meta.logger.warn("Generic video discovery failed", {
+ detail: error instanceof Error ? error.message : String(error),
+ });
+ return [];
}
- // Lockdown forbids outbound requests that touch the target URL. avgrab
- // fetches the source on our behalf, so skip it here.
- if (meta.options.lockdown) {
- return document;
+ if (response.status === 404) {
+ meta.logger.debug("avgrab /videos endpoint is unavailable");
+ return [];
}
- if (!config.AVGRAB_SERVICE_URL) {
- meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
- document.warning =
- "Video format is not available (service not configured)." +
- (document.warning ? " " + document.warning : "");
- return document;
+ if (!response.ok) {
+ const error = await response
+ .json()
+ .catch(() => ({ detail: "Unknown error" }));
+ meta.logger.warn("Generic video discovery failed", {
+ detail: isRecord(error) ? error.detail : undefined,
+ });
+ return [];
+ }
+
+ const data = await response.json().catch(() => null);
+ if (!isRecord(data) || !Array.isArray(data.videos)) {
+ meta.logger.warn("Generic video discovery returned an invalid response");
+ return [];
+ }
+
+ return data.videos.flatMap(item => {
+ const normalized = normalizeVideoItem(item);
+ return normalized ? [normalized] : [];
+ });
+}
+
+function shouldTryLegacyDownload(videos: VideoItem[]): boolean {
+ return !videos.some(
+ video => video.kind !== "page" || video.source !== "provider",
+ );
+}
+
+function isYouTubeURL(url: string): boolean {
+ try {
+ const hostname = new URL(url).hostname;
+ return (
+ hostname === "youtube.com" ||
+ hostname.endsWith(".youtube.com") ||
+ hostname === "youtu.be"
+ );
+ } catch {
+ return false;
}
+}
+async function fetchLegacyVideoIfSupported(meta: Meta, document: Document) {
const urlRegex = await getSupportedUrlRegex();
+
if (!urlRegex.test(meta.url)) {
- throw new VideoUnsupportedUrlError();
+ return;
}
const requestBody = {
@@ -90,5 +188,53 @@ export async function fetchVideo(
}
document.video = data.public_url;
+}
+
+export async function fetchVideo(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (!hasFormatOfType(meta.options.formats, "video")) {
+ return document;
+ }
+
+ // Lockdown forbids outbound requests that touch the target URL. avgrab
+ // fetches the source on our behalf, so skip it here.
+ if (meta.options.lockdown) {
+ return document;
+ }
+
+ if (!config.AVGRAB_SERVICE_URL) {
+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
+ document.warning =
+ "Video format is not available (service not configured)." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
+ if (isYouTubeURL(meta.url)) {
+ await fetchLegacyVideoIfSupported(meta, document);
+ return document;
+ }
+
+ const videos = await fetchGenericVideos(meta, document);
+ if (videos.length > 0) {
+ document.videos = videos;
+ }
+
+ if (!shouldTryLegacyDownload(videos)) {
+ return document;
+ }
+
+ try {
+ await fetchLegacyVideoIfSupported(meta, document);
+ } catch (error) {
+ if (videos.length > 0) {
+ meta.logger.warn("Skipping legacy video download", { error });
+ return document;
+ }
+ throw error;
+ }
+
return document;
}