Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add generic video discovery results (#3764)
Abimael Martell87cb6d72b930
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 22ac101ba..1c62a4c8f 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1197,6 +1197,7 @@ export type Document = {screenshot?: string;audio?: string;video?: string;+ videos?: VideoItem[];extract?: any;json?: any;summary?: string;@@ -1300,6 +1301,22 @@ export type Document = {};};+export type VideoItem = {+ url: string;+ sourceURL: string;+ source: string;+ kind?: string;+ provider?: string;+ title?: string;+ thumbnail?: string;+ description?: string;+ duration?: string;+ mimeType?: string;+ width?: number;+ height?: number;+ metadata?: Record<string, unknown>;+};+export type ErrorResponse = {success: false;code?: ErrorCodes;diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.tsindex c7d847767..0a93a024e 100644--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts@@ -1,4 +1,4 @@-import { fetchVideo } from "../video";+import { fetchVideo, resetVideoTransformerCacheForTests } from "../video";import { config } from "../../../../config";describe("fetchVideo", () => {@@ -8,6 +8,7 @@ describe("fetchVideo", () => {afterEach(() => {global.fetch = originalFetch;config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;+ resetVideoTransformerCacheForTests();jest.clearAllMocks();});@@ -16,7 +17,7 @@ describe("fetchVideo", () => {if (url.endsWith("/supported-urls")) {return {ok: true,- json: async () => ({ regex: "https://example\\.com/video" }),+ json: async () => ({ regex: "https://www\\.youtube\\.com/watch" }),};}@@ -31,12 +32,63 @@ describe("fetchVideo", () => {return fetchSpy;}+ it("returns generic videos without downloading or requiring a supported provider URL", async () => {+ const video = {+ url: "https://cdn.example.com/product.mp4",+ sourceURL: "https://example.com/product",+ source: "metadata",+ kind: "file",+ provider: "example.com",+ title: "Product clip",+ thumbnail: "https://cdn.example.com/poster.jpg",+ mimeType: "video/mp4",+ };+ const fetchSpy = jest.fn(async (url: string) => {+ if (url.endsWith("/videos")) {+ return {+ ok: true,+ json: async () => ({ videos: [video] }),+ };+ }++ throw new Error(`Unexpected request: ${url}`);+ });+ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";++ const meta: any = {+ url: "https://example.com/product",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = { rawHtml: "<html></html>" };++ await fetchVideo(meta, document);++ expect(fetchSpy).toHaveBeenCalledTimes(1);+ expect(fetchSpy).toHaveBeenCalledWith(+ "https://avgrab.example/videos",+ expect.objectContaining({+ method: "POST",+ body: JSON.stringify({+ url: "https://example.com/product",+ html: "<html></html>",+ }),+ }),+ );+ expect(document.videos).toEqual([video]);+ expect(document.video).toBeUndefined();+ });+it("does not issue any fetch when lockdown is true, even if video format is requested", async () => {const fetchSpy = jest.fn();global.fetch = fetchSpy as any;const meta: any = {- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",options: {lockdown: true,formats: [{ type: "video" }],@@ -57,7 +109,7 @@ describe("fetchVideo", () => {global.fetch = fetchSpy as any;const meta: any = {- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",options: {lockdown: false,formats: [{ type: "markdown" }],@@ -109,7 +161,7 @@ describe("fetchVideo", () => {];const meta: any = {- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",audioCookies: cookies,options: {lockdown: false,@@ -128,17 +180,98 @@ describe("fetchVideo", () => {const body = downloadCall![1]?.body;expect(typeof body).toBe("string");expect(JSON.parse(body as string)).toEqual({- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",cookies,});expect(document.video).toBe("https://storage.example/video.mp4");});+ it("skips generic discovery for YouTube URLs and preserves legacy video output", async () => {+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {+ if (url.endsWith("/videos")) {+ throw new Error("Generic discovery should not run for YouTube URLs");+ }+ if (url.endsWith("/supported-urls")) {+ return {+ ok: true,+ json: async () => ({ regex: "https://www\\.youtube\\.com/watch" }),+ };+ }++ return {+ ok: true,+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),+ };+ });+ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";++ const meta: any = {+ url: "https://www.youtube.com/watch?v=abc123",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchVideo(meta, document);++ expect(fetchSpy).toHaveBeenCalledTimes(2);+ expect(fetchSpy).not.toHaveBeenCalledWith(+ "https://avgrab.example/videos",+ expect.anything(),+ );+ expect(document.videos).toBeUndefined();+ expect(document.video).toBe("https://storage.example/video.mp4");+ });++ it("falls back to legacy download when generic discovery request fails", async () => {+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {+ if (url.endsWith("/videos")) {+ throw new Error("connection reset");+ }+ if (url.endsWith("/supported-urls")) {+ return {+ ok: true,+ json: async () => ({ regex: "https://example\\.com/video" }),+ };+ }++ return {+ ok: true,+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),+ };+ });+ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";++ const meta: any = {+ url: "https://example.com/video",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchVideo(meta, document);++ expect(meta.logger.warn).toHaveBeenCalledWith(+ "Generic video discovery failed",+ { detail: "connection reset" },+ );+ expect(document.videos).toBeUndefined();+ expect(document.video).toBe("https://storage.example/video.mp4");+ });+it("omits cookies from the avgrab request when no video cookies are available", async () => {const fetchSpy = mockSuccessfulAvgrab();const meta: any = {- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",options: {lockdown: false,formats: [{ type: "video" }],@@ -156,11 +289,11 @@ describe("fetchVideo", () => {const body = downloadCall![1]?.body;expect(typeof body).toBe("string");expect(JSON.parse(body as string)).toEqual({- url: "https://example.com/video",+ url: "https://www.youtube.com/watch?v=abc123",});});- it("rejects unsupported URLs", async () => {+ it("leaves video fields empty when no generic videos are found and the URL is not supported by legacy download", async () => {mockSuccessfulAvgrab();const meta: any = {@@ -173,6 +306,8 @@ describe("fetchVideo", () => {};const document: any = {};- await expect(fetchVideo(meta, document)).rejects.toThrow(/video/i);+ await expect(fetchVideo(meta, document)).resolves.toBe(document);+ expect(document.video).toBeUndefined();+ expect(document.videos).toBeUndefined();});});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 44481a40c..1a9b35d82 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -500,7 +500,14 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasVideo = hasFormatOfType(meta.options.formats, "video");if (!hasVideo && document.video !== undefined) {delete document.video;- } else if (hasVideo && document.video === undefined) {+ }+ if (!hasVideo && document.videos !== undefined) {+ delete document.videos;+ } else if (+ hasVideo &&+ document.video === undefined &&+ document.videos === undefined+ ) {meta.logger.warn("Request had format: video, but there was no video field in the result.",);diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.tsindex 5a61b59e0..63880558c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts@@ -1,13 +1,17 @@import { Meta } from "..";-import { Document } from "../../../controllers/v2/types";+import { Document, VideoItem } from "../../../controllers/v2/types";import { config } from "../../../config";import { hasFormatOfType } from "../../../lib/format-utils";-import { VideoUnsupportedUrlError } from "../error";let cachedUrlRegex: RegExp | null = null;let cacheTimestamp = 0;const CACHE_TTL_MS = 5 * 60 * 1000;+export function resetVideoTransformerCacheForTests() {+ cachedUrlRegex = null;+ cacheTimestamp = 0;+}+async function getSupportedUrlRegex(): Promise<RegExp> {if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {return cachedUrlRegex;@@ -34,31 +38,125 @@ async function getSupportedUrlRegex(): Promise<RegExp> {return cachedUrlRegex;}-export async function fetchVideo(+function isRecord(value: unknown): value is Record<string, unknown> {+ return value !== null && typeof value === "object" && !Array.isArray(value);+}++function optionalString(value: unknown): string | undefined {+ return typeof value === "string" && value.length > 0 ? value : undefined;+}++function optionalNumber(value: unknown): number | undefined {+ return typeof value === "number" && Number.isFinite(value)+ ? value+ : undefined;+}++function normalizeVideoItem(value: unknown): VideoItem | null {+ if (!isRecord(value)) {+ return null;+ }++ const url = optionalString(value.url);+ const sourceURL = optionalString(value.sourceURL);+ const source = optionalString(value.source);+ if (!url || !sourceURL || !source) {+ return null;+ }++ return {+ url,+ sourceURL,+ source,+ kind: optionalString(value.kind),+ provider: optionalString(value.provider),+ title: optionalString(value.title),+ thumbnail: optionalString(value.thumbnail),+ description: optionalString(value.description),+ duration: optionalString(value.duration),+ mimeType: optionalString(value.mimeType),+ width: optionalNumber(value.width),+ height: optionalNumber(value.height),+ metadata: isRecord(value.metadata) ? value.metadata : undefined,+ };+}++async function fetchGenericVideos(meta: Meta,document: Document,-): Promise<Document> {- if (!hasFormatOfType(meta.options.formats, "video")) {- return document;+): Promise<VideoItem[]> {+ const requestBody = {+ url: meta.rewrittenUrl ?? meta.url,+ ...(document.rawHtml || document.html+ ? { html: document.rawHtml ?? document.html }+ : {}),+ };++ let response: Response;+ try {+ response = await fetch(`${config.AVGRAB_SERVICE_URL}/videos`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(requestBody),+ });+ } catch (error) {+ meta.logger.warn("Generic video discovery failed", {+ detail: error instanceof Error ? error.message : String(error),+ });+ return [];}- // Lockdown forbids outbound requests that touch the target URL. avgrab- // fetches the source on our behalf, so skip it here.- if (meta.options.lockdown) {- return document;+ if (response.status === 404) {+ meta.logger.debug("avgrab /videos endpoint is unavailable");+ return [];}- if (!config.AVGRAB_SERVICE_URL) {- meta.logger.warn("AVGRAB_SERVICE_URL is not configured");- document.warning =- "Video format is not available (service not configured)." +- (document.warning ? " " + document.warning : "");- return document;+ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ meta.logger.warn("Generic video discovery failed", {+ detail: isRecord(error) ? error.detail : undefined,+ });+ return [];+ }++ const data = await response.json().catch(() => null);+ if (!isRecord(data) || !Array.isArray(data.videos)) {+ meta.logger.warn("Generic video discovery returned an invalid response");+ return [];+ }++ return data.videos.flatMap(item => {+ const normalized = normalizeVideoItem(item);+ return normalized ? [normalized] : [];+ });+}++function shouldTryLegacyDownload(videos: VideoItem[]): boolean {+ return !videos.some(+ video => video.kind !== "page" || video.source !== "provider",+ );+}++function isYouTubeURL(url: string): boolean {+ try {+ const hostname = new URL(url).hostname;+ return (+ hostname === "youtube.com" ||+ hostname.endsWith(".youtube.com") ||+ hostname === "youtu.be"+ );+ } catch {+ return false;}+}+async function fetchLegacyVideoIfSupported(meta: Meta, document: Document) {const urlRegex = await getSupportedUrlRegex();+if (!urlRegex.test(meta.url)) {- throw new VideoUnsupportedUrlError();+ return;}const requestBody = {@@ -90,5 +188,53 @@ export async function fetchVideo(}document.video = data.public_url;+}++export async function fetchVideo(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (!hasFormatOfType(meta.options.formats, "video")) {+ return document;+ }++ // Lockdown forbids outbound requests that touch the target URL. avgrab+ // fetches the source on our behalf, so skip it here.+ if (meta.options.lockdown) {+ return document;+ }++ if (!config.AVGRAB_SERVICE_URL) {+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");+ document.warning =+ "Video format is not available (service not configured)." ++ (document.warning ? " " + document.warning : "");+ return document;+ }++ if (isYouTubeURL(meta.url)) {+ await fetchLegacyVideoIfSupported(meta, document);+ return document;+ }++ const videos = await fetchGenericVideos(meta, document);+ if (videos.length > 0) {+ document.videos = videos;+ }++ if (!shouldTryLegacyDownload(videos)) {+ return document;+ }++ try {+ await fetchLegacyVideoIfSupported(meta, document);+ } catch (error) {+ if (videos.length > 0) {+ meta.logger.warn("Skipping legacy video download", { error });+ return document;+ }+ throw error;+ }+return document;}