Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(avgrab): get site support from service directly (#3229)
Gergő Móricz22c25eb287d2
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex ba1c03d1a..02ef95fc6 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -2014,7 +2014,7 @@ describe("Attribute formats", () => {describeIf(!TEST_SELF_HOST)("Audio format", () => {it.concurrent(- "should return audio field with signed GCS URL for a YouTube video",+ "should return audio field with signed GCS URL for a supported video URL",async () => {const data = await scrape({@@ -2032,7 +2032,7 @@ describe("Attribute formats", () => {);it.concurrent(- "should reject non-YouTube URL with audio format",+ "should reject unsupported URL with audio format",async () => {const result = await scrapeWithFailure({@@ -2042,7 +2042,7 @@ describe("Attribute formats", () => {identity,);- expect(result.error).toMatch(/audio.*youtube/i);+ expect(result.error).toMatch(/audio/i);},scrapeTimeout,);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c94d24b93..1c936de53 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -27,24 +27,6 @@ import {} from "../../services/webhook/schema";import { BrandingProfile } from "../../types/branding";-function isYouTubeUrl(url: string): boolean {- try {- const parsed = new globalThis.URL(url);- if (- parsed.hostname === "youtube.com" ||- parsed.hostname.endsWith(".youtube.com")- ) {- return parsed.pathname === "/watch" && !!parsed.searchParams.get("v");- }- if (parsed.hostname === "youtu.be") {- return parsed.pathname !== "/";- }- return false;- } catch {- return false;- }-}-// Base URL schema with common validation logicexport const URL = z.preprocess(x => {@@ -832,10 +814,6 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase).refine(waitForRefine, waitForRefineOpts)- .refine(- x => !x.formats?.some(f => f.type === "audio") || isYouTubeUrl(x.url),- "The audio format is only supported for YouTube URLs (youtube.com/watch?v=... or youtu.be/...)",- ).transform(extractTransformRequired);export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 23134ffff..e280ab801 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -29,6 +29,7 @@ import {ScrapeJobCancelledError,ScrapeRetryLimitError,BrandingNotSupportedError,+ AudioUnsupportedUrlError,} from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -59,6 +60,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,+ SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,// Zod errorsBAD_REQUEST: null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex e31cd8369..640e846da 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -24,6 +24,7 @@ export type ErrorCodes =| "SCRAPE_ACTIONS_NOT_SUPPORTED"| "SCRAPE_BRANDING_NOT_SUPPORTED"| "AGENT_INDEX_ONLY"+ | "SCRAPE_AUDIO_UNSUPPORTED_URL"| "CRAWL_DENIAL"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST";diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 1d644449b..f0d6c5143 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -469,6 +469,28 @@ export class AgentIndexOnlyError extends TransportableError {}}+export class AudioUnsupportedUrlError extends TransportableError {+ constructor(message?: string) {+ super(+ "SCRAPE_AUDIO_UNSUPPORTED_URL",+ message ?? "The audio format does not support the provided URL",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new AudioUnsupportedUrlError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class BrandingNotSupportedError extends TransportableError {constructor(message: string) {super("SCRAPE_BRANDING_NOT_SUPPORTED", message);diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsindex 3798fe01c..553c00cd3 100644--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -2,6 +2,37 @@ import { Meta } from "..";import { Document } from "../../../controllers/v2/types";import { config } from "../../../config";import { hasFormatOfType } from "../../../lib/format-utils";+import { AudioUnsupportedUrlError } from "../error";++let cachedUrlRegex: RegExp | null = null;+let cacheTimestamp = 0;+const CACHE_TTL_MS = 5 * 60 * 1000;++async function getSupportedUrlRegex(): Promise<RegExp> {+ if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {+ return cachedUrlRegex;+ }++ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);+ if (!res.ok) {+ throw new Error(+ "Failed to fetch supported URL patterns from audio service",+ );+ }++ const data = await res.json().catch(() => null);+ if (!data || typeof data.regex !== "string") {+ throw new Error("Audio service returned invalid supported URL patterns");+ }++ try {+ cachedUrlRegex = new RegExp(data.regex);+ } catch {+ throw new Error("Audio service returned invalid supported URL patterns");+ }+ cacheTimestamp = Date.now();+ return cachedUrlRegex;+}export async function fetchAudio(meta: Meta,@@ -19,6 +50,11 @@ export async function fetchAudio(return document;}+ const urlRegex = await getSupportedUrlRegex();+ if (!urlRegex.test(meta.url)) {+ throw new AudioUnsupportedUrlError();+ }+const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {method: "POST",headers: { "Content-Type": "application/json" },