Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(avgrab): get site support from service directly (#3229)

Gergő Móricz22c25eb287d2
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index ba1c03d1a..02ef95fc6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -2014,7 +2014,7 @@ describe("Attribute formats", () => {
describeIf(!TEST_SELF_HOST)("Audio format", () => {
it.concurrent(
- "should return audio field with signed GCS URL for a YouTube video",
+ "should return audio field with signed GCS URL for a supported video URL",
async () => {
const data = await scrape(
{
@@ -2032,7 +2032,7 @@ describe("Attribute formats", () => {
);
it.concurrent(
- "should reject non-YouTube URL with audio format",
+ "should reject unsupported URL with audio format",
async () => {
const result = await scrapeWithFailure(
{
@@ -2042,7 +2042,7 @@ describe("Attribute formats", () => {
identity,
);
- expect(result.error).toMatch(/audio.*youtube/i);
+ expect(result.error).toMatch(/audio/i);
},
scrapeTimeout,
);
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index c94d24b93..1c936de53 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -27,24 +27,6 @@ import {
} from "../../services/webhook/schema";
import { BrandingProfile } from "../../types/branding";
-function isYouTubeUrl(url: string): boolean {
- try {
- const parsed = new globalThis.URL(url);
- if (
- parsed.hostname === "youtube.com" ||
- parsed.hostname.endsWith(".youtube.com")
- ) {
- return parsed.pathname === "/watch" && !!parsed.searchParams.get("v");
- }
- if (parsed.hostname === "youtu.be") {
- return parsed.pathname !== "/";
- }
- return false;
- } catch {
- return false;
- }
-}
-
// Base URL schema with common validation logic
export const URL = z.preprocess(
x => {
@@ -832,10 +814,6 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase)
.refine(waitForRefine, waitForRefineOpts)
- .refine(
- x => !x.formats?.some(f => f.type === "audio") || isYouTubeUrl(x.url),
- "The audio format is only supported for YouTube URLs (youtube.com/watch?v=... or youtu.be/...)",
- )
.transform(extractTransformRequired);
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 23134ffff..e280ab801 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -29,6 +29,7 @@ import {
ScrapeJobCancelledError,
ScrapeRetryLimitError,
BrandingNotSupportedError,
+ AudioUnsupportedUrlError,
} from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -59,6 +60,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,
SCRAPE_SITEMAP_ERROR: SitemapError,
CRAWL_DENIAL: CrawlDenialError,
+ SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,
// Zod errors
BAD_REQUEST: null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index e31cd8369..640e846da 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -24,6 +24,7 @@ export type ErrorCodes =
| "SCRAPE_ACTIONS_NOT_SUPPORTED"
| "SCRAPE_BRANDING_NOT_SUPPORTED"
| "AGENT_INDEX_ONLY"
+ | "SCRAPE_AUDIO_UNSUPPORTED_URL"
| "CRAWL_DENIAL"
| "BAD_REQUEST_INVALID_JSON"
| "BAD_REQUEST";
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 1d644449b..f0d6c5143 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -469,6 +469,28 @@ export class AgentIndexOnlyError extends TransportableError {
}
}
+export class AudioUnsupportedUrlError extends TransportableError {
+ constructor(message?: string) {
+ super(
+ "SCRAPE_AUDIO_UNSUPPORTED_URL",
+ message ?? "The audio format does not support the provided URL",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new AudioUnsupportedUrlError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class BrandingNotSupportedError extends TransportableError {
constructor(message: string) {
super("SCRAPE_BRANDING_NOT_SUPPORTED", message);
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
index 3798fe01c..553c00cd3 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -2,6 +2,37 @@ import { Meta } from "..";
import { Document } from "../../../controllers/v2/types";
import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
+import { AudioUnsupportedUrlError } from "../error";
+
+let cachedUrlRegex: RegExp | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+async function getSupportedUrlRegex(): Promise<RegExp> {
+ if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+ return cachedUrlRegex;
+ }
+
+ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);
+ if (!res.ok) {
+ throw new Error(
+ "Failed to fetch supported URL patterns from audio service",
+ );
+ }
+
+ const data = await res.json().catch(() => null);
+ if (!data || typeof data.regex !== "string") {
+ throw new Error("Audio service returned invalid supported URL patterns");
+ }
+
+ try {
+ cachedUrlRegex = new RegExp(data.regex);
+ } catch {
+ throw new Error("Audio service returned invalid supported URL patterns");
+ }
+ cacheTimestamp = Date.now();
+ return cachedUrlRegex;
+}
export async function fetchAudio(
meta: Meta,
@@ -19,6 +50,11 @@ export async function fetchAudio(
return document;
}
+ const urlRegex = await getSupportedUrlRegex();
+ if (!urlRegex.test(meta.url)) {
+ throw new AudioUnsupportedUrlError();
+ }
+
const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {
method: "POST",
headers: { "Content-Type": "application/json" },