Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(audio): route audio format via audio feature flag (#3428)
Abimael Martell9f53bee1f284
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsnew file mode 100644index 000000000..0a3f04b7b--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -0,0 +1,90 @@+/**+ * Unit test for audio-format engine routing via buildFallbackList.+ * Verifies that requesting audio format selects only index + tlsclient+ * engines (chrome-cdp and others are excluded) and that non-audio+ * requests still route through chrome-cdp as primary.+ */++// Avoid jest ESM-parse issues on transitive `uuid` import when pulling in engines.+jest.mock("uuid", () => ({+ v4: () => "test-uuid-v4",+ v7: () => "test-uuid-v7",+ validate: () => true,+}));++describe("Audio format engine routing (buildFallbackList)", () => {+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;++ const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;+ const originalIndexUrl = process.env.INDEX_SUPABASE_URL;++ beforeAll(() => {+ process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";+ process.env.INDEX_SUPABASE_URL = "http://test-index-supabase";++ jest.isolateModules(() => {+ buildFallbackList =+ require("../../../scraper/scrapeURL/engines").buildFallbackList;+ });+ });++ afterAll(() => {+ if (originalFireEngineUrl === undefined) {+ delete process.env.FIRE_ENGINE_BETA_URL;+ } else {+ process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;+ }+ if (originalIndexUrl === undefined) {+ delete process.env.INDEX_SUPABASE_URL;+ } else {+ process.env.INDEX_SUPABASE_URL = originalIndexUrl;+ }+ });++ const buildStubMeta = (featureFlags: string[]) =>+ ({+ id: "test",+ url: "https://www.youtube.com/watch?v=abc",+ options: {+ formats: [],+ maxAge: 3600000,+ },+ internalOptions: { teamId: "test" },+ featureFlags: new Set(featureFlags),+ mock: null,+ logger: {+ info: jest.fn(),+ warn: jest.fn(),+ debug: jest.fn(),+ error: jest.fn(),+ child: jest.fn().mockReturnThis(),+ },+ }) as any;++ it("routes audio format to index then tlsclient only", async () => {+ const fallback = await buildFallbackList(buildStubMeta(["audio"]));+ const engines = fallback.map(f => f.engine);++ // Cache-first (index, quality 1000), then tlsclient (quality 10).+ // index;documents and tlsclient;stealth drop out via the positive-quality filter.+ expect(engines).toEqual(["index", "fire-engine;tlsclient"]);+ });++ it("excludes chrome-cdp engines when audio format is requested", async () => {+ const fallback = await buildFallbackList(buildStubMeta(["audio"]));+ const engines = fallback.map(f => f.engine);++ expect(engines).not.toContain("fire-engine;chrome-cdp");+ expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp");+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");+ expect(engines).not.toContain("fetch");+ });++ it("still allows chrome-cdp for non-audio requests", async () => {+ const fallback = await buildFallbackList(buildStubMeta([]));+ const engines = fallback.map(f => f.engine);++ expect(engines).toContain("fire-engine;chrome-cdp");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex e8ebbd88c..4305ad47b 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -79,6 +79,7 @@ const featureFlags = ["screenshot@fullScreen","pdf","document",+ "audio","atsv","location","mobile",@@ -102,6 +103,7 @@ const featureFlagOptions: {"screenshot@fullScreen": { priority: 10 },pdf: { priority: 100 },document: { priority: 100 },+ audio: { priority: 100 },atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if notuseFastMode: { priority: 90 },location: { priority: 10 },@@ -208,6 +210,7 @@ const engineOptions: {"screenshot@fullScreen": true,pdf: false,document: false,+ audio: true,atsv: false,mobile: true,location: true,@@ -227,6 +230,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,+ audio: false,atsv: false,location: true,mobile: true,@@ -246,6 +250,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,+ audio: false,atsv: false,location: true,mobile: true,@@ -265,6 +270,7 @@ const engineOptions: {"screenshot@fullScreen": true,pdf: true,document: true,+ audio: true,atsv: false,location: true,mobile: true,@@ -284,6 +290,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,+ audio: false,atsv: false,location: true,mobile: true,@@ -303,6 +310,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,+ audio: false,atsv: false,location: true,mobile: true,@@ -322,6 +330,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: false,+ audio: false,atsv: false,location: false,mobile: false,@@ -341,6 +350,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: false,+ audio: true,atsv: true,location: true,mobile: false,@@ -360,6 +370,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: false,+ audio: true,atsv: true,location: true,mobile: false,@@ -379,6 +390,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: false,+ audio: false,atsv: false,location: false,mobile: false,@@ -398,6 +410,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: true,document: false,+ audio: false,atsv: false,location: false,mobile: false,@@ -417,6 +430,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: true,+ audio: false,atsv: false,location: false,mobile: false,@@ -436,6 +450,7 @@ const engineOptions: {"screenshot@fullScreen": false,pdf: false,document: false,+ audio: false,atsv: false,location: false,mobile: false,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 06cca603b..ada28140b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -171,6 +171,10 @@ function buildFeatureFlags(flags.add("branding");}+ if (hasFormatOfType(options.formats, "audio")) {+ flags.add("audio");+ }+if (options.waitFor !== 0) {flags.add("waitFor");}@@ -325,19 +329,6 @@ async function buildMetaObject(}}- // HACK(hotfix): audio format should go through tlsclient.- if (- internalOptions.forceEngine === undefined &&- hasFormatOfType(options.formats, "audio")- ) {- internalOptions = Object.assign(internalOptions, {- forceEngine: [- "fire-engine;tlsclient",- "fire-engine;tlsclient;stealth",- ] as Engine[],- });- }-const logger = _logger.child({module: "ScrapeURL",scrapeId: id,