Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(audio): route audio format via audio feature flag (#3428)

Abimael Martell9f53bee1f284
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
new file mode 100644
index 000000000..0a3f04b7b
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -0,0 +1,90 @@
+/**
+ * Unit test for audio-format engine routing via buildFallbackList.
+ * Verifies that requesting audio format selects only index + tlsclient
+ * engines (chrome-cdp and others are excluded) and that non-audio
+ * requests still route through chrome-cdp as primary.
+ */
+
+// Avoid jest ESM-parse issues on transitive `uuid` import when pulling in engines.
+jest.mock("uuid", () => ({
+ v4: () => "test-uuid-v4",
+ v7: () => "test-uuid-v7",
+ validate: () => true,
+}));
+
+describe("Audio format engine routing (buildFallbackList)", () => {
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;
+
+ const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
+ const originalIndexUrl = process.env.INDEX_SUPABASE_URL;
+
+ beforeAll(() => {
+ process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
+ process.env.INDEX_SUPABASE_URL = "http://test-index-supabase";
+
+ jest.isolateModules(() => {
+ buildFallbackList =
+ require("../../../scraper/scrapeURL/engines").buildFallbackList;
+ });
+ });
+
+ afterAll(() => {
+ if (originalFireEngineUrl === undefined) {
+ delete process.env.FIRE_ENGINE_BETA_URL;
+ } else {
+ process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;
+ }
+ if (originalIndexUrl === undefined) {
+ delete process.env.INDEX_SUPABASE_URL;
+ } else {
+ process.env.INDEX_SUPABASE_URL = originalIndexUrl;
+ }
+ });
+
+ const buildStubMeta = (featureFlags: string[]) =>
+ ({
+ id: "test",
+ url: "https://www.youtube.com/watch?v=abc",
+ options: {
+ formats: [],
+ maxAge: 3600000,
+ },
+ internalOptions: { teamId: "test" },
+ featureFlags: new Set(featureFlags),
+ mock: null,
+ logger: {
+ info: jest.fn(),
+ warn: jest.fn(),
+ debug: jest.fn(),
+ error: jest.fn(),
+ child: jest.fn().mockReturnThis(),
+ },
+ }) as any;
+
+ it("routes audio format to index then tlsclient only", async () => {
+ const fallback = await buildFallbackList(buildStubMeta(["audio"]));
+ const engines = fallback.map(f => f.engine);
+
+ // Cache-first (index, quality 1000), then tlsclient (quality 10).
+ // index;documents and tlsclient;stealth drop out via the positive-quality filter.
+ expect(engines).toEqual(["index", "fire-engine;tlsclient"]);
+ });
+
+ it("excludes chrome-cdp engines when audio format is requested", async () => {
+ const fallback = await buildFallbackList(buildStubMeta(["audio"]));
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).not.toContain("fire-engine;chrome-cdp");
+ expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");
+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp");
+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");
+ expect(engines).not.toContain("fetch");
+ });
+
+ it("still allows chrome-cdp for non-audio requests", async () => {
+ const fallback = await buildFallbackList(buildStubMeta([]));
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).toContain("fire-engine;chrome-cdp");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index e8ebbd88c..4305ad47b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -79,6 +79,7 @@ const featureFlags = [
"screenshot@fullScreen",
"pdf",
"document",
+ "audio",
"atsv",
"location",
"mobile",
@@ -102,6 +103,7 @@ const featureFlagOptions: {
"screenshot@fullScreen": { priority: 10 },
pdf: { priority: 100 },
document: { priority: 100 },
+ audio: { priority: 100 },
atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
useFastMode: { priority: 90 },
location: { priority: 10 },
@@ -208,6 +210,7 @@ const engineOptions: {
"screenshot@fullScreen": true,
pdf: false,
document: false,
+ audio: true,
atsv: false,
mobile: true,
location: true,
@@ -227,6 +230,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: true,
mobile: true,
@@ -246,6 +250,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: true,
mobile: true,
@@ -265,6 +270,7 @@ const engineOptions: {
"screenshot@fullScreen": true,
pdf: true,
document: true,
+ audio: true,
atsv: false,
location: true,
mobile: true,
@@ -284,6 +290,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: true,
mobile: true,
@@ -303,6 +310,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: true,
mobile: true,
@@ -322,6 +330,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: false,
mobile: false,
@@ -341,6 +350,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: false,
+ audio: true,
atsv: true,
location: true,
mobile: false,
@@ -360,6 +370,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: false,
+ audio: true,
atsv: true,
location: true,
mobile: false,
@@ -379,6 +390,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: false,
mobile: false,
@@ -398,6 +410,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: true,
document: false,
+ audio: false,
atsv: false,
location: false,
mobile: false,
@@ -417,6 +430,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: true,
+ audio: false,
atsv: false,
location: false,
mobile: false,
@@ -436,6 +450,7 @@ const engineOptions: {
"screenshot@fullScreen": false,
pdf: false,
document: false,
+ audio: false,
atsv: false,
location: false,
mobile: false,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 06cca603b..ada28140b 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -171,6 +171,10 @@ function buildFeatureFlags(
flags.add("branding");
}
+ if (hasFormatOfType(options.formats, "audio")) {
+ flags.add("audio");
+ }
+
if (options.waitFor !== 0) {
flags.add("waitFor");
}
@@ -325,19 +329,6 @@ async function buildMetaObject(
}
}
- // HACK(hotfix): audio format should go through tlsclient.
- if (
- internalOptions.forceEngine === undefined &&
- hasFormatOfType(options.formats, "audio")
- ) {
- internalOptions = Object.assign(internalOptions, {
- forceEngine: [
- "fire-engine;tlsclient",
- "fire-engine;tlsclient;stealth",
- ] as Engine[],
- });
- }
-
const logger = _logger.child({
module: "ScrapeURL",
scrapeId: id,