Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): proxy Agent state to extract-v3 (#4667)
Gergő Móricz4ed4d2851161
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex bc81b50fc..8904fcdc3 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -566,7 +566,7 @@ const configSchema = z.object({DISABLE_ENGPICKER: z.stringbool().optional(),DISABLE_MONITORING: z.stringbool().default(false),- EXTRACT_V3_BETA_URL: z.string().optional(),+ EXTRACT_V3_BETA_URL: z.string().url().optional(),AGENT_INTEROP_SECRET: z.string().refine(value => value.trim().length > 0, {diff --git a/apps/api/src/controllers/v2/__tests__/agent-cancel.test.ts b/apps/api/src/controllers/v2/__tests__/agent-cancel.test.tsnew file mode 100644index 000000000..2d1afe979--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/agent-cancel.test.ts@@ -0,0 +1,94 @@+import type { Response } from "express";+import type { Mock } from "vitest";+import { getAgentJobAccess } from "../../../lib/operational-job-access";+import { agentCancelController } from "../agent-cancel";+import type { AgentCancelResponse, RequestWithAuth } from "../types";++vi.mock("../../../lib/operational-job-access", () => ({+ getAgentJobAccess: vi.fn(),+}));++describe("agentCancelController", () => {+ const req = {+ params: { jobId: "job-123" },+ auth: { team_id: "team-123" },+ } as RequestWithAuth<{ jobId: string }, AgentCancelResponse, any>;++ const buildRes = () =>+ ({+ status: vi.fn().mockReturnThis(),+ json: vi.fn(),+ }) as unknown as Response<AgentCancelResponse>;++ beforeEach(() => {+ vi.clearAllMocks();+ (getAgentJobAccess as Mock).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: Date.now() + 60_000,+ });+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("returns the terminal-state conflict from extract-v3", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(+ new Response(JSON.stringify({ error: "Agent already finished" }), {+ status: 409,+ headers: { "content-type": "application/json" },+ }),+ ),+ );+ const res = buildRes();++ await agentCancelController(req, res);++ expect(res.status).toHaveBeenCalledWith(409);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ error: "Agent already finished",+ });+ });++ it("does not report success when extract-v3 rejects cancellation", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(new Response(null, { status: 500 })),+ );+ const res = buildRes();++ await agentCancelController(req, res);++ expect(res.status).toHaveBeenCalledWith(500);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ error: "Failed to cancel agent",+ });+ });++ it("preserves the public error for repeated cancellation", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(+ new Response(+ JSON.stringify({ error: "Agent has already been cancelled" }),+ {+ status: 409,+ headers: { "content-type": "application/json" },+ },+ ),+ ),+ );+ const res = buildRes();++ await agentCancelController(req, res);++ expect(res.json).toHaveBeenCalledWith({+ success: false,+ error: "Agent is already cancelled",+ });+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.tsindex 2badccd9d..284a29691 100644--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts@@ -3,20 +3,15 @@ import type { Response } from "express";import { agentStatusController } from "../agent-status";import { agentRequestSchema } from "../types";import type { RequestWithAuth } from "../types";-import { supabaseGetAgentByIdDirect } from "../../../lib/supabase-jobs";import { getAgentJobAccess } from "../../../lib/operational-job-access";-import { getJobFromGCS } from "../../../lib/gcs-jobs";--vi.mock("../../../lib/supabase-jobs", () => ({- supabaseGetAgentByIdDirect: vi.fn(),-}));+import { getExtractV3AgentStatus } from "../../../lib/extract-v3-status";vi.mock("../../../lib/operational-job-access", () => ({getAgentJobAccess: vi.fn(),}));-vi.mock("../../../lib/gcs-jobs", () => ({- getJobFromGCS: vi.fn(),+vi.mock("../../../lib/extract-v3-status", () => ({+ getExtractV3AgentStatus: vi.fn(),}));describe("agentStatusController", () => {@@ -40,13 +35,13 @@ describe("agentStatusController", () => {teamId: "team-123",expiresAtMs: Date.now() + 60_000,});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: true,- options: { model: "spark-1-mini" },- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "success",+ model: "spark-1-mini",+ data: { result: "ok" },});- (getJobFromGCS as Mock).mockResolvedValue({ result: "ok" });const res = buildRes();await agentStatusController(baseReq, res);@@ -62,11 +57,10 @@ describe("agentStatusController", () => {teamId: "team-123",expiresAtMs: Date.now() + 60_000,});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: false,- options: null,- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "failed",});const res = buildRes();@@ -74,7 +68,11 @@ describe("agentStatusController", () => {expect(res.status).toHaveBeenCalledWith(200);expect(res.json).toHaveBeenCalledWith(- expect.objectContaining({ model: "spark-1-pro" }),+ expect.objectContaining({+ success: true,+ status: "failed",+ model: "spark-1-pro",+ }),);});it("returns effort from agent options", async () => {@@ -82,13 +80,14 @@ describe("agentStatusController", () => {teamId: "team-123",expiresAtMs: Date.now() + 60_000,});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: true,- options: { model: "spark-2", effort: "high" },- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "success",+ model: "spark-2",+ effort: "high",+ data: { result: "ok" },});- (getJobFromGCS as Mock).mockResolvedValue({ result: "ok" });const res = buildRes();await agentStatusController(baseReq, res);@@ -104,11 +103,11 @@ describe("agentStatusController", () => {teamId: "team-123",expiresAtMs: Date.now() + 60_000,});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: false,- options: { model: "spark-2" },- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "failed",+ model: "spark-2",});const res = buildRes();@@ -120,6 +119,41 @@ describe("agentStatusController", () => {expect(body.effort).toBeUndefined();});+ it("returns terminal state and result metadata from extract-v3", async () => {+ (getAgentJobAccess as Mock).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: Date.now() + 60_000,+ });+ (getExtractV3AgentStatus as Mock).mockResolvedValue({+ id: "job-123",+ success: true,+ status: "success",+ model: "spark-2",+ effort: "medium",+ data: { result: "ok" },+ message: "Done",+ threadId: "thread-123",+ threadTurn: 2,+ mode: "chat",+ creditsUsed: 7,+ });++ const res = buildRes();+ await agentStatusController(baseReq, res);++ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ status: "completed",+ data: { result: "ok" },+ message: "Done",+ threadId: "thread-123",+ threadTurn: 2,+ mode: "chat",+ creditsUsed: 7,+ }),+ );+ });+it.each([// python-sdk < 4.37.1 cannot parse "spark-2" and is lied to["[email protected]", "spark-1-pro"],@@ -142,11 +176,11 @@ describe("agentStatusController", () => {expiresAtMs: Date.now() + 60_000,clientOrigin: origin,});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: false,- options: { model: "spark-2" },- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "failed",+ model: "spark-2",});const res = buildRes();@@ -165,11 +199,11 @@ describe("agentStatusController", () => {expiresAtMs: Date.now() + 60_000,clientOrigin: "[email protected]",});- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({+ (getExtractV3AgentStatus as Mock).mockResolvedValue({id: "job-123",- is_successful: false,- options: { model: "spark-1-mini" },- created_at: "2025-01-01T00:00:00Z",+ success: true,+ status: "failed",+ model: "spark-1-mini",});const res = buildRes();diff --git a/apps/api/src/controllers/v2/agent-cancel.ts b/apps/api/src/controllers/v2/agent-cancel.tsindex 272f5f423..0221f455c 100644--- a/apps/api/src/controllers/v2/agent-cancel.ts+++ b/apps/api/src/controllers/v2/agent-cancel.ts@@ -1,6 +1,5 @@import { Response } from "express";import { AgentCancelResponse, RequestWithAuth } from "./types";-import { supabaseGetAgentByIdDirect } from "../../lib/supabase-jobs";import { config } from "../../config";import { getAgentJobAccess } from "../../lib/operational-job-access";@@ -21,14 +20,6 @@ export async function agentCancelController(});}- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);- if (agent) {- return res.status(409).json({- success: false,- error: "Agent already finished",- });- }-const resp = await fetch(config.EXTRACT_V3_BETA_URL + "/internal/extracts/" + req.params.jobId,{@@ -40,9 +31,22 @@ export async function agentCancelController();if (resp.status === 409) {+ const body = (await resp.json().catch(() => null)) as {+ error?: string;+ } | null;return res.status(409).json({success: false,- error: "Agent is already cancelled",+ error:+ body?.error === "Agent already finished"+ ? body.error+ : "Agent is already cancelled",+ });+ }++ if (!resp.ok) {+ return res.status(500).json({+ success: false,+ error: "Failed to cancel agent",});}diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.tsindex 3de1a3d96..4c4bb7e84 100644--- a/apps/api/src/controllers/v2/agent-status.ts+++ b/apps/api/src/controllers/v2/agent-status.ts@@ -7,11 +7,8 @@ import {AgentSuggestion,RequestWithAuth,} from "./types";-import { supabaseGetAgentByIdDirect } from "../../lib/supabase-jobs";-import { logger as _logger, logger } from "../../lib/logger";-import { getJobFromGCS } from "../../lib/gcs-jobs";-import { config } from "../../config";import { getAgentJobAccess } from "../../lib/operational-job-access";+import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";// python-sdk versions before 4.37.1 validate the status response's `model`// with pydantic as Literal["spark-1-pro", "spark-1-mini"], so any other value@@ -45,32 +42,6 @@ function isIncompatiblePythonSdkOrigin(origin: unknown): boolean {);}-// Runs that produce more than a JSON result store an envelope instead of the-// raw result, so `data` keeps its original meaning for every existing object.-const AGENT_RESULT_KEY = "__agentResult";--type StoredAgentResult = {- [AGENT_RESULT_KEY]: 1;- data: unknown | null;- message: string | null;- suggestions?: AgentSuggestion[];- pendingApproval?: AgentPendingApproval;- // The per-run summary rides the envelope because options holds what the run- // was asked to do, not what it did.- exchange?: AgentExchangeSummary;-};--function isStoredAgentResult(value: unknown): value is StoredAgentResult {- return (- typeof value === "object" &&- value !== null &&- !Array.isArray(value) &&- (value as Record<string, unknown>)[AGENT_RESULT_KEY] === 1- );-}--// The agent service persists these next to `model`/`effort`, so they come from-// the same two sources and are absent on runs that predate threads.type ThreadOptions = {threadId?: string;threadTurn?: number;@@ -102,61 +73,14 @@ export async function agentStatusController(});}- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);+ const agent = await getExtractV3AgentStatus(req.params.jobId);- let model: "spark-1-pro" | "spark-1-mini" | "spark-2";+ let model: "spark-1-pro" | "spark-1-mini" | "spark-2" =+ agent.model ?? "spark-1-pro";// The agent service persists the effort of a run that used it. Older rows// and runs that picked a model have no effort, so this stays undefined.- let effort: "low" | "medium" | "high" | undefined;- let thread: ThreadOptions = {};- if (agent) {- model = (agent.options?.model ?? "spark-1-pro") as- | "spark-1-pro"- | "spark-1-mini"- | "spark-2";- effort = agent.options?.effort as "low" | "medium" | "high" | undefined;- thread = readThreadOptions(agent.options);- } else {- try {- const optionsRequest = await fetch(- config.EXTRACT_V3_BETA_URL +- "/v2/extract/" +- req.params.jobId +- "/options",- {- headers: {- Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,- },- },- );-- if (optionsRequest.status !== 200) {- logger.warn("Failed to get agent request details", {- status: optionsRequest.status,- method: "agentStatusController",- module: "api/v2",- text: await optionsRequest.text(),- });- model = "spark-1-pro"; // fall back to this value- } else {- const options = await optionsRequest.json();- model = (options.model ?? "spark-1-pro") as- | "spark-1-pro"- | "spark-1-mini"- | "spark-2";- effort = options.effort as "low" | "medium" | "high" | undefined;- thread = readThreadOptions(options);- }- } catch (error) {- logger.warn("Failed to get agent request details", {- error,- method: "agentStatusController",- module: "api/v2",- extractId: req.params.jobId,- });- model = "spark-1-pro"; // fall back to this value- }- }+ const effort = agent.effort;+ const thread = readThreadOptions(agent);// Lie about the model to python-sdk versions that cannot parse "spark-2"// (see isIncompatiblePythonSdkOrigin). Only spark-2 needs disguising — a@@ -170,43 +94,21 @@ export async function agentStatusController(model = "spark-1-pro";}- let data: any = undefined;- let message: string | undefined;- let suggestions: AgentSuggestion[] | undefined;- let pendingApproval: AgentPendingApproval | undefined;- let exchange: AgentExchangeSummary | undefined;- if (agent?.is_successful) {- const stored: unknown = await getJobFromGCS(agent.id);- if (isStoredAgentResult(stored)) {- data = stored.data ?? null;- message = stored.message ?? undefined;- suggestions = stored.suggestions;- pendingApproval = stored.pendingApproval;- exchange = stored.exchange;- } else {- data = stored;- }- }-return res.status(200).json({success: true,- status: !agent- ? "processing"- : agent.is_successful- ? "completed"- : "failed",- error: agent?.error || undefined,- data,+ status: agent.status === "success" ? "completed" : agent.status,+ error: agent.error,+ data: agent.data,model,effort,threadId: thread.threadId,threadTurn: thread.threadTurn,mode: thread.mode,- message,- suggestions,- pendingApproval,- exchange,+ message: agent.message,+ suggestions: agent.suggestions as AgentSuggestion[] | undefined,+ pendingApproval: agent.pendingApproval as AgentPendingApproval | undefined,+ exchange: agent.exchange as AgentExchangeSummary | undefined,expiresAt: new Date(access.expiresAtMs).toISOString(),- creditsUsed: agent?.credits_cost,+ creditsUsed: agent.creditsUsed,});}diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex a78e4e91e..89ed86563 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -6,15 +6,13 @@ import {getExtractExpiry,getExtractResult,} from "../../lib/extract/extract-redis";-import {- supabaseGetAgentByIdDirect,- supabaseGetExtractByIdDirect,-} from "../../lib/supabase-jobs";+import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";import { logger as _logger } from "../../lib/logger";import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";+import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -52,24 +50,15 @@ export async function extractStatusController(}if (access.kind === "agent") {- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);-- let data: any = undefined;- if (agent?.is_successful) {- data = await getJobFromGCS(agent.id);- }+ const agent = await getExtractV3AgentStatus(req.params.jobId);return res.status(200).json({success: true,- status: !agent- ? "processing"- : agent.is_successful- ? "completed"- : "failed",- error: agent?.error || undefined,- data,+ status: agent.status === "success" ? "completed" : agent.status,+ error: agent.error,+ data: agent.data,expiresAt: new Date(access.expiresAtMs).toISOString(),- creditsUsed: agent?.credits_cost,+ creditsUsed: agent.creditsUsed,});}}diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.tsnew file mode 100644index 000000000..36fa2aded--- /dev/null+++ b/apps/api/src/lib/extract-v3-status.ts@@ -0,0 +1,45 @@+import { config } from "../config";++export type ExtractV3AgentStatus = {+ success: true;+ id: string;+ status: "processing" | "success" | "failed";+ data?: unknown;+ error?: string;+ model?: "spark-1-pro" | "spark-1-mini" | "spark-2";+ effort?: "low" | "medium" | "high";+ threadId?: string;+ threadTurn?: number;+ mode?: "extract" | "chat";+ message?: string;+ suggestions?: unknown[];+ pendingApproval?: unknown;+ exchange?: unknown;+ creditsUsed?: number;+};++export async function getExtractV3AgentStatus(+ id: string,+): Promise<ExtractV3AgentStatus> {+ if (!config.EXTRACT_V3_BETA_URL) {+ throw new Error("Agent beta is not enabled.");+ }++ const url = new URL(+ `${config.EXTRACT_V3_BETA_URL.replace(/\/$/, "")}/internal/extracts/${encodeURIComponent(id)}`,+ );+ const response = await fetch(url, {+ headers: {+ Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,+ },+ signal: AbortSignal.timeout(10_000),+ });++ if (!response.ok) {+ throw new Error(+ `Agent status service returned ${response.status}: ${await response.text()}`,+ );+ }++ return (await response.json()) as ExtractV3AgentStatus;+}diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.tsindex 06e9ac93b..e75bdc9e5 100644--- a/apps/api/src/lib/supabase-jobs.ts+++ b/apps/api/src/lib/supabase-jobs.ts@@ -113,18 +113,3 @@ export const supabaseGetCrawlRequestById = async (requestId: string) => {.limit(1);return data ?? null;};--export const supabaseGetAgentByIdDirect = async (- agentId: string,-): Promise<any> => {- try {- const [data] = await db- .select()- .from(schema.agents)- .where(eq(schema.agents.id, agentId))- .limit(1);- return data ?? null;- } catch (error) {- return null;- }-};