Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): proxy Agent state to extract-v3 (#4667)

Gergő Móricz4ed4d2851161
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index bc81b50fc..8904fcdc3 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -566,7 +566,7 @@ const configSchema = z.object({
DISABLE_ENGPICKER: z.stringbool().optional(),
DISABLE_MONITORING: z.stringbool().default(false),
- EXTRACT_V3_BETA_URL: z.string().optional(),
+ EXTRACT_V3_BETA_URL: z.string().url().optional(),
AGENT_INTEROP_SECRET: z
.string()
.refine(value => value.trim().length > 0, {
diff --git a/apps/api/src/controllers/v2/__tests__/agent-cancel.test.ts b/apps/api/src/controllers/v2/__tests__/agent-cancel.test.ts
new file mode 100644
index 000000000..2d1afe979
--- /dev/null
+++ b/apps/api/src/controllers/v2/__tests__/agent-cancel.test.ts
@@ -0,0 +1,94 @@
+import type { Response } from "express";
+import type { Mock } from "vitest";
+import { getAgentJobAccess } from "../../../lib/operational-job-access";
+import { agentCancelController } from "../agent-cancel";
+import type { AgentCancelResponse, RequestWithAuth } from "../types";
+
+vi.mock("../../../lib/operational-job-access", () => ({
+ getAgentJobAccess: vi.fn(),
+}));
+
+describe("agentCancelController", () => {
+ const req = {
+ params: { jobId: "job-123" },
+ auth: { team_id: "team-123" },
+ } as RequestWithAuth<{ jobId: string }, AgentCancelResponse, any>;
+
+ const buildRes = () =>
+ ({
+ status: vi.fn().mockReturnThis(),
+ json: vi.fn(),
+ }) as unknown as Response<AgentCancelResponse>;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ (getAgentJobAccess as Mock).mockResolvedValue({
+ teamId: "team-123",
+ expiresAtMs: Date.now() + 60_000,
+ });
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("returns the terminal-state conflict from extract-v3", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: "Agent already finished" }), {
+ status: 409,
+ headers: { "content-type": "application/json" },
+ }),
+ ),
+ );
+ const res = buildRes();
+
+ await agentCancelController(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ error: "Agent already finished",
+ });
+ });
+
+ it("does not report success when extract-v3 rejects cancellation", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(new Response(null, { status: 500 })),
+ );
+ const res = buildRes();
+
+ await agentCancelController(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(500);
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ error: "Failed to cancel agent",
+ });
+ });
+
+ it("preserves the public error for repeated cancellation", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(
+ new Response(
+ JSON.stringify({ error: "Agent has already been cancelled" }),
+ {
+ status: 409,
+ headers: { "content-type": "application/json" },
+ },
+ ),
+ ),
+ );
+ const res = buildRes();
+
+ await agentCancelController(req, res);
+
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ error: "Agent is already cancelled",
+ });
+ });
+});
diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
index 2badccd9d..284a29691 100644
--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
@@ -3,20 +3,15 @@ import type { Response } from "express";
import { agentStatusController } from "../agent-status";
import { agentRequestSchema } from "../types";
import type { RequestWithAuth } from "../types";
-import { supabaseGetAgentByIdDirect } from "../../../lib/supabase-jobs";
import { getAgentJobAccess } from "../../../lib/operational-job-access";
-import { getJobFromGCS } from "../../../lib/gcs-jobs";
-
-vi.mock("../../../lib/supabase-jobs", () => ({
- supabaseGetAgentByIdDirect: vi.fn(),
-}));
+import { getExtractV3AgentStatus } from "../../../lib/extract-v3-status";
vi.mock("../../../lib/operational-job-access", () => ({
getAgentJobAccess: vi.fn(),
}));
-vi.mock("../../../lib/gcs-jobs", () => ({
- getJobFromGCS: vi.fn(),
+vi.mock("../../../lib/extract-v3-status", () => ({
+ getExtractV3AgentStatus: vi.fn(),
}));
describe("agentStatusController", () => {
@@ -40,13 +35,13 @@ describe("agentStatusController", () => {
teamId: "team-123",
expiresAtMs: Date.now() + 60_000,
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: true,
- options: { model: "spark-1-mini" },
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "success",
+ model: "spark-1-mini",
+ data: { result: "ok" },
});
- (getJobFromGCS as Mock).mockResolvedValue({ result: "ok" });
const res = buildRes();
await agentStatusController(baseReq, res);
@@ -62,11 +57,10 @@ describe("agentStatusController", () => {
teamId: "team-123",
expiresAtMs: Date.now() + 60_000,
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: false,
- options: null,
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "failed",
});
const res = buildRes();
@@ -74,7 +68,11 @@ describe("agentStatusController", () => {
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith(
- expect.objectContaining({ model: "spark-1-pro" }),
+ expect.objectContaining({
+ success: true,
+ status: "failed",
+ model: "spark-1-pro",
+ }),
);
});
it("returns effort from agent options", async () => {
@@ -82,13 +80,14 @@ describe("agentStatusController", () => {
teamId: "team-123",
expiresAtMs: Date.now() + 60_000,
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: true,
- options: { model: "spark-2", effort: "high" },
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "success",
+ model: "spark-2",
+ effort: "high",
+ data: { result: "ok" },
});
- (getJobFromGCS as Mock).mockResolvedValue({ result: "ok" });
const res = buildRes();
await agentStatusController(baseReq, res);
@@ -104,11 +103,11 @@ describe("agentStatusController", () => {
teamId: "team-123",
expiresAtMs: Date.now() + 60_000,
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: false,
- options: { model: "spark-2" },
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "failed",
+ model: "spark-2",
});
const res = buildRes();
@@ -120,6 +119,41 @@ describe("agentStatusController", () => {
expect(body.effort).toBeUndefined();
});
+ it("returns terminal state and result metadata from extract-v3", async () => {
+ (getAgentJobAccess as Mock).mockResolvedValue({
+ teamId: "team-123",
+ expiresAtMs: Date.now() + 60_000,
+ });
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
+ id: "job-123",
+ success: true,
+ status: "success",
+ model: "spark-2",
+ effort: "medium",
+ data: { result: "ok" },
+ message: "Done",
+ threadId: "thread-123",
+ threadTurn: 2,
+ mode: "chat",
+ creditsUsed: 7,
+ });
+
+ const res = buildRes();
+ await agentStatusController(baseReq, res);
+
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ status: "completed",
+ data: { result: "ok" },
+ message: "Done",
+ threadId: "thread-123",
+ threadTurn: 2,
+ mode: "chat",
+ creditsUsed: 7,
+ }),
+ );
+ });
+
it.each([
// python-sdk < 4.37.1 cannot parse "spark-2" and is lied to
["[email protected]", "spark-1-pro"],
@@ -142,11 +176,11 @@ describe("agentStatusController", () => {
expiresAtMs: Date.now() + 60_000,
clientOrigin: origin,
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: false,
- options: { model: "spark-2" },
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "failed",
+ model: "spark-2",
});
const res = buildRes();
@@ -165,11 +199,11 @@ describe("agentStatusController", () => {
expiresAtMs: Date.now() + 60_000,
clientOrigin: "[email protected]",
});
- (supabaseGetAgentByIdDirect as Mock).mockResolvedValue({
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
id: "job-123",
- is_successful: false,
- options: { model: "spark-1-mini" },
- created_at: "2025-01-01T00:00:00Z",
+ success: true,
+ status: "failed",
+ model: "spark-1-mini",
});
const res = buildRes();
diff --git a/apps/api/src/controllers/v2/agent-cancel.ts b/apps/api/src/controllers/v2/agent-cancel.ts
index 272f5f423..0221f455c 100644
--- a/apps/api/src/controllers/v2/agent-cancel.ts
+++ b/apps/api/src/controllers/v2/agent-cancel.ts
@@ -1,6 +1,5 @@
import { Response } from "express";
import { AgentCancelResponse, RequestWithAuth } from "./types";
-import { supabaseGetAgentByIdDirect } from "../../lib/supabase-jobs";
import { config } from "../../config";
import { getAgentJobAccess } from "../../lib/operational-job-access";
@@ -21,14 +20,6 @@ export async function agentCancelController(
});
}
- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
- if (agent) {
- return res.status(409).json({
- success: false,
- error: "Agent already finished",
- });
- }
-
const resp = await fetch(
config.EXTRACT_V3_BETA_URL + "/internal/extracts/" + req.params.jobId,
{
@@ -40,9 +31,22 @@ export async function agentCancelController(
);
if (resp.status === 409) {
+ const body = (await resp.json().catch(() => null)) as {
+ error?: string;
+ } | null;
return res.status(409).json({
success: false,
- error: "Agent is already cancelled",
+ error:
+ body?.error === "Agent already finished"
+ ? body.error
+ : "Agent is already cancelled",
+ });
+ }
+
+ if (!resp.ok) {
+ return res.status(500).json({
+ success: false,
+ error: "Failed to cancel agent",
});
}
diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.ts
index 3de1a3d96..4c4bb7e84 100644
--- a/apps/api/src/controllers/v2/agent-status.ts
+++ b/apps/api/src/controllers/v2/agent-status.ts
@@ -7,11 +7,8 @@ import {
AgentSuggestion,
RequestWithAuth,
} from "./types";
-import { supabaseGetAgentByIdDirect } from "../../lib/supabase-jobs";
-import { logger as _logger, logger } from "../../lib/logger";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
-import { config } from "../../config";
import { getAgentJobAccess } from "../../lib/operational-job-access";
+import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";
// python-sdk versions before 4.37.1 validate the status response's `model`
// with pydantic as Literal["spark-1-pro", "spark-1-mini"], so any other value
@@ -45,32 +42,6 @@ function isIncompatiblePythonSdkOrigin(origin: unknown): boolean {
);
}
-// Runs that produce more than a JSON result store an envelope instead of the
-// raw result, so `data` keeps its original meaning for every existing object.
-const AGENT_RESULT_KEY = "__agentResult";
-
-type StoredAgentResult = {
- [AGENT_RESULT_KEY]: 1;
- data: unknown | null;
- message: string | null;
- suggestions?: AgentSuggestion[];
- pendingApproval?: AgentPendingApproval;
- // The per-run summary rides the envelope because options holds what the run
- // was asked to do, not what it did.
- exchange?: AgentExchangeSummary;
-};
-
-function isStoredAgentResult(value: unknown): value is StoredAgentResult {
- return (
- typeof value === "object" &&
- value !== null &&
- !Array.isArray(value) &&
- (value as Record<string, unknown>)[AGENT_RESULT_KEY] === 1
- );
-}
-
-// The agent service persists these next to `model`/`effort`, so they come from
-// the same two sources and are absent on runs that predate threads.
type ThreadOptions = {
threadId?: string;
threadTurn?: number;
@@ -102,61 +73,14 @@ export async function agentStatusController(
});
}
- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
+ const agent = await getExtractV3AgentStatus(req.params.jobId);
- let model: "spark-1-pro" | "spark-1-mini" | "spark-2";
+ let model: "spark-1-pro" | "spark-1-mini" | "spark-2" =
+ agent.model ?? "spark-1-pro";
// The agent service persists the effort of a run that used it. Older rows
// and runs that picked a model have no effort, so this stays undefined.
- let effort: "low" | "medium" | "high" | undefined;
- let thread: ThreadOptions = {};
- if (agent) {
- model = (agent.options?.model ?? "spark-1-pro") as
- | "spark-1-pro"
- | "spark-1-mini"
- | "spark-2";
- effort = agent.options?.effort as "low" | "medium" | "high" | undefined;
- thread = readThreadOptions(agent.options);
- } else {
- try {
- const optionsRequest = await fetch(
- config.EXTRACT_V3_BETA_URL +
- "/v2/extract/" +
- req.params.jobId +
- "/options",
- {
- headers: {
- Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,
- },
- },
- );
-
- if (optionsRequest.status !== 200) {
- logger.warn("Failed to get agent request details", {
- status: optionsRequest.status,
- method: "agentStatusController",
- module: "api/v2",
- text: await optionsRequest.text(),
- });
- model = "spark-1-pro"; // fall back to this value
- } else {
- const options = await optionsRequest.json();
- model = (options.model ?? "spark-1-pro") as
- | "spark-1-pro"
- | "spark-1-mini"
- | "spark-2";
- effort = options.effort as "low" | "medium" | "high" | undefined;
- thread = readThreadOptions(options);
- }
- } catch (error) {
- logger.warn("Failed to get agent request details", {
- error,
- method: "agentStatusController",
- module: "api/v2",
- extractId: req.params.jobId,
- });
- model = "spark-1-pro"; // fall back to this value
- }
- }
+ const effort = agent.effort;
+ const thread = readThreadOptions(agent);
// Lie about the model to python-sdk versions that cannot parse "spark-2"
// (see isIncompatiblePythonSdkOrigin). Only spark-2 needs disguising — a
@@ -170,43 +94,21 @@ export async function agentStatusController(
model = "spark-1-pro";
}
- let data: any = undefined;
- let message: string | undefined;
- let suggestions: AgentSuggestion[] | undefined;
- let pendingApproval: AgentPendingApproval | undefined;
- let exchange: AgentExchangeSummary | undefined;
- if (agent?.is_successful) {
- const stored: unknown = await getJobFromGCS(agent.id);
- if (isStoredAgentResult(stored)) {
- data = stored.data ?? null;
- message = stored.message ?? undefined;
- suggestions = stored.suggestions;
- pendingApproval = stored.pendingApproval;
- exchange = stored.exchange;
- } else {
- data = stored;
- }
- }
-
return res.status(200).json({
success: true,
- status: !agent
- ? "processing"
- : agent.is_successful
- ? "completed"
- : "failed",
- error: agent?.error || undefined,
- data,
+ status: agent.status === "success" ? "completed" : agent.status,
+ error: agent.error,
+ data: agent.data,
model,
effort,
threadId: thread.threadId,
threadTurn: thread.threadTurn,
mode: thread.mode,
- message,
- suggestions,
- pendingApproval,
- exchange,
+ message: agent.message,
+ suggestions: agent.suggestions as AgentSuggestion[] | undefined,
+ pendingApproval: agent.pendingApproval as AgentPendingApproval | undefined,
+ exchange: agent.exchange as AgentExchangeSummary | undefined,
expiresAt: new Date(access.expiresAtMs).toISOString(),
- creditsUsed: agent?.credits_cost,
+ creditsUsed: agent.creditsUsed,
});
}
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index a78e4e91e..89ed86563 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -6,15 +6,13 @@ import {
getExtractExpiry,
getExtractResult,
} from "../../lib/extract/extract-redis";
-import {
- supabaseGetAgentByIdDirect,
- supabaseGetExtractByIdDirect,
-} from "../../lib/supabase-jobs";
+import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";
import { logger as _logger } from "../../lib/logger";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { getExtractJobAccess } from "../../lib/operational-job-access";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
+import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";
async function getExtractData(id: string): Promise<any> {
// Try GCS first if configured
@@ -52,24 +50,15 @@ export async function extractStatusController(
}
if (access.kind === "agent") {
- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
-
- let data: any = undefined;
- if (agent?.is_successful) {
- data = await getJobFromGCS(agent.id);
- }
+ const agent = await getExtractV3AgentStatus(req.params.jobId);
return res.status(200).json({
success: true,
- status: !agent
- ? "processing"
- : agent.is_successful
- ? "completed"
- : "failed",
- error: agent?.error || undefined,
- data,
+ status: agent.status === "success" ? "completed" : agent.status,
+ error: agent.error,
+ data: agent.data,
expiresAt: new Date(access.expiresAtMs).toISOString(),
- creditsUsed: agent?.credits_cost,
+ creditsUsed: agent.creditsUsed,
});
}
}
diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.ts
new file mode 100644
index 000000000..36fa2aded
--- /dev/null
+++ b/apps/api/src/lib/extract-v3-status.ts
@@ -0,0 +1,45 @@
+import { config } from "../config";
+
+export type ExtractV3AgentStatus = {
+ success: true;
+ id: string;
+ status: "processing" | "success" | "failed";
+ data?: unknown;
+ error?: string;
+ model?: "spark-1-pro" | "spark-1-mini" | "spark-2";
+ effort?: "low" | "medium" | "high";
+ threadId?: string;
+ threadTurn?: number;
+ mode?: "extract" | "chat";
+ message?: string;
+ suggestions?: unknown[];
+ pendingApproval?: unknown;
+ exchange?: unknown;
+ creditsUsed?: number;
+};
+
+export async function getExtractV3AgentStatus(
+ id: string,
+): Promise<ExtractV3AgentStatus> {
+ if (!config.EXTRACT_V3_BETA_URL) {
+ throw new Error("Agent beta is not enabled.");
+ }
+
+ const url = new URL(
+ `${config.EXTRACT_V3_BETA_URL.replace(/\/$/, "")}/internal/extracts/${encodeURIComponent(id)}`,
+ );
+ const response = await fetch(url, {
+ headers: {
+ Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,
+ },
+ signal: AbortSignal.timeout(10_000),
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Agent status service returned ${response.status}: ${await response.text()}`,
+ );
+ }
+
+ return (await response.json()) as ExtractV3AgentStatus;
+}
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
index 06e9ac93b..e75bdc9e5 100644
--- a/apps/api/src/lib/supabase-jobs.ts
+++ b/apps/api/src/lib/supabase-jobs.ts
@@ -113,18 +113,3 @@ export const supabaseGetCrawlRequestById = async (requestId: string) => {
.limit(1);
return data ?? null;
};
-
-export const supabaseGetAgentByIdDirect = async (
- agentId: string,
-): Promise<any> => {
- try {
- const [data] = await db
- .select()
- .from(schema.agents)
- .where(eq(schema.agents.id, agentId))
- .limit(1);
- return data ?? null;
- } catch (error) {
- return null;
- }
-};