Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): remove deprecated integration controllers (#3528)
Erik Engervall8fe0dd8e8146
diff --git a/apps/api/src/controllers/v0/admin/create-user.ts b/apps/api/src/controllers/v0/admin/create-user.tsdeleted file mode 100644index 10d21b764..000000000--- a/apps/api/src/controllers/v0/admin/create-user.ts+++ /dev/null@@ -1,416 +0,0 @@-import { logger as _logger, logger } from "../../../lib/logger";-import { Request, Response } from "express";-import { supabase_service } from "../../../services/supabase";-import crypto from "crypto";-import { z } from "zod";-import { apiKeyToFcApiKey } from "../../../lib/parseApi";-import {- autumnService,- isAutumnEnabled,-} from "../../../services/autumn/autumn.service";--async function addCoupon(teamId: string, integration: any) {- if (!integration.coupon_credits) {- return;- }-- const expiresAt = integration.coupon_expiry_ms- ? new Date(Date.now() + integration.coupon_expiry_ms).toISOString()- : null;-- const { error } = await supabase_service.from("coupons").insert({- team_id: teamId,- credits: integration.coupon_credits,- status: "active",- from_auto_recharge: false,- initial_credits: integration.coupon_credits,- code: integration.coupon_code,- is_extract: false,- expires_at: expiresAt,- });-- if (error) {- throw error;- }-- if (integration.coupon_rate_limits) {- const { error: overrideError } = await (supabase_service as any)- .from("team_overrides")- .insert({- team_id: teamId,- rate_limits: integration.coupon_rate_limits,- expires_at: expiresAt,- internal_comment: `Integration coupon (${integration.display_name || integration.slug || "unknown"})`,- });-- if (overrideError) {- throw overrideError;- }- }-- if (integration.coupon_concurrency) {- // Look up the org for this team (created by handle_new_user trigger)- const { data: teamWithOrg } = await supabase_service- .from("teams")- .select("org_id")- .eq("id", teamId)- .single();-- if (teamWithOrg?.org_id) {- const { error: orgOverrideError } = await (supabase_service as any)- .from("organization_overrides")- .insert({- org_id: teamWithOrg.org_id,- concurrent_browsers: integration.coupon_concurrency,- expires_at: expiresAt,- internal_comment: `Integration coupon (${integration.display_name || integration.slug || "unknown"})`,- });-- if (orgOverrideError) {- throw orgOverrideError;- }- }- }-}--/**- * Generates a synthetic email for partner integrations that don't provide real emails.- * Format: <externalUserId>@<integration-slug>.partner.firecrawl.dev- */-function generateSyntheticEmail(- externalUserId: string,- integrationSlug: string,-): string {- // Sanitize the externalUserId to be email-safe (alphanumeric, dots, hyphens, underscores)- const sanitizedId = externalUserId.replace(/[^a-zA-Z0-9._-]/g, "_");- return `${sanitizedId}@${integrationSlug}.partner.firecrawl.dev`;-}--export async function integCreateUserController(req: Request, res: Response) {- let logger = _logger.child({- module: "v0/admin/create-user",- method: "createUserController",- });-- try {- const auth = req.headers.authorization;- if (!auth) {- return res.status(401).json({ error: "Unauthorized" });- }-- const [type, token] = auth.split(" ");- if (type !== "Bearer") {- return res.status(401).json({ error: "Unauthorized" });- }-- // sha-256 hash the token- if (!token) {- return res.status(401).json({ error: "Unauthorized" });- }- const hashedToken = crypto.createHash("sha256").update(token).digest("hex");-- // Look up integration by key- const { data: integration, error: integrationError } =- await supabase_service- .from("user_referring_integration")- .select("*")- .eq("key", hashedToken)- .single();-- if (integrationError || !integration) {- return res.status(401).json({ error: "Unauthorized" });- }-- logger = logger.child({- integration: integration.slug,- });-- // Accept either email OR externalUserId (for partners that don't share emails)- const bodySchema = z- .object({- email: z.string().email().optional(),- externalUserId: z.string().min(1).optional(),- })- .refine(data => data.email || data.externalUserId, {- message: "Either email or externalUserId must be provided",- });-- const body = bodySchema.parse(req.body);-- // Determine the email to use- const email = body.email- ? body.email- : generateSyntheticEmail(body.externalUserId!, integration.slug);-- const { data: preexistingUser, error: preexistingUserError } =- await supabase_service- .from("users")- .select("*")- .eq("email", email)- .limit(1);- if (preexistingUserError) {- logger.error("Failed to look up preexisting user", {- error: preexistingUserError,- });- return res- .status(500)- .json({ error: "Failed to look up preexisting user" });- }-- let teamId: string;- let apiKey: string;- let alreadyExisted = false;-- if (preexistingUser.length > 0) {- const { data: userTeams, error: userTeamsError } = await supabase_service- .from("user_teams")- .select("*")- .eq("user_id", preexistingUser[0].id);- if (userTeamsError) {- logger.error("Failed to look up user teams", {- error: userTeamsError,- });- return res.status(500).json({ error: "Failed to look up user teams" });- }-- // check if a team of the same referrer already exists- const { data: existingTeam, error: existingTeamError } =- await supabase_service- .from("teams")- .select("*")- .eq("referrer_integration", integration.slug)- .in(- "id",- userTeams.map(team => team.team_id),- )- .limit(1);- if (existingTeamError) {- logger.error("Failed to look up existing team", {- error: existingTeamError,- });- return res- .status(500)- .json({ error: "Failed to look up existing team" });- }-- if (existingTeam.length > 0) {- teamId = existingTeam[0].id;-- const { data: existingApiKey, error: existingApiKeyError } =- await supabase_service- .from("api_keys")- .select("*")- .eq("team_id", teamId)- .limit(1);- if (existingApiKeyError) {- logger.error("Failed to look up existing api key", {- error: existingApiKeyError,- });- return res- .status(500)- .json({ error: "Failed to look up existing api key" });- }-- if (existingApiKey.length > 0) {- apiKey = existingApiKey[0].key;- } else {- return res.status(500).json({- error: "No api key found for existing team with the same referrer",- });- }-- if (isAutumnEnabled()) {- await autumnService.ensureTeamProvisioned({- teamId,- orgId: existingTeam[0].org_id,- });- }-- alreadyExisted = true;-- logger.info("Found existing team from existing user", {- teamId,- });- } else {- // Create an org first (mirrors handle_new_user trigger)- const { data: newOrg, error: newOrgError } = await supabase_service- .from("organizations")- .insert({ name: "pending" })- .select()- .single();- if (newOrgError) {- logger.error("Failed to create organization", { error: newOrgError });- return res- .status(500)- .json({ error: "Failed to create organization" });- }-- // create a new team with this referrer, linked to the org- const { data: newTeam, error: newTeamError } = await supabase_service- .from("teams")- .insert({- name: "via " + (integration.display_name ?? integration.slug),- referrer_integration: integration.slug,- org_id: newOrg.id,- } as any)- .select()- .single();- if (newTeamError) {- logger.error("Failed to create new team", { error: newTeamError });- return res.status(500).json({ error: "Failed to create new team" });- }- teamId = newTeam.id;-- // Update org name to match team ID (convention)- const { error: orgNameError } = await supabase_service- .from("organizations")- .update({ name: teamId })- .eq("id", newOrg.id);- if (orgNameError) {- logger.warn("Failed to update org name to team ID", {- error: orgNameError,- orgId: newOrg.id,- teamId,- });- }-- // Link team to org in join table- const { error: orgLinkError } = await supabase_service- .from("organization_teams")- .insert({- org_id: newOrg.id,- team_id: teamId,- });- if (orgLinkError) {- logger.error("Failed to link team to organization", {- error: orgLinkError,- });- return res- .status(500)- .json({ error: "Failed to link team to organization" });- }-- if (isAutumnEnabled()) {- await autumnService.ensureTeamProvisioned({- teamId,- orgId: newOrg.id,- });- }-- const { error: newUserTeamError } = await supabase_service- .from("user_teams")- .insert({- user_id: preexistingUser[0].id,- team_id: teamId,- });-- if (newUserTeamError) {- logger.error("Failed to add user to team", {- error: newUserTeamError,- });- return res.status(500).json({ error: "Failed to add user to team" });- }-- const { data: newApiKey, error: newApiKeyError } =- await supabase_service- .from("api_keys")- .insert({- name: "Default",- team_id: teamId,- owner_id: preexistingUser[0].id,- })- .select()- .single();- if (newApiKeyError) {- logger.error("Failed to create new api key", {- error: newApiKeyError,- });- return res- .status(500)- .json({ error: "Failed to create new api key" });- }- apiKey = newApiKey.key;-- await addCoupon(teamId, integration);-- logger.info("Created new team from existing user", {- teamId,- });- }- } else {- const { data: newUser, error: newUserError } =- await supabase_service.auth.admin.createUser({- email: email,- email_confirm: true,- user_metadata: {- referrer_integration: integration.slug,- ...(body.externalUserId && {- external_user_id: body.externalUserId,- }),- },- });-- if (newUserError) {- logger.error("Failed to create user", { error: newUserError });- return res.status(500).json({ error: "Failed to create user" });- }-- const { data: newUserFc, error: newUserFcError } = await supabase_service- .from("users")- .select("*")- .eq("id", newUser.user.id)- .single();- if (newUserFcError || !newUserFc) {- logger.error("Failed to look up new user", { error: newUserFcError });- return res.status(500).json({ error: "Failed to look up new user" });- }-- teamId = newUserFc.team_id;-- const { data: apiKeyFc, error: apiKeyFcError } = await supabase_service- .from("api_keys")- .select("*")- .eq("team_id", teamId)- .single();- if (apiKeyFcError || !apiKeyFc) {- logger.error("Failed to look up api key", { error: apiKeyFcError });- return res.status(500).json({ error: "Failed to look up api key" });- }-- apiKey = apiKeyFc.key;-- if (isAutumnEnabled()) {- // Look up org_id for the trigger-created team so ensureTeamProvisioned- // can evaluate the stable percent gate without a redundant query.- const { data: triggerTeam } = await supabase_service- .from("teams")- .select("org_id")- .eq("id", teamId)- .single();-- await autumnService.ensureTeamProvisioned({- teamId,- orgId: triggerTeam?.org_id ?? undefined,- });- }-- await addCoupon(teamId, integration);-- logger.info("Created new user from scratch", {- teamId,- });- }-- return res.status(200).json({- apiKey: apiKeyToFcApiKey(apiKey),- alreadyExisted,- });- } catch (error) {- if (error instanceof z.ZodError) {- return res.status(400).json({ error: error.message });- } else {- logger.error("Failed to create user", { error });- return res.status(500).json({ error: "Internal server error" });- }- }-}diff --git a/apps/api/src/controllers/v0/admin/rotate-api-key.ts b/apps/api/src/controllers/v0/admin/rotate-api-key.tsdeleted file mode 100644index c96086aa9..000000000--- a/apps/api/src/controllers/v0/admin/rotate-api-key.ts+++ /dev/null@@ -1,150 +0,0 @@-import { logger as _logger } from "../../../lib/logger";-import { Request, Response } from "express";-import { supabase_service } from "../../../services/supabase";-import crypto from "crypto";-import { z } from "zod";-import { validate as isUuid } from "uuid";-import { parseApi, apiKeyToFcApiKey } from "../../../lib/parseApi";-import { clearACUC } from "../../auth";--export async function integRotateApiKeyController(req: Request, res: Response) {- let logger = _logger.child({- module: "v0/admin/rotate-api-key",- method: "rotateApiKeyController",- });-- try {- const auth = req.headers.authorization;- if (!auth) {- return res.status(401).json({ error: "Unauthorized" });- }-- const [type, token] = auth.split(" ");- if (type !== "Bearer") {- return res.status(401).json({ error: "Unauthorized" });- }-- if (!token) {- return res.status(401).json({ error: "Unauthorized" });- }- const hashedToken = crypto.createHash("sha256").update(token).digest("hex");-- // Look up integration by key- const { data: integration, error: integrationError } =- await supabase_service- .from("user_referring_integration")- .select("*")- .eq("key", hashedToken)- .single();-- if (integrationError || !integration) {- return res.status(401).json({ error: "Unauthorized" });- }-- logger = logger.child({- integration: integration.slug,- });-- const bodySchema = z.object({- apiKey: z.string(),- });-- const body = bodySchema.parse(req.body);-- const normalizedApiKey = parseApi(body.apiKey);-- if (!isUuid(normalizedApiKey)) {- return res.status(400).json({ error: "API key is invalid" });- }-- const { data: apiKeyData, error: apiKeyError } = await supabase_service- .from("api_keys")- .select("*")- .eq("key", normalizedApiKey)- .limit(1);-- if (apiKeyError) {- throw apiKeyError;- }-- if (!apiKeyData || apiKeyData.length === 0) {- return res.status(404).json({ error: "API key not found" });- }-- const teamId = apiKeyData[0].team_id;- const ownerId = apiKeyData[0].owner_id;-- logger = logger.child({ teamId });-- const { data: teamData, error: teamError } = await supabase_service- .from("teams")- .select("*")- .eq("id", teamId)- .limit(1);-- if (teamError) {- throw teamError;- }-- if (!teamData || teamData.length === 0) {- return res.status(404).json({ error: "API key not found" });- }-- const team = teamData[0];-- if (team.referrer_integration !== integration.slug) {- return res.status(404).json({ error: "API key not found" });- }-- const { data: newApiKey, error: newApiKeyError } = await supabase_service- .from("api_keys")- .insert({- name: "Default",- team_id: teamId,- owner_id: ownerId,- })- .select()- .single();-- if (newApiKeyError) {- logger.error("Failed to create new API key", { error: newApiKeyError });- return res.status(500).json({ error: "Failed to create new API key" });- }-- const { error: deleteError } = await supabase_service- .from("api_keys")- .delete()- .eq("key", normalizedApiKey)- .eq("team_id", teamId)- .eq("owner_id", ownerId);-- if (deleteError) {- logger.error("Failed to delete leaked API key", { error: deleteError });- logger.warn("Old API key may still be active", {- oldKey: normalizedApiKey,- });- }-- // Clear ACUC Redis cache so the deleted key stops authenticating immediately- try {- await clearACUC(normalizedApiKey);- } catch (cacheError) {- logger.warn("Failed to clear ACUC cache for rotated key", {- error: cacheError,- });- }-- logger.info("Rotated API key", { teamId });-- return res.status(200).json({- apiKey: apiKeyToFcApiKey(newApiKey.key),- });- } catch (error) {- if (error instanceof z.ZodError) {- return res.status(400).json({ error: error.message });- } else {- logger.error("Failed to rotate API key", { error });- return res.status(500).json({ error: "Internal server error" });- }- }-}diff --git a/apps/api/src/controllers/v0/admin/validate-api-key.ts b/apps/api/src/controllers/v0/admin/validate-api-key.tsdeleted file mode 100644index 0604721bf..000000000--- a/apps/api/src/controllers/v0/admin/validate-api-key.ts+++ /dev/null@@ -1,158 +0,0 @@-import { logger as _logger } from "../../../lib/logger";-import { Request, Response } from "express";-import { supabase_service } from "../../../services/supabase";-import crypto from "crypto";-import { z } from "zod";-import { validate as isUuid } from "uuid";-import { parseApi } from "../../../lib/parseApi";--/**- * Extracts the external user ID from a synthetic email if it matches the pattern.- * Returns null if the email is not a synthetic one for this integration.- */-function extractExternalUserId(- email: string,- integrationSlug: string,-): string | null {- const syntheticDomain = `@${integrationSlug}.partner.firecrawl.dev`;- if (email.endsWith(syntheticDomain)) {- return email.slice(0, -syntheticDomain.length);- }- return null;-}--export async function integValidateApiKeyController(- req: Request,- res: Response,-) {- let logger = _logger.child({- module: "v0/admin/validate-api-key",- method: "validateApiKeyController",- });-- try {- const auth = req.headers.authorization;- if (!auth) {- return res.status(401).json({ error: "Unauthorized" });- }-- const [type, token] = auth.split(" ");- if (type !== "Bearer") {- return res.status(401).json({ error: "Unauthorized" });- }-- // sha-256 hash the token- if (!token) {- return res.status(401).json({ error: "Unauthorized" });- }- const hashedToken = crypto.createHash("sha256").update(token).digest("hex");-- // Look up integration by key- const { data: integration, error: integrationError } =- await supabase_service- .from("user_referring_integration")- .select("*")- .eq("key", hashedToken)- .single();-- if (integrationError || !integration) {- return res.status(401).json({ error: "Unauthorized" });- }-- logger = logger.child({- integration: integration.slug,- });-- const bodySchema = z.object({- apiKey: z.string(),- });-- const body = bodySchema.parse(req.body);-- const normalizedApiKey = parseApi(body.apiKey);-- if (!isUuid(normalizedApiKey)) {- return res.status(400).json({ error: "API key is invalid" });- }-- const { data: apiKeyData, error: apiKeyError } = await supabase_service- .from("api_keys")- .select("*")- .eq("key", normalizedApiKey)- .limit(1);-- if (apiKeyError) {- throw apiKeyError;- }-- if (!apiKeyData || apiKeyData.length === 0) {- return res.status(404).json({ error: "API key not identifiable" });- }-- const teamId = apiKeyData[0].team_id;-- logger = logger.child({- teamId,- });-- const { data: teamData, error: teamError } = await supabase_service- .from("teams")- .select("*")- .eq("id", teamId)- .limit(1);- if (teamError) {- throw teamError;- }-- if (!teamData || teamData.length === 0) {- return res.status(404).json({ error: "API key not identifiable" });- }-- const team = teamData[0];-- if (team.referrer_integration !== integration.slug) {- return res.status(404).json({ error: "API key not identifiable" });- }-- const { data: userTeams, error: userTeamsError } = await supabase_service- .from("user_teams")- .select("*")- .eq("team_id", teamId)- .limit(1);- if (userTeamsError) {- throw userTeamsError;- }-- if (!userTeams || userTeams.length === 0) {- throw new Error("user_teams in invalid state");- }-- const userId = userTeams[0].user_id;-- const { data: userData, error: userError } = await supabase_service- .from("users")- .select("*")- .eq("id", userId)- .limit(1);- if (userError) {- throw userError;- }-- if (!userData || userData.length === 0) {- throw new Error("users in invalid state");- }-- const user = userData[0];-- const externalUserId = extractExternalUserId(user.email, integration.slug);-- return res.status(200).json({- teamName: team.name,- email: user.email,- ...(externalUserId && { externalUserId }),- });- } catch (error) {- logger.error("Error validating API key", { error });- return res.status(500).json({ error: "Internal server error" });- }-}diff --git a/apps/api/src/controllers/v2/agent-signup-confirm.ts b/apps/api/src/controllers/v2/agent-signup-confirm.tsdeleted file mode 100644index 9ae8ca839..000000000--- a/apps/api/src/controllers/v2/agent-signup-confirm.ts+++ /dev/null@@ -1,334 +0,0 @@-import { Request, Response } from "express";-import { RateLimiterRedis } from "rate-limiter-flexible";-import { z } from "zod";-import { logger as _logger } from "../../lib/logger";-import {- clearAgentSponsorCache,- getAgentSponsorByToken,- markSponsorBlocked,- markSponsorVerified,-} from "../../services/agent-sponsor";-import { redisRateLimitClient } from "../../services/rate-limiter";-import { supabase_rr_service, supabase_service } from "../../services/supabase";-import { clearACUC } from "../auth";--const confirmBlockRateLimiter = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "agent_signup_confirm_ip",- points: 10,- duration: 3600, // 1 hour-});--const agentSignupTokenSchema = z.object({- agent_signup_token: z.string().min(1),-});--/**- * POST /v2/agent-signup/confirm- * Confirms a pending agent sponsor, merging the sandboxed key into the sponsor's real account.- */-export async function agentSignupConfirmController(- req: Request,- res: Response,-) {- const logger = _logger.child({- module: "v2/agent-signup-confirm",- method: "agentSignupConfirmController",- });-- try {- const incomingIP = req.ip || req.socket.remoteAddress || "unknown";- try {- await confirmBlockRateLimiter.consume(incomingIP);- } catch {- return res.status(429).json({- success: false,- error: "Too many attempts. Please try again later.",- });- }-- const { agent_signup_token } = agentSignupTokenSchema.parse(req.body);-- const sponsor = await getAgentSponsorByToken({ agent_signup_token });- if (!sponsor) {- return res.status(404).json({- success: false,- error: "Invalid or expired verification token.",- });- }-- if (sponsor.status === "verified") {- return res.status(200).json({- success: true,- message: "This agent key has already been confirmed.",- });- }-- if (sponsor.status === "blocked") {- return res.status(403).json({- success: false,- error: "This agent signup has been blocked.",- });- }-- // Check deadline- const deadline = new Date(sponsor.verification_deadline);- if (deadline < new Date()) {- return res.status(403).json({- success: false,- error:- "Verification deadline has passed. Please log in to manage your account.",- login_url: "https://firecrawl.dev/signin",- });- }-- // Look up existing user by sponsor email- const { data: existingUser } = await supabase_rr_service- .from("users")- .select("id, team_id")- .eq("email", sponsor.email)- .limit(1);-- if (existingUser && existingUser.length > 0) {- // Merge: move the agent API key from the sandboxed team to the existing user's team- const realTeamId = existingUser[0].team_id;- const realUserId = existingUser[0].id;-- // Move the API key to the real team- const { error: moveKeyError } = await supabase_service- .from("api_keys")- .update({ team_id: realTeamId, owner_id: realUserId } as any)- .eq("id", sponsor.api_key_id);-- if (moveKeyError) {- logger.error("Failed to move API key to real team", {- error: moveKeyError,- });- return res- .status(500)- .json({ success: false, error: "Failed to confirm agent key." });- }-- // Carry over credit_usage from sandboxed team to real team- const { error: creditMoveError } = await supabase_service- .from("credit_usage")- .update({ team_id: realTeamId } as any)- .eq("team_id", sponsor.sandboxed_team_id);-- if (creditMoveError) {- logger.warn("Failed to carry over credit usage from sandboxed team", {- error: creditMoveError,- sandboxedTeamId: sponsor.sandboxed_team_id,- realTeamId,- });- // Non-fatal: continue with merge- }-- // Ban the sandboxed team to deactivate it- const { error: banError } = await supabase_service- .from("teams")- .update({ banned: true })- .eq("id", sponsor.sandboxed_team_id);-- if (banError) {- logger.warn("Failed to ban sandboxed team", { error: banError });- }-- logger.info("Agent key merged into existing account", {- email: sponsor.email,- realTeamId,- sandboxedTeamId: sponsor.sandboxed_team_id,- apiKeyId: sponsor.api_key_id,- });- } else {- // No existing user: the sandboxed account becomes the real account- // Update the auth.users email from synthetic to the real sponsor email- const { data: sandboxedTeamUsers } = await supabase_service- .from("users")- .select("id")- .eq("team_id", sponsor.sandboxed_team_id)- .limit(1);-- if (sandboxedTeamUsers && sandboxedTeamUsers.length > 0) {- const userId = sandboxedTeamUsers[0].id;-- // Update auth.users email via admin API- const { error: updateAuthError } =- await supabase_service.auth.admin.updateUserById(userId, {- email: sponsor.email,- });-- if (updateAuthError) {- logger.error("Failed to update auth user email", {- error: updateAuthError,- });- return res- .status(500)- .json({ success: false, error: "Failed to confirm agent key." });- }-- // Update public.users email- const { error: updateUserError } = await supabase_service- .from("users")- .update({ email: sponsor.email })- .eq("id", userId);-- if (updateUserError) {- logger.warn("Failed to update public.users email", {- error: updateUserError,- });- }- }-- logger.info("Sandboxed account promoted to real account", {- email: sponsor.email,- sandboxedTeamId: sponsor.sandboxed_team_id,- apiKeyId: sponsor.api_key_id,- });- }-- // Mark sponsor as verified and clear sponsor cache before clearing ACUC,- // so that when ACUC is rebuilt it sees verified status (no sandbox cap).- await markSponsorVerified({ sponsorId: sponsor.id });- await clearAgentSponsorCache({ apiKeyId: sponsor.api_key_id });-- // Clear ACUC so the key picks up the new plan / verified status- const { data: apiKeyData } = await supabase_service- .from("api_keys")- .select("key")- .eq("id", sponsor.api_key_id)- .single();-- if (apiKeyData) {- await clearACUC(apiKeyData.key);- }-- return res.status(200).json({- success: true,- message: "Agent key confirmed and linked to your account.",- });- } catch (error) {- if (error instanceof z.ZodError) {- return res.status(400).json({- success: false,- error: "Invalid request: agent_signup_token is required.",- });- }- logger.error("Unexpected error in agent signup confirm", { error });- return res- .status(500)- .json({ success: false, error: "Internal server error" });- }-}--/**- * POST /v2/agent-signup/block- * Blocks an agent sponsor, disabling the sandboxed key.- */-export async function agentSignupBlockController(req: Request, res: Response) {- const logger = _logger.child({- module: "v2/agent-signup-block",- method: "agentSignupBlockController",- });-- try {- const incomingIP = req.ip || req.socket.remoteAddress || "unknown";- try {- await confirmBlockRateLimiter.consume(incomingIP);- } catch {- return res.status(429).json({- success: false,- error: "Too many attempts. Please try again later.",- });- }-- const { agent_signup_token } = agentSignupTokenSchema.parse(req.body);-- const sponsor = await getAgentSponsorByToken({ agent_signup_token });- if (!sponsor) {- return res.status(404).json({- success: false,- error: "Invalid verification token.",- });- }-- if (sponsor.status === "blocked") {- return res.status(200).json({- success: true,- message: "This agent signup has already been blocked.",- });- }-- if (sponsor.status === "verified") {- return res.status(409).json({- success: false,- error:- "This agent key has already been confirmed and cannot be blocked.",- });- }-- // Fetch the API key value before deletion so we can clear ACUC cache- const { data: apiKeyData } = await supabase_service- .from("api_keys")- .select("key")- .eq("id", sponsor.api_key_id)- .single();-- // Disable the API key- const { error: deleteKeyError } = await supabase_service- .from("api_keys")- .delete()- .eq("id", sponsor.api_key_id);-- if (deleteKeyError) {- logger.warn("Failed to delete agent API key", { error: deleteKeyError });- }-- // Clear ACUC Redis cache so the deleted key stops authenticating immediately- if (apiKeyData) {- try {- await clearACUC(apiKeyData.key);- } catch (cacheError) {- logger.warn("Failed to clear ACUC cache for blocked agent key", {- error: cacheError,- });- }- }-- // Ban the sandboxed team- const { error: banError } = await supabase_service- .from("teams")- .update({ banned: true })- .eq("id", sponsor.sandboxed_team_id);-- if (banError) {- logger.warn("Failed to ban sandboxed team", { error: banError });- }-- // Mark sponsor as blocked- await markSponsorBlocked({ sponsorId: sponsor.id });- await clearAgentSponsorCache({ apiKeyId: sponsor.api_key_id });-- logger.info("Agent signup blocked", {- email: sponsor.email,- sandboxedTeamId: sponsor.sandboxed_team_id,- apiKeyId: sponsor.api_key_id,- });-- return res.status(200).json({- success: true,- message: "Agent key has been blocked and disabled.",- });- } catch (error) {- if (error instanceof z.ZodError) {- return res.status(400).json({- success: false,- error: "Invalid request: agent_signup_token is required.",- });- }- logger.error("Unexpected error in agent signup block", { error });- return res- .status(500)- .json({ success: false, error: "Internal server error" });- }-}diff --git a/apps/api/src/controllers/v2/agent-signup.ts b/apps/api/src/controllers/v2/agent-signup.tsdeleted file mode 100644index e320c173e..000000000--- a/apps/api/src/controllers/v2/agent-signup.ts+++ /dev/null@@ -1,487 +0,0 @@-import crypto from "crypto";-import { Request, Response } from "express";-import qs from "qs";-import { RateLimiterRedis } from "rate-limiter-flexible";-import { Resend } from "resend";-import { z } from "zod";-import { config } from "../../config";-import { logger as _logger } from "../../lib/logger";-import { apiKeyToFcApiKey } from "../../lib/parseApi";-import { redisRateLimitClient } from "../../services/rate-limiter";-import { supabase_rr_service, supabase_service } from "../../services/supabase";--const PUBLIC_EMAIL_DOMAINS = new Set([- "gmail.com",- "googlemail.com",- "outlook.com",- "hotmail.com",- "live.com",- "yahoo.com",- "yahoo.co.uk",- "icloud.com",- "me.com",- "mac.com",- "aol.com",- "protonmail.com",- "proton.me",- "mail.com",- "zoho.com",- "yandex.com",- "gmx.com",- "gmx.net",- "tutanota.com",- "fastmail.com",-]);--const GMAIL_DOMAINS = new Set(["gmail.com", "googlemail.com"]);--/**- * Normalize a public-provider email for rate limiting so that alias tricks- * (dots in Gmail, +suffix in most providers) all map to the same bucket.- * Only used for rate-limit keys β the original email is stored in the DB.- */-function normalizeEmailForRateLimit(email: string, domain: string): string {- let [local] = email.split("@");-- // Strip +suffix (supported by Gmail, Outlook, Proton, Fastmail, etc.)- const plusIdx = local.indexOf("+");- if (plusIdx !== -1) {- local = local.slice(0, plusIdx);- }-- // Gmail/Googlemail also ignores dots in the local part- if (GMAIL_DOMAINS.has(domain)) {- local = local.replace(/\./g, "");- }-- return `${local}@${domain}`;-}--// Rate limit values β used by limiters and error copy so they stay in sync-const AGENT_SIGNUP_IP_LIMIT = 1;-const AGENT_SIGNUP_DOMAIN_LIMIT = 20;-const AGENT_SIGNUP_IP_LIMIT_SIDEGUIDE = 9; // 3x default-const AGENT_SIGNUP_DOMAIN_LIMIT_SIDEGUIDE = 60; // 3x default--// Rate limiters-const ipRateLimiter = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "agent_signup_ip",- points: AGENT_SIGNUP_IP_LIMIT,- duration: 86400, // 24 hours-});--// Per-domain (or per-email for public providers) limit to curb abuse while allowing-// legitimate multi-agent signups. Tune points if product requirements change.-const domainRateLimiter = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "agent_signup_domain",- points: AGENT_SIGNUP_DOMAIN_LIMIT,- duration: 86400, // 24 hours-});--// Higher limits for *+test*@sideguide.dev only. sideguide.dev is internal-only; external users-// cannot receive mail or hold accounts there, so this path is not abusable.-const ipRateLimiterSideguide = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "agent_signup_ip_sideguide",- points: AGENT_SIGNUP_IP_LIMIT_SIDEGUIDE,- duration: 86400, // 24 hours-});--const domainRateLimiterSideguide = new RateLimiterRedis({- storeClient: redisRateLimitClient,- keyPrefix: "agent_signup_domain_sideguide",- points: AGENT_SIGNUP_DOMAIN_LIMIT_SIDEGUIDE,- duration: 86400, // 24 hours-});--const agentSignupSchema = z.object({- email: z- .string()- .email()- .refine(- e =>- !e.includes("+") ||- (e.endsWith("@sideguide.dev") && e.includes("+test")),- {- message: "Email addresses with '+' are not allowed for agent signup.",- },- ),- agent_name: z.string().min(1).max(100),- accept_terms: z.literal(true, {- message:- "You must accept the terms here. https://www.firecrawl.dev/terms-of-service",- }),-});--/** Insert payload for agent_sponsors (nullable cols in DB are optional here). */-type AgentSponsorInsert = {- email: string;- status: "pending" | "blocked" | "verified";- verification_deadline: string;- agent_name: string;- sandboxed_team_id: string;- api_key_id: number;- verification_token: string;-- requesting_ip?: string;- tos_version?: string;- tos_hash?: string;-};--export async function agentSignupController(req: Request, res: Response) {- const logger = _logger.child({- module: "v2/agent-signup",- method: "agentSignupController",- });-- try {- // Parse and validate input- const body = agentSignupSchema.parse(req.body);- const email = body.email.toLowerCase();- const { agent_name } = body;-- const incomingIP = req.ip || req.socket.remoteAddress || "unknown";- const [emailPrefix, emailDomain] = email.split("@");- // sideguide.dev is an internal domain: only Sideguide team have mailboxes there. Even if- // someone used this pattern to get the higher limits, each signup still gets only 50 credits- // and keys stay sandboxed (no confirm/merge); limits are 3x default, not unbounded.- const isSideguideEmail =- emailDomain === "sideguide.dev" && emailPrefix.includes("+test");-- // Always rate limit; use higher limits only for internal sideguide.dev +test addresses- const ipLimiter = isSideguideEmail ? ipRateLimiterSideguide : ipRateLimiter;- const domainLimiter = isSideguideEmail- ? domainRateLimiterSideguide- : domainRateLimiter;- const ipLimitMsg = isSideguideEmail- ? `Rate limit exceeded. Maximum ${AGENT_SIGNUP_IP_LIMIT_SIDEGUIDE} agent signup requests per day per IP for sideguide test emails.`- : `Rate limit exceeded. Maximum ${AGENT_SIGNUP_IP_LIMIT} agent signup requests per day per IP.`;- const domainLimitMsg = isSideguideEmail- ? "Too many agent signups for this email. Please try again later."- : "Too many agent signups for this email domain. Please try again later.";-- try {- await ipLimiter.consume(incomingIP);- } catch {- return res.status(429).json({- success: false,- error: ipLimitMsg,- });- }-- const domainKey = PUBLIC_EMAIL_DOMAINS.has(emailDomain)- ? normalizeEmailForRateLimit(email, emailDomain)- : emailDomain;- try {- await domainLimiter.consume(domainKey);- } catch {- return res.status(429).json({- success: false,- error: domainLimitMsg,- });- }-- // Check for existing blocked sponsor record (use primary for strong consistency)- const { data: blockedSponsor } = await supabase_service- .from("agent_sponsors")- .select("id")- .eq("email", email)- .eq("status", "blocked")- .limit(1);-- if (blockedSponsor && blockedSponsor.length > 0) {- return res.status(403).json({- success: false,- error: "This email has blocked agent signups.",- });- }-- // Check for existing pending sponsor record (use primary for strong consistency)- const { data: pendingSponsor } = await supabase_service- .from("agent_sponsors")- .select("id, verification_deadline")- .eq("email", email)- .eq("status", "pending")- .limit(1);-- if (pendingSponsor && pendingSponsor.length > 0) {- const deadline = new Date(pendingSponsor[0].verification_deadline);- if (deadline > new Date()) {- return res.status(409).json({- success: false,- error:- "A pending agent signup confirmation has already been sent to this email.",- login_url: "https://firecrawl.dev/signin",- });- } else {- return res.status(403).json({- success: false,- error:- "Previous agent signup verification has expired. Please log in to manage your account.",- login_url: "https://firecrawl.dev/signin",- });- }- }-- // Create sandboxed account via Supabase auth- const sandboxId = crypto.randomUUID();- const syntheticEmail = `agent-${sandboxId}@agent.sandbox.firecrawl.dev`;-- const { data: newUser, error: newUserError } =- await supabase_service.auth.admin.createUser({- email: syntheticEmail,- email_confirm: true,- user_metadata: {- referrer_integration: "agent_signup",- agent_name: agent_name,- },- });-- if (newUserError) {- logger.error("Failed to create sandboxed user", { error: newUserError });- return res- .status(500)- .json({ success: false, error: "Failed to create agent account." });- }-- // Wait briefly for the trigger to complete, then fetch the created records- // The handle_new_user_6 trigger creates user, team, org, api_key- let teamId: string | null = null;- let apiKeyRecord: { id: number; key: string } | null = null;-- // Poll for trigger completion (the trigger runs synchronously in the same transaction)- const { data: fcUser, error: fcUserError } = await supabase_service- .from("users")- .select("team_id")- .eq("id", newUser.user.id)- .single();-- if (fcUserError || !fcUser) {- logger.error("Failed to look up sandboxed user after creation", {- error: fcUserError,- });- return res- .status(500)- .json({ success: false, error: "Failed to create agent account." });- }-- teamId = fcUser.team_id;-- const { data: apiKeyData, error: apiKeyError } = await supabase_service- .from("api_keys")- .select("id, key")- .eq("team_id", teamId)- .limit(1)- .single();-- if (apiKeyError || !apiKeyData) {- logger.error("Failed to look up API key for sandboxed team", {- error: apiKeyError,- });- return res- .status(500)- .json({ success: false, error: "Failed to create agent account." });- }-- apiKeyRecord = apiKeyData;-- if (!teamId) {- return res- .status(500)- .json({ success: false, error: "Failed to create agent account." });- }-- // Mark the API key as agent_provisioned- const { error: updateKeyError } = await supabase_service- .from("api_keys")- .update({ agent_provisioned: true } as any)- .eq("id", apiKeyRecord.id);-- if (updateKeyError) {- logger.error("Failed to mark API key as agent_provisioned", {- error: updateKeyError,- });- return res- .status(500)- .json({ success: false, error: "Failed to create agent account." });- }-- // Generate verification token- const verificationToken = crypto.randomBytes(32).toString("hex");-- // Compute deadline (5 days from now)- const deadline = new Date();- deadline.setDate(deadline.getDate() + 5);-- // Create sponsor record- const sponsorRow: AgentSponsorInsert = {- email,- status: "pending",- verification_deadline: deadline.toISOString(),- agent_name,- sandboxed_team_id: teamId,- api_key_id: apiKeyRecord.id,- requesting_ip: incomingIP,- tos_version: "2024-11-05", // Date of last revision in ToS at https://firecrawl.dev/terms-of-service- tos_hash: crypto- .createHash("sha256")- .update("accept_terms:true")- .digest("hex"),- verification_token: verificationToken,- };- const { error: sponsorError } = await supabase_service- .from("agent_sponsors")- .insert(sponsorRow);-- if (sponsorError) {- logger.error("Failed to create sponsor record", { error: sponsorError });- return res- .status(500)- .json({ success: false, error: "Failed to create agent signup." });- }-- // Send confirmation email- const confirmUrl = `https://firecrawl.dev/agent-confirm?${qs.stringify({- agent_signup_token: verificationToken,- agent_signup_action: "confirm",- })}`;- const blockUrl = `https://firecrawl.dev/agent-confirm?${qs.stringify({- agent_signup_token: verificationToken,- agent_signup_action: "block",- })}`;-- if (config.RESEND_API_KEY) {- logger.info("Sending agent sponsor confirmation email", {- to: email,- agent_name,- });- try {- const resend = new Resend(config.RESEND_API_KEY);- const sendResult = await resend.emails.send({- from: "Firecrawl <[email protected]>",- to: [email],- reply_to: "[email protected]",- subject: `An AI agent "${agent_name}" created an API key under your email β Firecrawl`,- html: `- <div style="font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; line-height: 1.6; color: #333; max-width: 600px; margin: 40px auto; padding: 20px;">- <div style="margin-bottom: 30px;">- <img src="https://www.firecrawl.dev/brand/firecrawl-wordmark-500.png" alt="Firecrawl" style="max-width: 150px; height: auto;">- </div>- <p style="margin: 15px 0;">Hey there,</p>- <p style="margin: 15px 0;">An AI agent called <strong>${escapeHtml(agent_name)}</strong> just created a Firecrawl API key and listed your email as the account holder.</p>- <p style="margin: 15px 0;">The key is currently sandboxed with a <strong>50-credit limit</strong>. To link it to your account and unlock your full plan, please confirm:</p>- <p style="margin: 30px 0;">- <a href="${confirmUrl}" style="background-color: #FA5D19; color: white; padding: 12px 30px; text-decoration: none; border-radius: 6px; display: inline-block; font-weight: 600; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif;">Confirm & Link Key</a>- </p>- <p style="margin: 15px 0;">If you did not authorize this, you can block the key:</p>- <p style="margin: 15px 0;"><a href="${blockUrl}" style="color: #FF6B35;">Block this key</a></p>- <p style="margin: 15px 0;">This confirmation link expires on <strong>${deadline.toLocaleDateString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric" })}</strong>.</p>- <p style="margin: 15px 0;">If you have questions, reach out to us at <a href="mailto:[email protected]" style="color: #FF6B35;">[email protected]</a></p>- <p style="margin: 15px 0;">Best,<br>The Firecrawl Team π₯</p>- </div>- `,- });- if (sendResult.data?.id) {- logger.info("Agent sponsor confirmation email sent", {- to: email,- resendId: sendResult.data.id,- });- } else {- logger.warn(- "Agent sponsor confirmation email failed or returned no id",- {- to: email,- error: sendResult.error,- },- );- }- } catch (err) {- logger.error("Failed to send agent sponsor confirmation email", {- to: email,- error: err,- message: err instanceof Error ? err.message : String(err),- });- }- } else {- logger.warn(- "RESEND_API_KEY not set; skipping agent sponsor confirmation email",- {- to: email,- },- );- }-- // If sponsor email matches an existing user, queue in-app notification- const { data: existingUser } = await supabase_rr_service- .from("users")- .select("team_id")- .eq("email", email)- .limit(1);-- if (existingUser && existingUser.length > 0) {- await supabase_service- .from("user_notifications")- .insert({- team_id: existingUser[0].team_id,- notification_type: "agentSponsorConfirm",- sent_date: new Date().toISOString(),- timestamp: new Date().toISOString(),- metadata: {- agent_name,- confirm_url: confirmUrl,- block_url: blockUrl,- verification_token: verificationToken,- deadline: deadline.toISOString(),- },- } as any)- .then(({ error }) => {- if (error) {- logger.error("Failed to insert in-app notification", { error });- }- });- }-- logger.info("Agent signup completed", {- email,- agent_name,- teamId,- apiKeyId: apiKeyRecord.id,- });-- return res.status(201).json({- success: true,- api_key: apiKeyToFcApiKey(apiKeyRecord.key),- sponsor_status: "pending",- credit_limit: 50,- credits_remaining: 50,- verification_deadline_at: deadline.toISOString(),- tos_url: "https://firecrawl.dev/terms-of-service",- });- } catch (error) {- if (error instanceof z.ZodError) {- return res.status(400).json({- success: false,- error:- "Invalid request body: " +- (error as z.ZodError).issues- .map((e: z.ZodIssue) => e.message)- .join(", "),- });- }- logger.error("Unexpected error in agent signup", { error });- return res- .status(500)- .json({ success: false, error: "Internal server error" });- }-}--function escapeHtml(str: string): string {- return str- .replace(/&/g, "&")- .replace(/</g, "<")- .replace(/>/g, ">")- .replace(/"/g, """)- .replace(/'/g, "'");-}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex bddd88f30..4bccf2336 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -89,11 +89,6 @@ adminRouter.post(wrap(crawlMonitorController),);-// TODO: Remove dangling partner admin controller implementations (routes now proxy to-// firecrawl-integrations β see `lib/admin-integration-integrations-proxy.ts`):-// - `controllers/v0/admin/create-user.ts` (`integCreateUserController`)-// - `controllers/v0/admin/validate-api-key.ts` (`integValidateApiKeyController`)-// - `controllers/v0/admin/rotate-api-key.ts` (`integRotateApiKeyController`)adminRouter.post(`/admin/integration/create-user`,wrap(handleIntegrationAdminCreateUserProxy),diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 006d15b50..2d1db7e44 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -59,11 +59,6 @@ import {} from "../controllers/v2/browser";import { activityController } from "../controllers/v1/activity";import { supportProxyController } from "../controllers/v2/support-proxy";-import { agentSignupController } from "../controllers/v2/agent-signup";-import {- agentSignupConfirmController,- agentSignupBlockController,-} from "../controllers/v2/agent-signup-confirm";import {scrapeInteractController,scrapeStopInteractiveBrowserController,@@ -570,11 +565,6 @@ v2Router.post(wrap(supportProxyController),);-// Agent signup routes (public, no auth required β rate limiting is handled inside the controller)-// v2Router.post("/agent-signup", wrap(agentSignupController));-v2Router.post("/agent-signup/confirm", wrap(agentSignupConfirmController));-v2Router.post("/agent-signup/block", wrap(agentSignupBlockController));-// Only register x402 routes if X402_PAY_TO_ADDRESS is configuredif (isX402Enabled()) {v2Router.post(diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.tsindex ffc400ab6..c91b2a846 100644--- a/apps/api/src/services/notification/email_notification.ts+++ b/apps/api/src/services/notification/email_notification.ts@@ -53,7 +53,6 @@ const emailTemplates: Record<NotificationType, EmailTemplate> = {<p>You can modify your notification settings anytime at <a href='https://www.firecrawl.dev/app/account-settings'>firecrawl.dev/app/account-settings</a>.</p><br/>Thanks,<br/>Firecrawl Team<br/>`,},- // Agent sponsor confirm emails are sent directly in the agent-signup controller[NotificationType.AGENT_SPONSOR_CONFIRM]: {subject: "An AI agent requested an API key under your email - Firecrawl",html: "",