Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "feat(api): record saved browser profiles from the browser service (#4730)" (#4732)
Gergő Móriczd3b7d770823c
diff --git a/apps/api/src/__tests__/snips/v2/browser-profiles.test.ts b/apps/api/src/__tests__/snips/v2/browser-profiles.test.tsdeleted file mode 100644index 9787ce65a..000000000--- a/apps/api/src/__tests__/snips/v2/browser-profiles.test.ts+++ /dev/null@@ -1,235 +0,0 @@-import request from "supertest";-import { and, eq, inArray } from "drizzle-orm";-import { v7 as uuidv7 } from "uuid";-import { config } from "../../../config";-import { describeIf, TEST_API_URL, TEST_PRODUCTION } from "../lib";-import { idmux, Identity } from "./lib";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";-import { browserProfileStorageId } from "../../../lib/browser-profiles";--const WEBHOOK_SECRET = config.BROWSER_SERVICE_WEBHOOK_SECRET;--// The browser service reports each successful profile save to the same webhook-// as session.ended. These tests stand in for it, against a session row shaped-// like the one the API writes when a session is created with a profile.-describeIf(TEST_PRODUCTION && !!WEBHOOK_SECRET)(- "Browser profile registry (profile.saved webhook)",- () => {- let identity: Identity;- const sessionIds: string[] = [];- const profileNames: string[] = [];-- beforeAll(async () => {- identity = await idmux({ name: "browser-profiles", credits: 100 });- });-- afterAll(async () => {- if (profileNames.length > 0) {- await db- .delete(schema.browser_profiles)- .where(- and(- eq(schema.browser_profiles.team_id, identity.teamId),- inArray(schema.browser_profiles.name, profileNames),- ),- );- }- if (sessionIds.length > 0) {- await db- .delete(schema.browser_sessions)- .where(inArray(schema.browser_sessions.id, sessionIds));- }- });-- const newProfileName = () => {- const name = `snips-${uuidv7()}`;- profileNames.push(name);- return name;- };-- // Returns the browser-service session id the webhook addresses.- const createSession = async (profileName: string | null) => {- const id = uuidv7();- const browserId = `snips-${id}`;- await db.insert(schema.browser_sessions).values({- id,- team_id: identity.teamId,- browser_id: browserId,- workspace_id: "",- context_id: "",- cdp_url: "wss://example.invalid/cdp",- cdp_path: "https://example.invalid/view",- stream_web_view: false,- status: "destroyed",- ttl_total: 60,- should_bill: false,- profile_name: profileName,- });- sessionIds.push(id);- return browserId;- };-- const profileSaved = (- browserId: string,- profileId: string,- savedAt: string,- sizeBytes?: number,- ) => ({- eventId: uuidv7(),- eventType: "profile.saved",- sessionId: browserId,- profileId,- savedAt,- ...(sizeBytes !== undefined ? { sizeBytes } : {}),- attempt: 0,- });-- // A null secret omits the header.- const send = (body: object, secret: string | null = WEBHOOK_SECRET!) => {- const req = request(TEST_API_URL).post("/v2/browser/webhook/destroyed");- if (secret !== null) req.set("x-browser-service-secret", secret);- return req.send(body);- };-- const profileRow = async (name: string) => {- const [row] = await db- .select()- .from(schema.browser_profiles)- .where(- and(- eq(schema.browser_profiles.team_id, identity.teamId),- eq(schema.browser_profiles.name, name),- ),- );- return row;- };-- it("records a saved profile under the session's team", async () => {- const name = newProfileName();- const browserId = await createSession(name);-- const res = await send(- profileSaved(- browserId,- browserProfileStorageId(identity.teamId, name),- "2026-09-22T10:00:00.000Z",- 4096,- ),- );- expect(res.statusCode).toBe(200);-- const row = await profileRow(name);- expect(row).toBeDefined();- expect(new Date(row!.saved_at).toISOString()).toBe(- "2026-09-22T10:00:00.000Z",- );- expect(row!.size_bytes).toBe(4096);- });-- it("keeps the newest save when deliveries arrive out of order", async () => {- const name = newProfileName();- const profileId = browserProfileStorageId(identity.teamId, name);- const first = await createSession(name);- const second = await createSession(name);-- const newer = await send(- profileSaved(second, profileId, "2026-09-22T12:00:00.000Z", 9000),- );- expect(newer.statusCode).toBe(200);- // An older save's retry lands after the newer one.- const older = await send(- profileSaved(first, profileId, "2026-09-22T11:00:00.000Z", 1000),- );- expect(older.statusCode).toBe(200);-- const row = await profileRow(name);- expect(new Date(row!.saved_at).toISOString()).toBe(- "2026-09-22T12:00:00.000Z",- );- expect(row!.size_bytes).toBe(9000);- });-- it("keeps the last known size when a newer save reports none", async () => {- const name = newProfileName();- const profileId = browserProfileStorageId(identity.teamId, name);- const first = await createSession(name);- const second = await createSession(name);-- expect(- (- await send(- profileSaved(first, profileId, "2026-09-22T11:00:00.000Z", 1000),- )- ).statusCode,- ).toBe(200);- expect(- (- await send(- profileSaved(second, profileId, "2026-09-22T12:00:00.000Z"),- )- ).statusCode,- ).toBe(200);-- const row = await profileRow(name);- expect(new Date(row!.saved_at).toISOString()).toBe(- "2026-09-22T12:00:00.000Z",- );- expect(row!.size_bytes).toBe(1000);- });-- it("rejects a save whose profile id does not match the session", async () => {- const name = newProfileName();- const browserId = await createSession(name);-- const res = await send(- profileSaved(- browserId,- browserProfileStorageId(identity.teamId, `${name}-other`),- "2026-09-22T10:00:00.000Z",- ),- );- expect(res.statusCode).toBe(409);- expect(await profileRow(name)).toBeUndefined();- });-- it("ignores saves for unknown sessions and sessions without a profile", async () => {- const name = newProfileName();- const profileId = browserProfileStorageId(identity.teamId, name);-- const unknown = await send(- profileSaved(- `snips-unknown-${uuidv7()}`,- profileId,- "2026-09-22T10:00:00.000Z",- ),- );- expect(unknown.statusCode).toBe(200);-- const withoutProfile = await createSession(null);- const noProfile = await send(- profileSaved(withoutProfile, profileId, "2026-09-22T10:00:00.000Z"),- );- expect(noProfile.statusCode).toBe(200);-- expect(await profileRow(name)).toBeUndefined();- });-- it("rejects malformed events and requests without the secret", async () => {- const name = newProfileName();- const browserId = await createSession(name);- const event = profileSaved(- browserId,- browserProfileStorageId(identity.teamId, name),- "2026-09-22T10:00:00.000Z",- );-- expect((await send({ ...event, savedAt: "yesterday" })).statusCode).toBe(- 400,- );- expect((await send(event, null)).statusCode).toBe(401);- expect((await send(event, "wrong-secret")).statusCode).toBe(401);- expect(await profileRow(name)).toBeUndefined();- });- },-);diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex c4f0b896e..de8542735 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -1,3 +1,4 @@+import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";import { Request, Response } from "express";import { z } from "zod";@@ -15,7 +16,6 @@ import {invalidateActiveBrowserSessionCount,didBrowserSessionUsePrompt,clearBrowserSessionPromptFlag,- upsertBrowserProfile,} from "../../lib/browser-sessions";import {getCombinedTeamActiveCount,@@ -38,11 +38,6 @@ import { autumnService } from "../../services/autumn/autumn.service";import { orgIdForTeam } from "../../lib/team-org";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { recordRequestCredits } from "../../lib/request-credits-store";-import {- browserProfileStorageId,- profileSavedEventSchema,- resolveProfileSave,-} from "../../lib/browser-profiles";import {getSafeMode,SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,@@ -337,8 +332,12 @@ export async function browserCreateController(// Build persistentStorage from profile if providedlet persistentStorage: { uniqueId: string; write: boolean } | undefined;if (profile) {+ const teamHash = createHash("sha256")+ .update(req.auth.team_id)+ .digest("hex")+ .slice(0, 16);persistentStorage = {- uniqueId: browserProfileStorageId(req.auth.team_id, profile.name),+ uniqueId: `${teamHash}_${profile.name}`,write: profile.saveChanges !== false,};}@@ -426,7 +425,6 @@ export async function browserCreateController(ttl_total: ttl,ttl_without_activity: activityTtl ?? null,credits_used: null,- profile_name: profile?.name ?? null,});} catch (err) {// If we can't persist, tear down the browser session@@ -794,66 +792,6 @@ export async function browserListController(});}-// Records that a session's persistent profile now has saved state, so the-// profile can be listed. Retried by the browser service on any non-2xx.-async function handleProfileSavedWebhook(- req: Request,- res: Response,- logger: typeof _logger,-) {- const parsed = profileSavedEventSchema.safeParse(req.body);- if (!parsed.success) {- logger.warn("Malformed profile.saved webhook", { error: parsed.error });- return res.status(400).json({ error: "Invalid profile.saved event" });- }- const event = parsed.data;- const browserId = event.sessionId;-- const session = await getBrowserSessionByBrowserId(browserId);- if (!session) {- logger.warn("No session found for profile.saved webhook", {- browserId,- eventId: event.eventId,- });- return res.status(200).json({ ok: true });- }-- const resolution = resolveProfileSave(session, event);- if (resolution.action === "ignore") {- logger.info("Not recording saved profile", {- browserId,- sessionId: session.id,- reason: resolution.reason,- });- return res.status(200).json({ ok: true });- }- if (resolution.action === "reject") {- logger.error("profile.saved webhook does not match the session's profile", {- browserId,- sessionId: session.id,- teamId: session.team_id,- profileId: event.profileId,- });- return res- .status(409)- .json({ error: "Profile does not match the session" });- }-- await upsertBrowserProfile({- teamId: resolution.teamId,- name: resolution.name,- savedAt: event.savedAt,- sizeBytes: event.sizeBytes,- });- logger.info("Recorded saved browser profile", {- browserId,- sessionId: session.id,- teamId: resolution.teamId,- eventId: event.eventId,- });- return res.status(200).json({ ok: true });-}-export async function browserWebhookDestroyedController(req: Request,res: Response,@@ -873,12 +811,6 @@ export async function browserWebhookDestroyedController(return res.status(401).json({ error: "Unauthorized" });}- // The browser service posts every outbox event here. session.ended (the- // original event, sent without a check on eventType) bills below.- if (req.body?.eventType === "profile.saved") {- return handleProfileSavedWebhook(req, res, logger);- }-const { sessionId, sessionDurationMs } = req.body as {sessionId?: string;sessionDurationMs?: number;diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 16b346f5a..5096c4087 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -1,3 +1,4 @@+import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";import { Response } from "express";import { z } from "zod";@@ -72,7 +73,6 @@ import { getScrapeJobAccess } from "../../lib/operational-job-access";import { readScrapeJobState } from "../../lib/job-state-store";import { scrapeQueue } from "../../services/worker/nuq-router";import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";-import { browserProfileStorageId } from "../../lib/browser-profiles";// ---------------------------------------------------------------------------// Schemas@@ -712,8 +712,12 @@ async function createSessionForScrape(let persistentStorage: { uniqueId: string; write: boolean } | undefined;if (profile) {+ const teamHash = createHash("sha256")+ .update(req.auth.team_id)+ .digest("hex")+ .slice(0, 16);persistentStorage = {- uniqueId: browserProfileStorageId(req.auth.team_id, profile.name),+ uniqueId: `${teamHash}_${profile.name}`,write: profile.saveChanges !== false,};}@@ -916,7 +920,6 @@ async function createSessionForScrape(ttl_total: ttl,ttl_without_activity: activityTtl ?? null,credits_used: null,- profile_name: profile?.name ?? null,});invalidateActiveBrowserSessionCount(req.auth.team_id).catch(() => {});diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 774bd1273..badc57f6b 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -16,7 +16,6 @@ import {check,foreignKey,index,- primaryKey,unique,} from "drizzle-orm/pg-core";@@ -144,25 +143,8 @@ export const browser_sessions = pgTable("browser_sessions", {credits_used: integer("credits_used"),cdp_interactive_path: text("cdp_interactive_path"),scrape_id: uuid("scrape_id"),- // The persistent profile the session was created with, if any.- profile_name: text("profile_name"),});-// Index of saved persistent browser profiles (see browser_profiles migration).-// Contents live in browser-service object storage; a row is upserted when the-// browser service reports a successful save.-export const browser_profiles = pgTable(- "browser_profiles",- {- team_id: uuid("team_id").notNull(),- name: text("name").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- saved_at: ts("saved_at").notNull(),- size_bytes: bigintNum("size_bytes"),- },- table => [primaryKey({ columns: [table.team_id, table.name] })],-);-export const crawls = pgTable("crawls", {id: uuid("id").notNull(),request_id: uuid("request_id").notNull(),diff --git a/apps/api/src/lib/browser-profiles.test.ts b/apps/api/src/lib/browser-profiles.test.tsdeleted file mode 100644index 5bea8b479..000000000--- a/apps/api/src/lib/browser-profiles.test.ts+++ /dev/null@@ -1,113 +0,0 @@-import {- browserProfileStorageId,- profileSavedEventSchema,- resolveProfileSave,-} from "./browser-profiles";--const TEAM = "00000000-0000-4000-8000-000000000001";-const OTHER_TEAM = "00000000-0000-4000-8000-000000000002";--describe("browserProfileStorageId", () => {- it("prefixes the name with the first 16 hex chars of sha256(teamId)", () => {- expect(browserProfileStorageId(TEAM, "my-profile")).toBe(- "11e594f481958c10_my-profile",- );- expect(browserProfileStorageId(OTHER_TEAM, "my-profile")).toBe(- "e79acd97ac880866_my-profile",- );- });-- it("keeps the name verbatim, underscores included", () => {- expect(browserProfileStorageId(TEAM, "a_b c")).toBe(- "11e594f481958c10_a_b c",- );- });-});--describe("profileSavedEventSchema", () => {- const event = {- eventId: "a4f5c6a0-4c1f-4a7f-9d0b-3e0f1d2c3b4a",- eventType: "profile.saved",- sessionId: "bs_123",- profileId: "11e594f481958c10_my-profile",- savedAt: "2026-09-22T19:30:00.000Z",- sizeBytes: 48213,- attempt: 2,- };-- it("accepts the event the browser service sends", () => {- expect(profileSavedEventSchema.parse(event)).toMatchObject({- sessionId: "bs_123",- profileId: "11e594f481958c10_my-profile",- savedAt: "2026-09-22T19:30:00.000Z",- sizeBytes: 48213,- });- });-- it("accepts an event without a size", () => {- const { sizeBytes: _, ...withoutSize } = event;- expect(profileSavedEventSchema.safeParse(withoutSize).success).toBe(true);- });-- it.each([- ["another event type", { eventType: "session.ended" }],- ["a missing savedAt", { savedAt: undefined }],- ["a savedAt that is not a timestamp", { savedAt: "yesterday" }],- ["a negative size", { sizeBytes: -1 }],- ["a fractional size", { sizeBytes: 1.5 }],- ["an empty profile id", { profileId: "" }],- ])("rejects %s", (_, override) => {- expect(- profileSavedEventSchema.safeParse({ ...event, ...override }).success,- ).toBe(false);- });-});--describe("resolveProfileSave", () => {- const profileId = browserProfileStorageId(TEAM, "my-profile");-- it("upserts under the session's own team and profile name", () => {- expect(- resolveProfileSave(- { team_id: TEAM, profile_name: "my-profile" },- { profileId },- ),- ).toEqual({ action: "upsert", teamId: TEAM, name: "my-profile" });- });-- it("ignores sessions that recorded no profile name", () => {- for (const profile_name of [null, undefined]) {- expect(- resolveProfileSave({ team_id: TEAM, profile_name }, { profileId }),- ).toEqual({ action: "ignore", reason: "no_profile_name" });- }- });-- it("ignores keyless callers, which have no team to list under", () => {- const keylessTeam = "preview_keyless_203.0.113.7";- expect(- resolveProfileSave(- { team_id: keylessTeam, profile_name: "my-profile" },- { profileId: browserProfileStorageId(keylessTeam, "my-profile") },- ),- ).toEqual({ action: "ignore", reason: "not_a_team" });- });-- it("rejects a storage id that belongs to another team", () => {- expect(- resolveProfileSave(- { team_id: TEAM, profile_name: "my-profile" },- { profileId: browserProfileStorageId(OTHER_TEAM, "my-profile") },- ),- ).toEqual({ action: "reject", reason: "profile_mismatch" });- });-- it("rejects a storage id for a different profile name", () => {- expect(- resolveProfileSave(- { team_id: TEAM, profile_name: "my-profile" },- { profileId: browserProfileStorageId(TEAM, "other-profile") },- ),- ).toEqual({ action: "reject", reason: "profile_mismatch" });- });-});diff --git a/apps/api/src/lib/browser-profiles.ts b/apps/api/src/lib/browser-profiles.tsdeleted file mode 100644index f6fa0991b..000000000--- a/apps/api/src/lib/browser-profiles.ts+++ /dev/null@@ -1,67 +0,0 @@-// Pure helpers for persistent browser profiles, kept free of the DB so the-// scrape engine can import the storage id. The table writes live in-// browser-sessions.ts.-import { createHash } from "crypto";-import { z } from "zod";--/**- * The id a profile's saved state is stored under in the browser service: the- * first 16 hex chars of sha256(teamId), then the profile name. /scrape, browser- * sessions and interact sessions must all derive it the same way, or they read- * and write different profiles.- */-export function browserProfileStorageId(teamId: string, name: string): string {- const teamHash = createHash("sha256")- .update(teamId)- .digest("hex")- .slice(0, 16);- return `${teamHash}_${name}`;-}--// Sent by the browser service's webhook outbox after it uploads a session's-// profile. It names the browser-service session and the storage id only.-export const profileSavedEventSchema = z.object({- eventType: z.literal("profile.saved"),- eventId: z.string().min(1),- sessionId: z.string().min(1),- profileId: z.string().min(1),- savedAt: z.iso.datetime({ offset: true }),- sizeBytes: z.number().int().nonnegative().optional(),-});--type ProfileSavedEvent = z.infer<typeof profileSavedEventSchema>;--type ProfileSaveResolution =- | { action: "upsert"; teamId: string; name: string }- | { action: "ignore"; reason: "no_profile_name" | "not_a_team" }- | { action: "reject"; reason: "profile_mismatch" };--/**- * Decides what a reported save means for the session it came from. The team- * and profile name come from our own session row, never from the event, and- * the event's storage id must be the one that row derives.- */-export function resolveProfileSave(- session: { team_id: string; profile_name?: string | null },- event: Pick<ProfileSavedEvent, "profileId">,-): ProfileSaveResolution {- // Sessions created before profile_name was recorded.- if (!session.profile_name) {- return { action: "ignore", reason: "no_profile_name" };- }- // Keyless callers (preview_keyless_<ip>) have no teams row to list under.- if (!z.uuid().safeParse(session.team_id).success) {- return { action: "ignore", reason: "not_a_team" };- }- if (- browserProfileStorageId(session.team_id, session.profile_name) !==- event.profileId- ) {- return { action: "reject", reason: "profile_mismatch" };- }- return {- action: "upsert",- teamId: session.team_id,- name: session.profile_name,- };-}diff --git a/apps/api/src/lib/browser-sessions.ts b/apps/api/src/lib/browser-sessions.tsindex 8ffebe725..c49ca50e6 100644--- a/apps/api/src/lib/browser-sessions.ts+++ b/apps/api/src/lib/browser-sessions.ts@@ -1,4 +1,4 @@-import { and, desc, eq, sql } from "drizzle-orm";+import { and, desc, eq } from "drizzle-orm";import { deleteKey, getValue, setValue } from "../services/redis";import { db } from "../db/connection";import * as schema from "../db/schema";@@ -29,7 +29,6 @@ interface BrowserSessionRow {ttl_total: number;ttl_without_activity: number | null;credits_used: number | null;- profile_name?: string | null; // persistent profile the session was created withcreated_at: string; // ISO timestampupdated_at: string; // ISO timestamp}@@ -283,35 +282,6 @@ export async function updateBrowserSessionCreditsUsed(}}-// Records a successful save of a persistent profile. Throws on failure so the-// browser service's webhook outbox retries the event.-export async function upsertBrowserProfile(input: {- teamId: string;- name: string;- savedAt: string;- sizeBytes: number | undefined;-}): Promise<void> {- const profiles = schema.browser_profiles;- await db- .insert(profiles)- .values({- team_id: input.teamId,- name: input.name,- saved_at: input.savedAt,- size_bytes: input.sizeBytes ?? null,- })- .onConflictDoUpdate({- target: [profiles.team_id, profiles.name],- // Retried deliveries can arrive out of order, so an older save never- // replaces a newer one. A save that reported no size keeps the last- // known size rather than erasing it.- set: {- saved_at: sql`GREATEST(${profiles.saved_at}, excluded.saved_at)`,- size_bytes: sql`CASE WHEN excluded.saved_at >= ${profiles.saved_at} THEN COALESCE(excluded.size_bytes, ${profiles.size_bytes}) ELSE ${profiles.size_bytes} END`,- },- });-}-// ---------------------------------------------------------------------------// Prompt usage tracking (Redis)// ---------------------------------------------------------------------------diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 3a2fe1cb7..79ea24b0f 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -58,7 +58,7 @@ import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { getBrandingScript } from "./brandingScript";import { abTestFireEngine } from "../../../../services/ab-test";import { scheduleABComparison } from "../../../../services/ab-test-comparison";-import { browserProfileStorageId } from "../../../../lib/browser-profiles";+import { createHash } from "node:crypto";/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */const BRANDING_DEFAULT_WAIT_MS = 2000;@@ -491,10 +491,7 @@ export async function scrapeURLWithFireEngineChromeCDP(...(forceNonRender ? { forceNonRender: true } : {}),persistentStorage: meta.options.profile? {- uniqueId: browserProfileStorageId(- meta.internalOptions.teamId,- meta.options.profile.name,- ),+ uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,}: undefined,...safeModeParams(meta.internalOptions.safeMode),