Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas31cefef1bc84
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 369c46bdb..3b62108b8 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -18,6 +18,7 @@ import {} from "../../lib/browser-sessions";import { RequestWithAuth } from "./types";import { billTeam } from "../../services/billing/credit_billing";+import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";const BROWSER_CREDITS_PER_HOUR = 100;@@ -352,6 +353,7 @@ export async function browserExecuteController(logger.info("Executing code in browser session", { language, timeout });+// Execute code via the browser servicelet execResult: BrowserServiceExecResponse;try {@@ -375,6 +377,15 @@ export async function browserExecuteController(stderrLength: execResult.stderr?.length,});+ enqueueBrowserSessionActivity({+ team_id: req.auth.team_id,+ session_id: id,+ language,+ timeout,+ exit_code: execResult.exitCode ?? null,+ killed: execResult.killed ?? false,+ });+const hasError = execResult.exitCode !== 0 || execResult.killed;return res.status(200).json({diff --git a/apps/api/src/lib/browser-session-activity.ts b/apps/api/src/lib/browser-session-activity.tsnew file mode 100644index 000000000..06b418da7--- /dev/null+++ b/apps/api/src/lib/browser-session-activity.ts@@ -0,0 +1,57 @@+import { redisEvictConnection } from "../services/redis";+import { supabase_service } from "../services/supabase";+import { logger as _logger } from "./logger";++const logger = _logger.child({ module: "browser-sessions" });++const QUEUE_KEY = "browser-session-activity-queue";+const BATCH_SIZE = 500;++interface BrowserSessionActivityEvent {+ team_id: string;+ session_id: string;+ language: string;+ timeout: number;+ exit_code: number | null;+ killed: boolean;+ created_at: string;+}++export function enqueueBrowserSessionActivity(+ event: Omit<BrowserSessionActivityEvent, "created_at">,+) {+ const row: BrowserSessionActivityEvent = {+ ...event,+ created_at: new Date().toISOString(),+ };++ redisEvictConnection+ .rpush(QUEUE_KEY, JSON.stringify(row))+ .catch(() => {});+}++export async function processBrowserSessionActivityJobs() {+ const raw =+ (await redisEvictConnection.lpop(QUEUE_KEY, BATCH_SIZE)) ?? [];+ if (raw.length === 0) return;++ const rows: BrowserSessionActivityEvent[] = raw.map((x) => JSON.parse(x));++ try {+ const { error } = await supabase_service+ .from("browser_session_activities")+ .insert(rows);++ if (error) {+ logger.error("Failed to insert browser session activities", {+ error,+ count: rows.length,+ });+ }+ } catch (err) {+ logger.error("Error inserting browser session activities", {+ err,+ count: rows.length,+ });+ }+}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex cb0d711b2..b527fc3c6 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -28,6 +28,7 @@ import { getSearchIndexClient } from "../../lib/search-index-client";// Search indexing is now handled by the separate search service// import { processSearchIndexJobs } from "../../lib/search-index/queue";import { processWebhookInsertJobs } from "../webhook";+import { processBrowserSessionActivityJobs } from "../../lib/browser-session-activity";import {scrapeOptions as scrapeOptionsSchema,crawlRequestSchema,@@ -683,6 +684,7 @@ async function tallyBilling() {const INDEX_INSERT_INTERVAL = 3000;const WEBHOOK_INSERT_INTERVAL = 15000;const OMCE_INSERT_INTERVAL = 5000;+const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;// Search indexing is now handled by separate search service, not this worker// const SEARCH_INDEX_INTERVAL = 10000;@@ -724,6 +726,11 @@ const OMCE_INSERT_INTERVAL = 5000;await processWebhookInsertJobs();}, WEBHOOK_INSERT_INTERVAL);+ const browserActivityInterval = setInterval(async () => {+ if (isShuttingDown) return;+ await processBrowserSessionActivityJobs();+ }, BROWSER_ACTIVITY_INSERT_INTERVAL);+const omceInserterInterval = setInterval(async () => {if (isShuttingDown) {return;@@ -793,6 +800,7 @@ const OMCE_INSERT_INTERVAL = 5000;clearInterval(indexInserterInterval);clearInterval(webhookInserterInterval);+ clearInterval(browserActivityInterval);clearInterval(omceInserterInterval);clearInterval(billingTallyInterval);