Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

18.2 KB
Raw Blame
import { z } from "zod";
import { promises as fs } from "fs";
import path from "path";
import { tool, stepCountIs, generateText as untracedGenerateText } from "ai";
import { logger as _logger } from "../logger";
import { getModel } from "../generic-ai";
import { executeHangarBrowser, BrowserExecutionResult } from "../hangar";
import { config } from "../../config";
import type { TeamFlags } from "../../controllers/v2/types";
import { isUsOnly } from "../us-only";
import {
  generateText,
  buildLangSmithProviderOptions,
  traceInteract,
  InteractTraceMetadata,
} from "./langsmith";

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const MAX_STEPS = 25;
const SNAPSHOT_TIMEOUT = 15;
const SNAPSHOT_MAX_CHARS = 40_000;

// Vertex is the preferred provider so usage is traceable via Vertex billing
// labels; the GenAI (Gemini) API is only a fallback when Vertex credentials
// aren't configured (e.g. self-hosted). Mirrors services/monitoring/search/tuning.ts.
function hasVertex(): boolean {
  return Boolean(config.VERTEX_CREDENTIALS) || config.VERTEX_USE_ADC;
}

// ---------------------------------------------------------------------------
// Debug log
// ---------------------------------------------------------------------------

const AGENT_LOG_DIR = path.join(
  __dirname,
  "../../logs/scrape-interact-agent-logs",
);

const IS_PRODUCTION = config.IS_PRODUCTION === true;

class AgentDebugLog {
  private filePath: string;
  private lines: string[] = [];
  private enabled: boolean;

  constructor(browserId: string, zeroDataRetention = false) {
    this.enabled = !IS_PRODUCTION && !zeroDataRetention;
    const ts = new Date().toISOString().replace(/[:.]/g, "-");
    this.filePath = path.join(AGENT_LOG_DIR, `${ts}_${browserId}.log`);
  }

  add(text: string) {
    if (!this.enabled) return;
    this.lines.push(text);
  }

  async flush() {
    if (!this.enabled || this.lines.length === 0) return;
    try {
      await fs.mkdir(AGENT_LOG_DIR, { recursive: true });
      await fs.appendFile(this.filePath, this.lines.join("\n") + "\n");
      this.lines = [];
    } catch (err) {
      _logger.error("AgentDebugLog flush failed", {
        filePath: this.filePath,
        agentLogDir: AGENT_LOG_DIR,
        error: err,
      });
    }
  }

  getPath() {
    return this.filePath;
  }
}

// ---------------------------------------------------------------------------
// System prompt
// ---------------------------------------------------------------------------

const SYSTEM_PROMPT = `You are an autonomous browser automation agent. You complete tasks by interacting with a browser using the browser tool. Be autonomous β€” break tasks into steps, execute ALL steps without stopping early, and be concise.

## Browser Tool β€” agent-browser commands
The browser tool runs agent-browser CLI commands. Each tool call should contain a single command or a short chain (joined with &&).

Commands:
  agent-browser snapshot                Get full accessibility tree with clickable refs (@e1, @e2...), prefer using -i
  agent-browser snapshot -i             Only interactive elements
  agent-browser snapshot -s "#css"      Scope snapshot to a CSS selector
  agent-browser click @e1               Click element by ref
  agent-browser fill @e2 "text"         Clear field and type text
  agent-browser type @e2 "text"         Type without clearing
  agent-browser select @e1 "option"     Select dropdown option
  agent-browser check @e1               Toggle checkbox
  agent-browser press Enter             Press a key
  agent-browser keyboard type "text"    Type at current focus
  agent-browser hover @e1               Hover element
  agent-browser scroll down 500         Scroll down (px)
  agent-browser scroll up 500           Scroll up (px)
  agent-browser get text @e1            Get text content of element
  agent-browser get title               Get page title
  agent-browser get url                 Get current URL
  agent-browser wait @e1                Wait for element to appear
  agent-browser wait --load networkidle Wait for network idle
  agent-browser wait --text "Welcome"   Wait for text to appear
  agent-browser wait 2000               Wait milliseconds
  agent-browser find text "X" click     Find element by text and click
  agent-browser find role button click --name "Submit"
  agent-browser find placeholder "Q" type "query"
  agent-browser frame @e2               Scope to iframe
  agent-browser frame main              Return to main frame
  agent-browser eval "js code"          Run JavaScript in page
  agent-browser back                    Go back

## Workflow
1. An initial page snapshot is provided β€” use it immediately, don't re-snapshot.
2. Interact with elements using @refs from the snapshot.
3. After interactions that change the page, call snapshot to see the updated state.
4. Use the new @refs from the latest snapshot for further interactions.
5. Repeat until the task is complete.

## Rules
1. You are already on the target page. Do NOT navigate to external sites or search engines.
2. @refs are invalidated after page changes β€” always snapshot again after interactions.
3. Chain independent commands with && to save round-trips.
4. When extracting data, use agent-browser get text or agent-browser eval to pull content.
5. If a command fails, try a different approach (different selector, wait first, use find, etc.).
6. NEVER open new tabs. Always work in the current tab. Do not use agent-browser tab new or agent-browser open. If you need to navigate within the site, click links directly.

## Output Format
Your final text response is what the user sees. It MUST be a clean, human-readable answer:
- If asked for a price β†’ respond with the product name and price (e.g. "iPhone 15 Pro Max: $1,199")
- If asked for a list β†’ respond with a clean list of items
- If asked to perform an action β†’ confirm what was done
- NEVER dump raw HTML, accessibility trees, or @ref identifiers in your final response
- Be concise and direct β€” just the answer the user asked for`;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

export interface AgentResult extends BrowserExecutionResult {
  output: string;
}

async function execInBrowser(
  browserId: string,
  code: string,
  timeout: number,
): Promise<BrowserExecutionResult> {
  return executeHangarBrowser(browserId, {
    code,
    language: "bash",
    timeout,
  });
}

async function getCurrentUrl(browserId: string): Promise<string> {
  try {
    const result = await execInBrowser(
      browserId,
      "agent-browser get url",
      SNAPSHOT_TIMEOUT,
    );
    return (result.stdout || result.result || "").trim();
  } catch {
    return "";
  }
}

async function takeSnapshot(browserId: string): Promise<string> {
  try {
    const result = await execInBrowser(
      browserId,
      "agent-browser snapshot -i",
      SNAPSHOT_TIMEOUT,
    );
    return (result.stdout || result.result || "").slice(0, SNAPSHOT_MAX_CHARS);
  } catch {
    return "";
  }
}

/** Align the CLI with the replay's page without discarding the user's other tabs. */
export async function selectBrowserAgentTab(browserId: string): Promise<void> {
  const target = await executeHangarBrowser(browserId, {
    code: `await (async () => {
      for (const candidate of [page, ...context.pages().filter(p => p !== page)]) {
        if (await candidate.evaluate(() => document.visibilityState === 'visible').catch(() => false)) {
          page = candidate;
          break;
        }
      }
      await page.bringToFront();
      const cdp = await context.newCDPSession(page);
      try {
        const { targetInfo } = await cdp.send('Target.getTargetInfo');
        console.log(targetInfo.targetId);
      } finally { await cdp.detach(); }
    })()`,
    language: "node",
    timeout: 10,
  });
  const targetId = (target.stdout || target.result || "").trim();
  if (
    target.exitCode !== 0 ||
    target.killed ||
    !/^[a-f0-9]{32}$/i.test(targetId)
  )
    throw new Error("Could not identify the replay browser tab.");
  const selected = await execInBrowser(
    browserId,
    `agent-browser tab ${targetId}`,
    10,
  );
  if (selected.exitCode !== 0 || selected.killed)
    throw new Error("Could not select the replay browser tab.");
}

// ---------------------------------------------------------------------------
// Main agent β€” tool-calling loop via AI SDK
// ---------------------------------------------------------------------------

interface BrowserAgentTraceContext {
  sessionId: string;
  scrapeId: string;
  teamId: string;
  orgId?: string;
  zeroDataRetention?: boolean;
  teamFlags?: TeamFlags | null;
  scrapeUrl?: string;
  targetUrl?: string;
  scrapeWaitForMs?: number;
  scrapeActions?: number;
  scrapeOrigin?: string;
}

// The tracing vendor is skipped for US-only teams, as it is for ZDR.
function isUntraced(trace: BrowserAgentTraceContext | undefined): boolean {
  return (
    trace?.zeroDataRetention === true || isUsOnly(trace?.teamFlags ?? undefined)
  );
}

// Throws UsOnlyUnavailableError when a US-only team has no US route, so
// callers can refuse before creating a session or reserving credits.
export function getBrowserAgentModel(
  trace?: Pick<BrowserAgentTraceContext, "zeroDataRetention" | "teamFlags">,
) {
  const modelOptions = { teamFlags: trace?.teamFlags, feature: "Interact" };
  return trace?.zeroDataRetention === true
    ? getModel("gpt-6-luna", "openai", {
        ignoreModelOverride: true,
        ...modelOptions,
      })
    : getModel(
        "gemini-3.5-flash",
        hasVertex() ? "vertex" : "google",
        modelOptions,
      );
}

export async function executePromptViaBrowserAgent(
  prompt: string,
  browserId: string,
  stepTimeout: number,
  logger: typeof _logger,
  trace?: BrowserAgentTraceContext,
): Promise<AgentResult> {
  const zeroDataRetention = trace?.zeroDataRetention === true;
  const untraced = isUntraced(trace);
  const model = getBrowserAgentModel(trace);
  const debugLog = new AgentDebugLog(browserId, zeroDataRetention);
  debugLog.add(`=== AGENT RUN ===`);
  debugLog.add(`Time:    ${new Date().toISOString()}`);
  debugLog.add(`Browser: ${browserId}`);
  debugLog.add(`Prompt:  ${prompt}\n`);
  logger.info("Agent debug log", { path: debugLog.getPath() });

  await selectBrowserAgentTab(browserId);

  const [initialSnapshot, initialUrl] = await Promise.all([
    takeSnapshot(browserId),
    getCurrentUrl(browserId),
  ]);

  debugLog.add(`URL: ${initialUrl}`);
  debugLog.add(
    `Snapshot (${initialSnapshot.length} chars):\n${initialSnapshot || "(empty)"}\n`,
  );
  debugLog.flush();

  let toolCallCount = 0;
  const allOutputs: string[] = [];
  let lastSnapshotResult = initialSnapshot;
  const actionLog: string[] = [];

  const browserTool = tool({
    description:
      "Run an agent-browser CLI command in the browser. Each call should be one command or a short && chain.",
    inputSchema: z.object({
      code: z.string().describe("The agent-browser command(s) to execute"),
    }),
    execute: async ({ code }) => {
      toolCallCount++;
      const start = Date.now();
      debugLog.add(`--- [${toolCallCount}] ${code} ---`);

      if (/agent-browser\s+(tab\s+new|open\s)/.test(code)) {
        const msg =
          "Blocked: opening new tabs/URLs is not allowed. Use click to navigate within the site.";
        debugLog.add(`BLOCKED: ${msg}\n`);
        actionLog.push(`${toolCallCount}. ${code} β†’ BLOCKED`);
        return { error: msg };
      }

      try {
        const result = await execInBrowser(browserId, code, stepTimeout);
        const output = (result.stdout || result.result || "").trim();

        await selectBrowserAgentTab(browserId);

        const elapsed = Date.now() - start;

        debugLog.add(`Exit: ${result.exitCode} (${elapsed}ms)`);
        if (output) debugLog.add(`Output:\n${output}`);
        if (result.stderr) debugLog.add(`Stderr:\n${result.stderr}`);
        debugLog.add("");
        debugLog.flush();

        if (code.includes("snapshot")) lastSnapshotResult = output;
        if (output) allOutputs.push(output);

        const brief = code.includes("snapshot")
          ? `(${output.length} chars)`
          : (output || result.stderr || "").slice(0, 120);
        const status =
          result.exitCode === 0 ? "OK" : `FAIL(${result.exitCode})`;
        actionLog.push(`${toolCallCount}. ${code} β†’ ${status}: ${brief}`);

        if (result.exitCode !== 0) {
          return {
            error: result.stderr || "Command failed",
            exit_code: result.exitCode,
            output,
          };
        }
        return { result: output || "(no output)" };
      } catch (err: unknown) {
        const msg = err instanceof Error ? err.message : String(err);
        debugLog.add(`Error: ${msg}\n`);
        debugLog.flush();
        actionLog.push(
          `${toolCallCount}. ${code} β†’ ERROR: ${msg.slice(0, 120)}`,
        );
        return { error: msg };
      }
    },
  });

  const langsmith = trace
    ? buildLangSmithProviderOptions(
        {
          thread_id: trace.sessionId,
          session_id: trace.sessionId,
          scrape_id: trace.scrapeId,
          team_id: trace.teamId,
          org_id: trace.orgId,
          browser_id: browserId,
          mode: "prompt",
          zeroDataRetention: untraced,
          scrape_url: trace.scrapeUrl,
          target_url: trace.targetUrl,
          scrape_wait_for_ms: trace.scrapeWaitForMs,
          scrape_actions: trace.scrapeActions,
          scrape_origin: trace.scrapeOrigin,
        } satisfies InteractTraceMetadata,
        {
          name: "interact:prompt",
          extra: { prompt_length: prompt.length },
        },
      )
    : undefined;

  try {
    // Bypass the LangSmith-wrapped SDK entirely for ZDR, even when tracing
    // is enabled globally. Use the same provider as ZDR JSON extraction.
    const generate = untraced ? untracedGenerateText : generateText;
    const result = await generate({
      model,
      experimental_telemetry: { isEnabled: false },
      system: SYSTEM_PROMPT,
      messages: [
        {
          role: "user" as const,
          content: [
            {
              type: "text" as const,
              text: `Current URL: ${initialUrl || "(unknown)"}\n\nPage snapshot:\n${initialSnapshot || "(empty β€” page may still be loading)"}\n\nTask: ${prompt}`,
            },
          ],
        },
      ],
      tools: { browser: browserTool },
      stopWhen: stepCountIs(MAX_STEPS),
      temperature: zeroDataRetention ? undefined : 0,
      // LangSmith's provider-options object is recognized by wrapAISDK but
      // does not satisfy AI SDK's SharedV3ProviderOptions shape, hence the
      // local cast keeps the rest of the type surface strict.
      ...(zeroDataRetention
        ? {
            providerOptions: {
              openai: {
                store: false,
                reasoningEffort: "medium",
              },
            },
          }
        : langsmith
          ? { providerOptions: { langsmith } as Record<string, any> }
          : {}),
      prepareStep: async ({ stepNumber, messages }) => {
        if (actionLog.length === 0) return {};
        return {
          messages: [
            ...messages,
            {
              role: "user" as const,
              content: [
                {
                  type: "text" as const,
                  text: `ACTION LOG (your commands so far):\n${actionLog.join("\n")}\n\nReview this log before your next action. Common mistakes to check for:\n- Typed text but forgot to press Enter\n- Clicked a link but didn't wait or re-snapshot\n- Used stale @refs from a previous snapshot\n- Scrolled but didn't snapshot to see new content`,
                },
              ],
            },
          ],
        };
      },
      onStepFinish: ({ text, toolCalls }) => {
        if (toolCalls?.length) {
          debugLog.add(`[Step: ${toolCalls.length} tool call(s)]`);
        }
        if (text) debugLog.add(`Assistant: ${text}`);
      },
    });

    debugLog.add(`\n=== END: completed (${toolCallCount} tool calls) ===\n`);
    await debugLog.flush();

    return {
      output: result.text || "",
      stdout: allOutputs.join("\n"),
      result: lastSnapshotResult,
      stderr: "",
      exitCode: 0,
      killed: false,
    };
  } catch (err: unknown) {
    const msg = err instanceof Error ? err.message : String(err);
    logger.error("Agent failed", { error: err });
    debugLog.add(`\n=== END: error β€” ${msg} ===\n`);
    await debugLog.flush();

    return {
      output: "",
      stdout: allOutputs.join("\n"),
      result: lastSnapshotResult,
      stderr: msg,
      exitCode: 1,
      killed: false,
    };
  }
}

// ---------------------------------------------------------------------------
// Code path β€” direct exec wrapped with the same trace metadata shape as the
// prompt path, so tracing details don't have to live in the controller.
// ---------------------------------------------------------------------------

export async function executeCodeViaBrowserSession(
  browserId: string,
  params: {
    code: string;
    language: string;
    timeout: number;
    origin?: string;
  },
  trace?: BrowserAgentTraceContext,
): Promise<BrowserExecutionResult> {
  // Arg must be named so langsmith's traceable sees the exec params as the
  // run's `inputs`; a zero-arg closure would record `{}` and strip the code,
  // language, timeout, and origin from every trace.
  const run = async (execParams: typeof params) =>
    executeHangarBrowser(browserId, {
      code: execParams.code,
      language: execParams.language,
      timeout: execParams.timeout,
    });

  if (!trace) return run(params);

  const traced = traceInteract(
    run,
    {
      thread_id: trace.sessionId,
      session_id: trace.sessionId,
      scrape_id: trace.scrapeId,
      team_id: trace.teamId,
      org_id: trace.orgId,
      browser_id: browserId,
      mode: "code",
      zeroDataRetention: isUntraced(trace),
      scrape_url: trace.scrapeUrl,
      target_url: trace.targetUrl,
      scrape_wait_for_ms: trace.scrapeWaitForMs,
      scrape_actions: trace.scrapeActions,
      scrape_origin: trace.scrapeOrigin,
    } satisfies InteractTraceMetadata,
    { name: "interact:code" },
  );

  return traced(params);
}