Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import type { Page } from "playwright";
import type { BrowserChallenge } from "../../shared/computer-control";
import type { Control } from "./control";
type Signal = {
cfMitigated?: string | null;
text: string;
visibleChallengeControl: boolean;
};
type Classification = Omit<BrowserChallenge, "requestId">;
/** Generic access denial and score-only blocks offer no actionable human challenge. */
export function classifyChallenge(signal: Signal): Classification | undefined {
if (signal.cfMitigated?.trim().toLowerCase() === "challenge")
return {
kind: "cloudflare",
reason:
"This site presented a Cloudflare security challenge. Please complete it in the existing browser, then hand control back.",
};
if (
signal.visibleChallengeControl &&
/verify (?:that )?you (?:are|'re) (?:a )?human|confirm (?:that )?you (?:are|'re) (?:a )?human|i(?:'|’)m not a robot|complete (?:the|this) (?:captcha|security check)|select all (?:images|squares)|prove (?:that )?you (?:are|'re) (?:a )?human/i.test(
signal.text,
)
) {
return {
kind: "visible-challenge",
reason:
"This page has a visible human verification challenge. Please complete it in the existing browser, then hand control back.",
};
}
}
/** Called only on navigate/read/snapshot. Does not solve, click, or reload a challenge. */
export async function detectChallenge(
page: Page,
control: Control,
options: { cfMitigated?: string | null; toolCallId?: string } = {},
): Promise<BrowserChallenge | undefined> {
const signal =
options.cfMitigated?.trim().toLowerCase() === "challenge"
? {
text: "",
visibleChallengeControl: false,
cfMitigated: options.cfMitigated,
}
: await page.evaluate(() => {
const visible = (element: Element) => {
const style = getComputedStyle(element);
const bounds = element.getBoundingClientRect();
return (
style.display !== "none" &&
style.visibility !== "hidden" &&
Number(style.opacity) !== 0 &&
bounds.width > 0 &&
bounds.height > 0 &&
bounds.bottom > 0 &&
bounds.right > 0 &&
bounds.top < innerHeight &&
bounds.left < innerWidth
);
};
const frames = Array.from(document.querySelectorAll("iframe[src]"));
const challengeFrames = frames.filter((frame) => {
const src = frame.getAttribute("src") ?? "";
return (
/(?:google\.com\/recaptcha|recaptcha\.net\/recaptcha|hcaptcha\.com\/|challenges\.cloudflare\.com\/)/i.test(
src,
) && visible(frame)
);
});
const controls = Array.from(
document.querySelectorAll(
'input[type="checkbox"], [role="checkbox"], button',
),
);
const visibleChallengeControl =
challengeFrames.length > 0 ||
controls.some(
(control) =>
visible(control) &&
/human|not a robot|verify|captcha/i.test(
control.getAttribute("aria-label") ??
control.closest("label")?.textContent ??
control.textContent ??
"",
),
);
return {
text: (document.body?.innerText ?? "").slice(0, 20_000),
visibleChallengeControl,
};
});
const result = classifyChallenge(signal);
if (!result) return undefined;
const request = control.requestHelp(
result.reason,
options.toolCallId,
result.kind,
// The challenge's own address, so a browser that restarts before anyone takes control is sent
// back to it rather than handing the person a blank page with nothing to solve.
page.url(),
).request;
if (!request)
throw new Error("The challenge handoff did not produce a request.");
return { ...result, requestId: request.id };
}