Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Reduce web fetch anti-bot false positives

wyt8cfabcdc500a
diff --git a/src/core-agent/src/tools/web-fetch.ts b/src/core-agent/src/tools/web-fetch.ts
index d4b67a38..c16c617e 100644
--- a/src/core-agent/src/tools/web-fetch.ts
+++ b/src/core-agent/src/tools/web-fetch.ts
@@ -122,7 +122,13 @@ export function classifyFetchContent(url: string, title: string | undefined, raw
const head = `${title || ""}\n${raw.slice(0, 6000)}\n${text.slice(0, 3000)}`;
const compactText = text.replace(/\s+/g, "");
- if (/_waf_[a-z0-9]+|captcha|cloudflare|access denied|bot detection|人机验证|安全验证|访问验证|验证码|反爬/i.test(head)) {
+ // Match anti-bot/WAF *challenge* markers, not infrastructure names. Bare
+ // `cloudflare`/`captcha`/`access denied` false-positived on any normal page
+ // that merely loads a Cloudflare CDN/Insights asset (cdnjs.cloudflare.com,
+ // static.cloudflareinsights.com) or a reCAPTCHA widget, wrongly telling the
+ // model "do not retry, this is a bot wall". These phrases only appear on the
+ // actual challenge/block page.
+ if (/_waf_[a-z0-9]+|cf-browser-verification|__cf_chl|cf_chl_opt|Attention Required!\s*\|\s*Cloudflare|Cloudflare Ray ID|Checking your browser before access|Just a moment\.\.\.|Enable JavaScript and cookies to continue|Verify (?:you are|you're)(?: a)? human|complete the security check|you don'?t have permission to access|人机(?:身份)?验证|安全验证|访问验证|滑动验证|请完成验证|反爬/i.test(head)) {
return {
code: "WAF_OR_BOT_CHECK",
message:
diff --git a/src/core-agent/test/web-fetch.test.ts b/src/core-agent/test/web-fetch.test.ts
index f93885e4..1b434b69 100644
--- a/src/core-agent/test/web-fetch.test.ts
+++ b/src/core-agent/test/web-fetch.test.ts
@@ -103,6 +103,29 @@ describe("web-fetch › classifyFetchContent", () => {
expect(issue).toMatchObject({ code: "JS_OR_NAV_SHELL" });
});
+ it("marks a real Cloudflare interstitial challenge as failed content", () => {
+ const issue = classifyFetchContent(
+ "https://protected.example.com/article",
+ "Just a moment...",
+ '<html><head><title>Just a moment...</title></head><body><div class="cf-browser-verification"></div>' +
+ "<p>Enable JavaScript and cookies to continue</p><script>window.__cf_chl_opt={};</script></body></html>",
+ "Just a moment... Enable JavaScript and cookies to continue",
+ );
+
+ expect(issue).toMatchObject({ code: "WAF_OR_BOT_CHECK" });
+ });
+
+ it("marks a Chinese security-verification wall as failed content", () => {
+ const issue = classifyFetchContent(
+ "https://www.example.cn/news/123",
+ "安全验证",
+ "<html><title>安全验证</title><body>请完成验证后访问</body></html>",
+ "安全验证 请完成验证后访问",
+ );
+
+ expect(issue).toMatchObject({ code: "WAF_OR_BOT_CHECK" });
+ });
+
it("does not classify normal article text as failed content", () => {
const issue = classifyFetchContent(
"https://example.com/article",
@@ -113,4 +136,32 @@ describe("web-fetch › classifyFetchContent", () => {
expect(issue).toBeNull();
});
+
+ it("does not flag a normal page that merely loads Cloudflare CDN / Insights assets (look-alike)", () => {
+ const issue = classifyFetchContent(
+ "https://example.com/docs",
+ "API Reference",
+ '<html><head><title>API Reference</title>' +
+ '<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>' +
+ '<script defer src="https://static.cloudflareinsights.com/beacon.min.js"></script></head>' +
+ "<body><article>Call the endpoint with a bearer token.</article></body></html>",
+ "API Reference\nCall the endpoint with a bearer token.",
+ );
+
+ expect(issue).toBeNull();
+ });
+
+ it("does not flag a normal page with a reCAPTCHA-protected contact form (look-alike)", () => {
+ const issue = classifyFetchContent(
+ "https://example.com/contact",
+ "Contact us",
+ '<html><head><title>Contact us</title>' +
+ '<script src="https://www.google.com/recaptcha/api.js" async defer></script></head>' +
+ '<body><h1>Contact us</h1><form><div class="g-recaptcha" data-sitekey="abc"></div>' +
+ "<p>We usually reply within one business day.</p></form></body></html>",
+ "Contact us\nWe usually reply within one business day.",
+ );
+
+ expect(issue).toBeNull();
+ });
});