Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
import { afterEach, describe, it, expect } from "vitest";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import {
resolveCharset,
decodeBytes,
classifyFetchContent,
configureWebFetchRenderer,
htmlToText,
decodeNumericEntity,
formatWebFetchNetworkFailure,
MAX_WEB_FETCH_RESPONSE_BYTES,
webFetchTool,
} from "../src/tools/web-fetch.js";
describe("web-fetch › resolveCharset", () => {
it("picks charset from Content-Type header when present", () => {
expect(resolveCharset("text/html; charset=GBK", Buffer.from(""))).toBe("gbk");
expect(resolveCharset("text/html;charset=gb2312", Buffer.from(""))).toBe("gb2312");
expect(resolveCharset("text/html; charset=\"utf-8\"", Buffer.from(""))).toBe("utf-8");
});
it("header trumps meta tag (authoritative)", () => {
const html = '<html><head><meta charset="utf-8"></head></html>';
expect(resolveCharset("text/html; charset=gbk", Buffer.from(html))).toBe("gbk");
});
it("falls back to <meta charset=...> from first 2KB when header is missing", () => {
const html = '<html><head><meta charset="GB2312"><title>x</title></head></html>';
expect(resolveCharset("text/html", Buffer.from(html))).toBe("gb2312");
});
it("parses legacy <meta http-equiv> form", () => {
const html =
'<html><head><meta http-equiv="Content-Type" content="text/html; charset=gbk"></head></html>';
expect(resolveCharset("text/html", Buffer.from(html))).toBe("gbk");
});
it("handles unquoted meta charset", () => {
const html = "<html><head><meta charset=big5></head></html>";
expect(resolveCharset("text/html", Buffer.from(html))).toBe("big5");
});
it("defaults to utf-8 when nothing is declared", () => {
expect(resolveCharset("text/html", Buffer.from("<html></html>"))).toBe("utf-8");
expect(resolveCharset("", Buffer.from(""))).toBe("utf-8");
});
it("only inspects the first 2KB of the body (no scan past head region)", () => {
const padding = " ".repeat(4096);
const html = `<html><head></head><body>${padding}<meta charset="gbk"></body></html>`;
expect(resolveCharset("text/html", Buffer.from(html))).toBe("utf-8");
});
});
describe("web-fetch › decodeBytes", () => {
it("round-trips UTF-8 Chinese text", () => {
const buf = Buffer.from("深圳天气", "utf-8");
expect(decodeBytes(buf, "utf-8")).toBe("深圳天气");
});
it("decodes GBK bytes correctly (the huangjinjiage.cn reproducer)", () => {
// "今日金价" in GBK = D5 EB D7 D2 E4 B3 BC DB (8 bytes, 4 chars)
// Wait actually: 今=BDF1 日=C8D5 金=BDF0 价=BCDB → bytes BD F1 C8 D5 BD F0 BC DB
const bytes = Buffer.from([0xbd, 0xf1, 0xc8, 0xd5, 0xbd, 0xf0, 0xbc, 0xdb]);
expect(decodeBytes(bytes, "gbk")).toBe("今日金价");
});
it("GBK bytes decoded as utf-8 produce replacement chars (the original bug)", () => {
// Verifies that the previous hardcoded utf-8 path garbled CN pages —
// locks in the root cause so future regressions are caught.
const bytes = Buffer.from([0xbd, 0xf1, 0xc8, 0xd5, 0xbd, 0xf0, 0xbc, 0xdb]);
const wrong = decodeBytes(bytes, "utf-8");
expect(wrong).not.toBe("今日金价");
expect(wrong).toMatch(/\uFFFD/); // U+FFFD replacement char present
});
it("unknown charset label falls back to utf-8 instead of throwing", () => {
const buf = Buffer.from("hello", "utf-8");
expect(decodeBytes(buf, "this-is-not-a-real-charset")).toBe("hello");
});
});
describe("web-fetch › htmlToText", () => {
it("decodes decimal astral entities via code points (emoji, not garbage)", () => {
// Old code used String.fromCharCode(128512), which truncates to U+F600.
expect(htmlToText("😀")).toBe("😀");
});
it("decodes hex astral entities via code points", () => {
expect(htmlToText("😀")).toBe("😀");
});
it("drops out-of-range numeric entities instead of emitting garbage", () => {
// 0x110000 is one past the Unicode ceiling; fromCodePoint would throw.
expect(htmlToText("a�b")).toBe("ab");
expect(htmlToText("a�b")).toBe("ab");
});
it("still decodes BMP numeric and named entities", () => {
expect(htmlToText("AB")).toBe("AB");
expect(htmlToText("Fish & Chips <3 …")).toBe("Fish & Chips <3 …");
});
it("separates adjacent table cells with a space", () => {
// Old code stripped </td>/</th> silently, concatenating cells into "ab".
expect(htmlToText("<table><tr><td>a</td><td>b</td></tr></table>")).toBe("a b");
expect(htmlToText("<table><tr><th>h1</th><th>h2</th></tr></table>")).toBe("h1 h2");
});
it("keeps rows on separate lines while spacing cells", () => {
const html = "<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>";
expect(htmlToText(html)).toBe("a b\nc d");
});
});
describe("web-fetch › decodeNumericEntity", () => {
it("returns astral code points intact", () => {
expect(decodeNumericEntity(0x1f600)).toBe("😀");
});
it("drops NaN, negative, and beyond-Unicode inputs", () => {
expect(decodeNumericEntity(Number.NaN)).toBe("");
expect(decodeNumericEntity(-1)).toBe("");
expect(decodeNumericEntity(0x110000)).toBe("");
});
});
describe("web-fetch › webFetchTool abort signal", () => {
function startNeverRespondingServer(): Promise<{ server: Server; url: string }> {
return new Promise((resolve) => {
const server = createServer(() => {
/* accept the request, never respond */
});
server.listen(0, "127.0.0.1", () => {
const { port } = server.address() as AddressInfo;
resolve({ server, url: `http://127.0.0.1:${port}/` });
});
});
}
function stop(server: Server): Promise<void> {
server.closeAllConnections?.();
return new Promise((resolve) => server.close(() => resolve()));
}
it("rejects promptly when the tool context signal aborts mid-flight", async () => {
const { server, url } = await startNeverRespondingServer();
try {
const ac = new AbortController();
const started = Date.now();
const pending = webFetchTool.execute({ url }, { signal: ac.signal, state: {} });
setTimeout(() => ac.abort(), 50);
const result = await pending;
const elapsed = Date.now() - started;
expect(result.isError).toBe(true);
expect(result.content).toContain("cancelled");
// Old code ignored ctx.signal and hung for the full 60s timeout.
expect(elapsed).toBeLessThan(5000);
} finally {
await stop(server);
}
}, 10_000);
it("returns immediately for an already-aborted signal", async () => {
const { server, url } = await startNeverRespondingServer();
try {
const ac = new AbortController();
ac.abort();
const started = Date.now();
const result = await webFetchTool.execute({ url }, { signal: ac.signal, state: {} });
expect(result.isError).toBe(true);
expect(Date.now() - started).toBeLessThan(5000);
} finally {
await stop(server);
}
}, 10_000);
});
describe("web-fetch › classifyFetchContent", () => {
it("marks WAF challenge bodies as failed content", () => {
const issue = classifyFetchContent(
"https://xueqiu.com/3439096517/390394657",
undefined,
'{"_waf_bd8ce2ce37":"Pfachz2vL1SL0SmQA"}3EJP1NTyp9ak5NoRM',
'{"_waf_bd8ce2ce37":"Pfachz2vL1SL0SmQA"}',
);
expect(issue).toMatchObject({ code: "WAF_OR_BOT_CHECK" });
});
it("marks missing social pages as failed content", () => {
const issue = classifyFetchContent(
"https://www.xiaohongshu.com/discovery/item/deleted",
"小红书 - 你访问的页面不见了",
"<html><title>小红书 - 你访问的页面不见了</title></html>",
"小红书 - 你访问的页面不见了",
);
expect(issue).toMatchObject({ code: "PAGE_NOT_FOUND" });
});
it("marks known article navigation shells as failed content", () => {
const issue = classifyFetchContent(
"https://www.cls.cn/detail/xk/68a020e69b01344433be9032",
"财联社电报:7*24小时滚动播报股市资讯",
"<html><title>财联社电报:7*24小时滚动播报股市资讯</title></html>",
"关于我们 网站声明 联系方式 用户反馈 网站地图 帮助 首页 电报 话题 盯盘 VIP FM 投研 下载",
);
expect(issue).toMatchObject({ code: "JS_OR_NAV_SHELL" });
});
it("marks a real Cloudflare interstitial challenge as failed content", () => {
const issue = classifyFetchContent(
"https://protected.example.com/article",
"Just a moment...",
'<html><head><title>Just a moment...</title></head><body><div class="cf-browser-verification"></div>' +
"<p>Enable JavaScript and cookies to continue</p><script>window.__cf_chl_opt={};</script></body></html>",
"Just a moment... Enable JavaScript and cookies to continue",
);
expect(issue).toMatchObject({ code: "WAF_OR_BOT_CHECK" });
});
it("marks a Chinese security-verification wall as failed content", () => {
const issue = classifyFetchContent(
"https://www.example.cn/news/123",
"安全验证",
"<html><title>安全验证</title><body>请完成验证后访问</body></html>",
"安全验证 请完成验证后访问",
);
expect(issue).toMatchObject({ code: "WAF_OR_BOT_CHECK" });
});
it("does not classify normal article text as failed content", () => {
const issue = classifyFetchContent(
"https://example.com/article",
"Quarterly results",
"<html><title>Quarterly results</title><article>Revenue grew 20 percent.</article></html>",
"Quarterly results\nRevenue grew 20 percent.",
);
expect(issue).toBeNull();
});
it("does not flag a normal page that merely loads Cloudflare CDN / Insights assets (look-alike)", () => {
const issue = classifyFetchContent(
"https://example.com/docs",
"API Reference",
'<html><head><title>API Reference</title>' +
'<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>' +
'<script defer src="https://static.cloudflareinsights.com/beacon.min.js"></script></head>' +
"<body><article>Call the endpoint with a bearer token.</article></body></html>",
"API Reference\nCall the endpoint with a bearer token.",
);
expect(issue).toBeNull();
});
it("does not flag a normal page with a reCAPTCHA-protected contact form (look-alike)", () => {
const issue = classifyFetchContent(
"https://example.com/contact",
"Contact us",
'<html><head><title>Contact us</title>' +
'<script src="https://www.google.com/recaptcha/api.js" async defer></script></head>' +
'<body><h1>Contact us</h1><form><div class="g-recaptcha" data-sitekey="abc"></div>' +
"<p>We usually reply within one business day.</p></form></body></html>",
"Contact us\nWe usually reply within one business day.",
);
expect(issue).toBeNull();
});
});
describe("web-fetch › safe network diagnostics", () => {
it("surfaces a nested undici timeout code without returning arbitrary cause text", () => {
const cause = Object.assign(new Error("connect timed out at 192.0.2.10:443"), {
code: "UND_ERR_CONNECT_TIMEOUT",
});
const error = new TypeError("fetch failed", { cause });
const result = formatWebFetchNetworkFailure(error);
expect(result).toBe(
"fetch failed [network_diagnostic category=timeout; codes=UND_ERR_CONNECT_TIMEOUT]",
);
expect(result).not.toContain("192.0.2.10");
});
it("inspects bounded AggregateError children and keeps only stable network codes", () => {
const error = new TypeError("fetch failed", {
cause: new AggregateError([
Object.assign(new Error("IPv6 route includes a private address"), { code: "ENETUNREACH" }),
Object.assign(new Error("IPv4 endpoint timed out"), { code: "ETIMEDOUT" }),
], "secret proxy details"),
});
const result = formatWebFetchNetworkFailure(error);
expect(result).toBe(
"fetch failed [network_diagnostic category=timeout; codes=ENETUNREACH,ETIMEDOUT]",
);
expect(result).not.toContain("private address");
expect(result).not.toContain("secret proxy details");
});
it("classifies TLS failures while suppressing certificate and path details", () => {
const error = Object.assign(
new Error("certificate for internal.example from /Users/test/corp.pem was rejected"),
{ code: "UNABLE_TO_VERIFY_LEAF_SIGNATURE" },
);
const result = formatWebFetchNetworkFailure(error);
expect(result).toBe(
"request failed [network_diagnostic category=tls; codes=UNABLE_TO_VERIFY_LEAF_SIGNATURE]",
);
expect(result).not.toContain("internal.example");
expect(result).not.toContain("/Users/test");
});
it("uses a generic network category when the cause has no approved code", () => {
const error = Object.assign(new Error("failed near /private/project/token.txt"), {
code: "E_PRIVATE_SECRET",
});
expect(formatWebFetchNetworkFailure(error)).toBe(
"request failed [network_diagnostic category=network; codes=unavailable]",
);
});
it("returns the safe diagnostic through the web_fetch tool error result", async () => {
const originalFetch = globalThis.fetch;
const cause = Object.assign(new Error("lookup secret.proxy.local failed"), {
code: "ENOTFOUND",
});
globalThis.fetch = async () => {
throw new TypeError("fetch failed", { cause });
};
try {
const result = await webFetchTool.execute(
{ url: "https://example.com/research" },
{ state: {} } as any,
);
expect(result.isError).toBe(true);
expect(result.content).toBe(
"Error fetching https://example.com/research: "
+ "fetch failed [network_diagnostic category=dns; codes=ENOTFOUND]",
);
expect(result.content).not.toContain("secret.proxy.local");
} finally {
globalThis.fetch = originalFetch;
}
});
});
// ── Blocked and client-rendered responses ───────────────────────────────────
//
// Scenario source: a research run reported Trustpilot, Reddit, and YouTube as
// "no content available". Reproducing those three fetches showed the pages were
// reachable and the extractor was losing them:
// - Trustpilot answered 403 with an AWS WAF interstitial; the body was
// discarded, so only a bare status reached the model.
// - Reddit answered 200 with an 8 KB JavaScript proof-of-work shell whose
// readable text was the word "Reddit"; it was reported as a SUCCESS and
// cached, so the empty body could not be told apart from an empty page.
// - YouTube answered 200 with 97% of its bytes inside <script>, which the
// extractor strips first, leaving footer boilerplate — also a "success".
/** Shape of the real Trustpilot interstitial: an AWS WAF challenge, 403. */
const AWS_WAF_INTERSTITIAL_HTML =
'<!doctype html><html lang="en"><head><title>Verifying Connection</title>'
+ '<script src="https://example.edge.sdk.awswaf.com/challenge.js" defer></script></head>'
+ '<body><div class="container"><h1>Verifying your connection...</h1>'
+ '<p class="status">Please wait while we verify your browser.</p></div></body></html>';
/** Shape of the real Reddit shell: HTTP 200, title only, body built by script. */
function jsChallengeShellHtml(): string {
const style = "main{align-items:center;display:flex;height:100vh;justify-content:center}".repeat(120);
return '<!doctype html><html lang="en"><head><title>Reddit</title>'
+ '<script nonce="x">document.addEventListener("DOMContentLoaded",async function(){'
+ 'var e=document.forms[0];e.elements.namedItem("solution").value=await solve();e.requestSubmit()},{once:!0});</script>'
+ `<style>${style}</style></head><body><main><form></form></main></body></html>`;
}
/** Shape of a script-delivered page: the body exists, but only inside JSON.
* The payload is kept large on purpose — the real page measured 1.37 MB of
* script against 225 characters of extractable text, and a toy-sized fixture
* would not reach the ratio this rule is meant to catch. */
function scriptDeliveredPageHtml(): string {
const payload = JSON.stringify({
contents: Array.from({ length: 20_000 }, (_, i) => `videoRenderer-${i}`),
});
return '<!doctype html><html><head><title>Rick Astley - Never Gonna Give You Up</title></head>'
+ `<body><div id="app"></div><script>var ytInitialData = ${payload};</script>`
+ "<footer>AboutPressCopyrightContact usCreatorsAdvertiseDevelopersTermsPrivacy</footer></body></html>";
}
const ARTICLE_BODY = "Higgsfield users reported credit consumption issues. ".repeat(30);
const RENDERED_ARTICLE_HTML =
`<!doctype html><html><head><title>Higgsfield reviews</title></head><body><article>${ARTICLE_BODY}</article></body></html>`;
function startStaticServer(status: number, body: string, contentType = "text/html", headers: Record<string, string> = {}): Promise<{ server: Server; url: string }> {
return new Promise((resolve) => {
const server = createServer((_req, res) => {
res.writeHead(status, { "content-type": contentType, ...headers });
res.end(body);
});
server.listen(0, "127.0.0.1", () => {
const { port } = server.address() as AddressInfo;
resolve({ server, url: `http://127.0.0.1:${port}/page` });
});
});
}
function stopServer(server: Server): Promise<void> {
server.closeAllConnections?.();
return new Promise((resolve) => server.close(() => resolve()));
}
describe("web-fetch › blocked and client-rendered responses", () => {
afterEach(() => {
configureWebFetchRenderer(null);
});
// SSE filing probe: the requested .pdf returned HTTP 200, text/html,
// x-tengine-error: denied by bot, and an obfuscated cookie/reload script.
// The server's explicit denial must survive even when its script changes
// or contains no human-readable challenge marker.
it.each([
[200, '<html><script>document.location.reload()</script></html>'],
[200, `<html><script>${"/* obfuscated challenge */".repeat(300)}document.location.reload()</script></html>`],
[403, ""],
])("recognizes a header-declared bot denial behind HTTP %i (%#)", async (status, body) => {
const { server, url } = await startStaticServer(status,
body, "text/html",
{ "x-tengine-error": "denied by bot" });
try {
const state = {};
const result = await webFetchTool.execute({ url }, { state });
expect(result.isError).toBe(true);
expect(result.content).toContain("WAF_OR_BOT_CHECK");
const replay = await webFetchTool.execute({ url }, { state });
expect(replay.isError).toBe(true);
expect(replay.content).toContain("E_WEB_FETCH_NONRETRYABLE_CACHE_HIT");
} finally {
await stopServer(server);
}
});
it("does not let a bot-denial header bypass the response-size failure boundary", async () => {
const { server, url } = await startStaticServer(200, "", "text/html", {
"x-tengine-error": "denied by bot",
"content-length": String(MAX_WEB_FETCH_RESPONSE_BYTES + 1),
});
let renderCalls = 0;
configureWebFetchRenderer(async () => {
renderCalls += 1;
return { html: RENDERED_ARTICLE_HTML };
});
try {
const result = await webFetchTool.execute({ url }, { state: {} });
expect(result.isError).toBe(true);
expect(result.content).toContain("E_FETCH_RESPONSE_TOO_LARGE");
expect(renderCalls).toBe(0);
} finally {
await stopServer(server);
}
});
it("bounds blocked-origin attempts while allowing a readable filing on another source", async () => {
const denied = await startStaticServer(200, "<html><script>reload()</script></html>",
"text/html", { "x-tengine-error": "denied by bot" });
const mirror = await startStaticServer(200,
'<html><title>Issuer interim report</title><article><h1>2026 interim report</h1>'
+ '<p>Unit: CNY. Reporting period: January–June 2026.</p>'
+ '<table><tr><th>Revenue</th><td>1200000</td></tr>'
+ '<tr><th>Net profit</th><td>150000</td></tr></table></article></html>');
let deniedRequests = 0;
denied.server.on("request", () => { deniedRequests += 1; });
try {
const state = {};
for (const suffix of ["?filing=1", "?filing=2"]) {
const failed = await webFetchTool.execute({ url: denied.url + suffix }, { state });
expect(failed.isError).toBe(true);
expect(failed.content).toContain("WAF_OR_BOT_CHECK");
}
const blocked = await webFetchTool.execute({ url: denied.url + "?filing=3" }, { state });
expect(blocked.isError).toBe(true);
expect(blocked.content).toContain("E_WEB_FETCH_ORIGIN_AUTH_BLOCKED");
expect(deniedRequests).toBe(2);
const recovered = await webFetchTool.execute({ url: mirror.url }, { state });
expect(recovered.isError).toBeFalsy();
expect(recovered.content).toContain(`URL: ${mirror.url}`);
expect(recovered.content).toContain("Reporting period: January–June 2026.");
expect(recovered.content).toContain("Revenue 1200000");
expect(recovered.content).toContain("Net profit 150000");
} finally {
await stopServer(denied.server);
await stopServer(mirror.server);
}
});
it.each([{}, { server: "Tengine" }, { "x-tengine-error": "unrelated diagnostic" }])(
"does not infer a bot denial from a server brand or an unrelated header: %j", async (headers) => {
const { server, url } = await startStaticServer(200, RENDERED_ARTICLE_HTML, "text/html", headers);
try {
const result = await webFetchTool.execute({ url }, { state: {} });
expect(result.isError).toBeFalsy();
expect(result.content).toContain(ARTICLE_BODY.trim());
} finally {
await stopServer(server);
}
},
);
it.each([true, false])("keeps the existing browser recovery evidence boundary (readable=%s)", async (readable) => {
const { server, url } = await startStaticServer(200,
'<html><script>document.location.reload()</script></html>', "text/html",
{ "x-tengine-error": "denied by bot" });
let renderCalls = 0;
configureWebFetchRenderer(async () => {
renderCalls += 1;
return { html: readable ? RENDERED_ARTICLE_HTML : jsChallengeShellHtml() };
});
try {
const result = await webFetchTool.execute({ url }, { state: {} });
expect(renderCalls).toBe(1);
if (readable) {
expect(result.isError).toBeFalsy();
expect(result.content).toContain(ARTICLE_BODY.trim());
expect(result.content).toContain("Retrieved by: browser rendering");
} else {
expect(result.isError).toBe(true);
expect(result.content).toContain("WAF_OR_BOT_CHECK");
expect(result.content).not.toContain("Retrieved by: browser rendering");
}
} finally {
await stopServer(server);
}
});
it("names the bot check behind a 403 instead of reporting only the status", async () => {
const { server, url } = await startStaticServer(403, AWS_WAF_INTERSTITIAL_HTML);
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
expect(result.isError).toBe(true);
expect(result.content).toContain("HTTP 403");
// Without the body the model cannot tell "needs credentials" from
// "blocked as a bot", and reports the source as unavailable.
expect(result.content).toContain("WAF_OR_BOT_CHECK");
} finally {
await stopServer(server);
}
});
it("fails a 200 JavaScript challenge shell instead of returning it as a successful fetch", async () => {
const { server, url } = await startStaticServer(200, jsChallengeShellHtml());
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
expect(result.isError).toBe(true);
expect(result.content).toContain("JS_OR_NAV_SHELL");
// The little text there was stays visible so the model can judge it.
expect(result.content).toContain("Extracted text preview:");
} finally {
await stopServer(server);
}
});
it("does not replay a shell as a cached success when the same URL is fetched again", async () => {
const { server, url } = await startStaticServer(200, jsChallengeShellHtml());
try {
const state = {};
await webFetchTool.execute({ url }, { state } as any);
const second = await webFetchTool.execute({ url }, { state } as any);
// The run cache previously replayed the empty body as WEB_FETCH_RUN_CACHE_HIT,
// which reads as "already fetched successfully".
expect(second.isError).toBe(true);
expect(second.content).toContain("E_WEB_FETCH_NONRETRYABLE_CACHE_HIT");
} finally {
await stopServer(server);
}
});
it("flags a page whose body is delivered only inside script payloads", () => {
const raw = scriptDeliveredPageHtml();
const issue = classifyFetchContent(
"https://www.youtube.com/watch?v=x",
"Rick Astley - Never Gonna Give You Up",
raw,
htmlToText(raw),
);
expect(issue).toMatchObject({ code: "JS_OR_NAV_SHELL" });
});
it("does not flag a genuinely short page that carries its own text", () => {
const raw = "<html><head><title>Status</title></head><body><p>All systems operational. "
+ "Last incident: none in the past 90 days.</p></body></html>";
expect(classifyFetchContent("https://example.com/status", "Status", raw, htmlToText(raw))).toBeNull();
});
it("does not flag a markup-heavy page that still carries a real body", () => {
const rows = '<tr><td>row</td><td>0000</td></tr>'.repeat(4000);
const article = "Quarterly revenue grew twenty percent across every region. ".repeat(90);
const raw = `<html><head><title>Results</title></head><body><article>${article}</article><table>${rows}</table></body></html>`;
expect(classifyFetchContent("https://example.com/results", "Results", raw, htmlToText(raw))).toBeNull();
});
});
describe("web-fetch › browser render fallback", () => {
afterEach(() => {
configureWebFetchRenderer(null);
});
it("returns the rendered body when the direct response was a shell", async () => {
const { server, url } = await startStaticServer(200, jsChallengeShellHtml());
configureWebFetchRenderer(async () => ({ html: RENDERED_ARTICLE_HTML }));
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
expect(result.isError).toBeFalsy();
expect(result.content).toContain("Higgsfield users reported credit consumption issues.");
// Provenance must survive into the evidence ledger.
expect(result.content).toContain("Retrieved by: browser rendering");
} finally {
await stopServer(server);
}
});
it("keeps the failure when rendering returns another shell", async () => {
const { server, url } = await startStaticServer(200, jsChallengeShellHtml());
configureWebFetchRenderer(async () => ({ html: jsChallengeShellHtml() }));
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
// Trusting a second shell would recreate the exact defect this fallback exists to fix.
expect(result.isError).toBe(true);
expect(result.content).toContain("JS_OR_NAV_SHELL");
expect(result.content).not.toContain("Retrieved by: browser rendering");
} finally {
await stopServer(server);
}
});
it("keeps the direct failure when the renderer itself fails", async () => {
const { server, url } = await startStaticServer(200, jsChallengeShellHtml());
configureWebFetchRenderer(async () => {
throw new Error("renderer unavailable");
});
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
expect(result.isError).toBe(true);
expect(result.content).toContain("JS_OR_NAV_SHELL");
expect(result.content).not.toContain("renderer unavailable");
} finally {
await stopServer(server);
}
});
it("does not spend a render on a page the site reports as missing", async () => {
const missing = '<html><head><title>Page not found</title></head><body>'
+ "<p>The page you requested could not be found.</p></body></html>";
const { server, url } = await startStaticServer(200, missing);
let renderCalls = 0;
configureWebFetchRenderer(async () => {
renderCalls += 1;
return { html: RENDERED_ARTICLE_HTML };
});
try {
const result = await webFetchTool.execute({ url }, { state: {} } as any);
expect(result.isError).toBe(true);
expect(result.content).toContain("PAGE_NOT_FOUND");
expect(renderCalls).toBe(0);
} finally {
await stopServer(server);
}
});
});