Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(playground): replace the Python config editor with a JSON params editor
ntohidi9340d09b260b
diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.htmlindex e874da47..f905652f 100644--- a/deploy/docker/static/playground/index.html+++ b/deploy/docker/static/playground/index.html@@ -33,10 +33,10 @@<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css"><script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js"></script>- <!-- CodeMirror (python mode) -->+ <!-- CodeMirror (JSON mode) --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.css"><script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.js"></script>- <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/python/python.min.js"></script>+ <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/javascript/javascript.min.js"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/edit/matchbrackets.min.js"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/selection/active-line.min.js"></script><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/theme/darcula.min.css">@@ -251,7 +251,7 @@<!-- Advanced config for /crawl endpoints --><details id="adv-config" class="mb-4"><summary class="text-sm text-secondary cursor-pointer">Advanced Config <span- class="text-xs text-primary">(Python → auto‑JSON)</span></summary>+ class="text-xs text-primary">(JSON params)</span></summary><!-- Toolbar --><div class="flex items-center justify-end space-x-3 mt-2">@@ -498,12 +498,24 @@});// ================ ADVANCED CONFIG EDITOR ================+ // The server validates every field against UNTRUSTED_FIELD_ALLOWLIST, so the+ // editor holds the `params` object literally rather than a Python snippet:+ // /config/dump stopped eval-ing code in the 0.8.x security work (issue #2260).+ const TEMPLATES = {+ CrawlerRunConfig: `{+ "stream": true,+ "cache_mode": "bypass"+}`,+ BrowserConfig: `{+ "headless": true,+ "viewport_width": 1280,+ "viewport_height": 720+}`,+ };+const cm = CodeMirror(document.getElementById('adv-editor'), {- value: `CrawlerRunConfig(- stream=True,- cache_mode=CacheMode.BYPASS,-)`,- mode: 'python',+ value: TEMPLATES.CrawlerRunConfig,+ mode: 'application/json',lineNumbers: true,theme: 'darcula',tabSize: 4,@@ -513,23 +525,9 @@lineWrapping: true,});- const TEMPLATES = {- CrawlerRunConfig: `CrawlerRunConfig(- stream=True,- cache_mode=CacheMode.BYPASS,-)`,- BrowserConfig: `BrowserConfig(- headless=True,- extra_args=[- "--no-sandbox",- "--disable-gpu",- ],-)`,- };-document.getElementById('cfg-type').addEventListener('change', (e) => {cm.setValue(TEMPLATES[e.target.value]);- document.getElementById('cfg-status').textContent = '';+ setCfgStatus('', 'text-secondary');});// Handle endpoint selection change to show appropriate options@@ -539,6 +537,10 @@const llmOptions = document.getElementById('llm-options');const advConfig = document.getElementById('adv-config');+ // Advanced Config is /crawl-only; clear its status so a previous+ // run's '✖ config error' does not linger on md/llm (#2260).+ setCfgStatus('', 'text-secondary');+// Hide all option sections firstmdOptions.classList.add('hidden');llmOptions.classList.add('hidden');@@ -559,30 +561,46 @@}});- async function pyConfigToJson() {- const code = cm.getValue().trim();- if (!code) return {};+ function setCfgStatus(text, cls) {+ const el = document.getElementById('cfg-status');+ el.textContent = text;+ el.className = 'text-xs ml-2 ' + cls;+ }++ // Validate the editor's params against /config/dump and return the+ // normalized { type, params } the crawl endpoints expect. Throws with the+ // server's message so the caller can surface the offending field.+ async function validateConfig() {+ const text = cm.getValue().trim();+ if (!text) return {};++ let params;+ try {+ params = JSON.parse(text);+ } catch (err) {+ setCfgStatus('✖ not valid JSON', 'text-red-400');+ throw new Error('Advanced Config is not valid JSON: ' + err.message);+ }+ if (params === null || typeof params !== 'object' || Array.isArray(params)) {+ setCfgStatus('✖ expected a JSON object', 'text-red-400');+ throw new Error('Advanced Config must be a JSON object of parameters');+ }+ if (!Object.keys(params).length) return {};- // Server requires `type` alongside `code` (CrawlerRunConfig | BrowserConfig).- // The UI already tracks this in #cfg-type; omitting it always 400s Advanced Config.const cfgType = document.getElementById('cfg-type').value;const res = await authFetch('/config/dump', {method: 'POST',headers: { 'Content-Type': 'application/json' },- body: JSON.stringify({ type: cfgType, code }),+ body: JSON.stringify({ type: cfgType, params }),});- const statusEl = document.getElementById('cfg-status');if (!res.ok) {const msg = await res.text();- statusEl.textContent = '✖ config error';- statusEl.className = 'text-xs text-red-400';+ setCfgStatus('✖ config error', 'text-red-400');throw new Error(msg || 'Invalid config');}- statusEl.textContent = '✓ parsed';- statusEl.className = 'text-xs text-green-400';-+ setCfgStatus('✓ validated', 'text-green-400');return await res.json();}@@ -686,12 +704,12 @@async function runCrawl() {const endpoint = document.getElementById('endpoint').value;const urls = document.getElementById('urls').value.trim().split(/\n/).filter(u => u);- // 1) grab python from CodeMirror, validate via /config/dump.+ // 1) grab the params from CodeMirror, validate via /config/dump.// Advanced Config is /crawl-only, so md/llm skip this entirely.let advConfig = {};if (endpoint === 'crawl' || endpoint === 'crawl_stream') {try {- const cfgJson = await pyConfigToJson(); // may throw+ const cfgJson = await validateConfig(); // may throwif (Object.keys(cfgJson).length) {const cfgType = document.getElementById('cfg-type').value;advConfig = cfgType === 'CrawlerRunConfig'@@ -699,18 +717,13 @@: { browser_config: cfgJson };}} catch (err) {- const codeText = cm.getValue();- const streamFlag = /stream\s*=\s*True/i.test(codeText);- if (streamFlag) {- // Fallback: minimal dump-shaped config so shouldUseStream + server load agree- advConfig = { crawler_config: { type: 'CrawlerRunConfig', params: { stream: true } } };- } else {- updateStatus('error');- document.querySelector('#response-content code').textContent =- JSON.stringify({ error: err.message }, null, 2);- forceHighlightElement(document.querySelector('#response-content code'));- return; // stop run- }+ // No fallback: a rejected config used to be swapped for a+ // stream-only one, which silently discarded everything else (#2260).+ updateStatus('error');+ document.querySelector('#response-content code').textContent =+ JSON.stringify({ error: err.message }, null, 2);+ forceHighlightElement(document.querySelector('#response-content code'));+ return; // stop run}}diff --git a/deploy/docker/tests/test_playground_config_protocol.py b/deploy/docker/tests/test_playground_config_protocol.pynew file mode 100644index 00000000..dcb330e2--- /dev/null+++ b/deploy/docker/tests/test_playground_config_protocol.py@@ -0,0 +1,127 @@+#!/usr/bin/env python3+"""The playground's Advanced Config panel must speak the protocol the server+still accepts.++Issue #2260: the panel posted {type, code} to /config/dump long after the 0.8.x+security work dropped the eval-based `code` path. `code` is forbidden under the+untrusted trust boundary, so the pre-flight always 400'd; a regex fallback in+runCrawl() then swapped the user's config for a stream-only one, so everything+else they typed was silently discarded.++These tests read the shipped page as text -- no browser, no server boot -- and+check the templates against the real untrusted gate.+"""++import json+import os+import re+import unittest++from crawl4ai.async_configs import (+ UNTRUSTED_FIELD_ALLOWLIST,+ BrowserConfig,+ CrawlerRunConfig,+ Provenance,+)++PLAYGROUND = os.path.join(+ os.path.dirname(os.path.abspath(__file__)), "..", "static", "playground", "index.html"+)+CONFIG_CLASSES = {"CrawlerRunConfig": CrawlerRunConfig, "BrowserConfig": BrowserConfig}+++def read_page() -> str:+ with open(PLAYGROUND, encoding="utf-8") as fh:+ return fh.read()+++def read_templates() -> dict:+ """Pull the TEMPLATES literal out of the page."""+ block = re.search(r"const TEMPLATES = \{(.*?)\n \};", read_page(), re.S)+ assert block, "TEMPLATES literal not found in the playground"+ return dict(re.findall(r"(\w+): `(\{.*?\})`", block.group(1), re.S))+++class TestConfigProtocol(unittest.TestCase):+ def test_config_dump_is_not_sent_a_code_field(self):+ """`code` is forbidden under UNTRUSTED, so sending it always 400s."""+ page = read_page()+ call = re.search(r"authFetch\('/config/dump'.*?\}\);", page, re.S)+ self.assertIsNotNone(call, "/config/dump call not found")+ self.assertNotIn("code", call.group(0).split("body:")[1],+ "playground must not post a `code` field to /config/dump")+ self.assertIn("params", call.group(0),+ "playground must post the {type, params} shape /config/dump accepts")++ def test_rejected_config_aborts_instead_of_falling_back(self):+ """The stream=True regex fallback is what made the config loss silent."""+ page = read_page()+ self.assertNotIn("streamFlag", page, "the regex fallback must be gone")+ self.assertNotRegex(page, r"stream\\s\*=\\s\*True",+ "the editor text must not be regex-scraped for stream")+++class TestTemplatesSurviveTheUntrustedGate(unittest.TestCase):+ def test_templates_are_valid_json_objects(self):+ templates = read_templates()+ self.assertEqual(set(templates), set(CONFIG_CLASSES))+ for name, body in templates.items():+ with self.subTest(config=name):+ params = json.loads(body)+ self.assertIsInstance(params, dict)+ self.assertTrue(params, "template must not be empty")++ def test_every_template_field_is_allowlisted(self):+ """A template field outside the allowlist would 400 on every run."""+ for name, body in read_templates().items():+ with self.subTest(config=name):+ unknown = set(json.loads(body)) - UNTRUSTED_FIELD_ALLOWLIST[name]+ self.assertFalse(unknown, f"{name} template uses forbidden field(s): {unknown}")++ def test_every_template_field_reaches_the_config_object(self):+ """Load each template the way /config/dump does and read the result back.++ dump() omits values equal to the class default, so assert on the object,+ not on the echoed payload.+ """+ for name, body in read_templates().items():+ with self.subTest(config=name):+ params = json.loads(body)+ obj = CONFIG_CLASSES[name].load(+ {"type": name, "params": params}, provenance=Provenance.UNTRUSTED+ )+ applied = {key: getattr(obj, key) for key in params}+ self.assertEqual(applied, params)++ def test_stream_reaches_the_shape_should_use_stream_reads(self):+ """runCrawl picks the streaming endpoint off crawler_config.params.stream."""+ params = json.loads(read_templates()["CrawlerRunConfig"])+ dumped = CrawlerRunConfig.load(+ {"type": "CrawlerRunConfig", "params": params}, provenance=Provenance.UNTRUSTED+ ).dump()+ self.assertIs(dumped["params"].get("stream"), True)++ def test_the_old_code_protocol_is_still_refused(self):+ """Guards the reason for this change: {type, code} cannot come back."""+ for name, cls in CONFIG_CLASSES.items():+ with self.subTest(config=name):+ with self.assertRaises(Exception) as ctx:+ cls.load({"type": name, "code": f"{name}()"}, provenance=Provenance.UNTRUSTED)+ self.assertIn("code", str(ctx.exception))+++class TestConfigStatusIsReset(unittest.TestCase):+ def test_status_is_cleared_when_the_endpoint_changes(self):+ """md/llm skip validation, so a stale '✖ config error' used to linger."""+ page = read_page()+ handler = re.search(+ r"getElementById\('endpoint'\)\.addEventListener\('change'.*?\n \}\);",+ page, re.S,+ )+ self.assertIsNotNone(handler, "endpoint change handler not found")+ self.assertIn("setCfgStatus('',", handler.group(0),+ "switching endpoint must clear #cfg-status")+++if __name__ == "__main__":+ unittest.main()