Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(playground): replace the Python config editor with a JSON params editor

ntohidi9340d09b260b
diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index e874da47..f905652f 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -33,10 +33,10 @@
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js"></script>
- <!-- CodeMirror (python mode) -->
+ <!-- CodeMirror (JSON mode) -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.js"></script>
- <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/python/python.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/javascript/javascript.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/edit/matchbrackets.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/selection/active-line.min.js"></script>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/theme/darcula.min.css">
@@ -251,7 +251,7 @@
<!-- Advanced config for /crawl endpoints -->
<details id="adv-config" class="mb-4">
<summary class="text-sm text-secondary cursor-pointer">Advanced Config <span
- class="text-xs text-primary">(Python → auto‑JSON)</span></summary>
+ class="text-xs text-primary">(JSON params)</span></summary>
<!-- Toolbar -->
<div class="flex items-center justify-end space-x-3 mt-2">
@@ -498,12 +498,24 @@
});
// ================ ADVANCED CONFIG EDITOR ================
+ // The server validates every field against UNTRUSTED_FIELD_ALLOWLIST, so the
+ // editor holds the `params` object literally rather than a Python snippet:
+ // /config/dump stopped eval-ing code in the 0.8.x security work (issue #2260).
+ const TEMPLATES = {
+ CrawlerRunConfig: `{
+ "stream": true,
+ "cache_mode": "bypass"
+}`,
+ BrowserConfig: `{
+ "headless": true,
+ "viewport_width": 1280,
+ "viewport_height": 720
+}`,
+ };
+
const cm = CodeMirror(document.getElementById('adv-editor'), {
- value: `CrawlerRunConfig(
- stream=True,
- cache_mode=CacheMode.BYPASS,
-)`,
- mode: 'python',
+ value: TEMPLATES.CrawlerRunConfig,
+ mode: 'application/json',
lineNumbers: true,
theme: 'darcula',
tabSize: 4,
@@ -513,23 +525,9 @@
lineWrapping: true,
});
- const TEMPLATES = {
- CrawlerRunConfig: `CrawlerRunConfig(
- stream=True,
- cache_mode=CacheMode.BYPASS,
-)`,
- BrowserConfig: `BrowserConfig(
- headless=True,
- extra_args=[
- "--no-sandbox",
- "--disable-gpu",
- ],
-)`,
- };
-
document.getElementById('cfg-type').addEventListener('change', (e) => {
cm.setValue(TEMPLATES[e.target.value]);
- document.getElementById('cfg-status').textContent = '';
+ setCfgStatus('', 'text-secondary');
});
// Handle endpoint selection change to show appropriate options
@@ -539,6 +537,10 @@
const llmOptions = document.getElementById('llm-options');
const advConfig = document.getElementById('adv-config');
+ // Advanced Config is /crawl-only; clear its status so a previous
+ // run's '✖ config error' does not linger on md/llm (#2260).
+ setCfgStatus('', 'text-secondary');
+
// Hide all option sections first
mdOptions.classList.add('hidden');
llmOptions.classList.add('hidden');
@@ -559,30 +561,46 @@
}
});
- async function pyConfigToJson() {
- const code = cm.getValue().trim();
- if (!code) return {};
+ function setCfgStatus(text, cls) {
+ const el = document.getElementById('cfg-status');
+ el.textContent = text;
+ el.className = 'text-xs ml-2 ' + cls;
+ }
+
+ // Validate the editor's params against /config/dump and return the
+ // normalized { type, params } the crawl endpoints expect. Throws with the
+ // server's message so the caller can surface the offending field.
+ async function validateConfig() {
+ const text = cm.getValue().trim();
+ if (!text) return {};
+
+ let params;
+ try {
+ params = JSON.parse(text);
+ } catch (err) {
+ setCfgStatus('✖ not valid JSON', 'text-red-400');
+ throw new Error('Advanced Config is not valid JSON: ' + err.message);
+ }
+ if (params === null || typeof params !== 'object' || Array.isArray(params)) {
+ setCfgStatus('✖ expected a JSON object', 'text-red-400');
+ throw new Error('Advanced Config must be a JSON object of parameters');
+ }
+ if (!Object.keys(params).length) return {};
- // Server requires `type` alongside `code` (CrawlerRunConfig | BrowserConfig).
- // The UI already tracks this in #cfg-type; omitting it always 400s Advanced Config.
const cfgType = document.getElementById('cfg-type').value;
const res = await authFetch('/config/dump', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ type: cfgType, code }),
+ body: JSON.stringify({ type: cfgType, params }),
});
- const statusEl = document.getElementById('cfg-status');
if (!res.ok) {
const msg = await res.text();
- statusEl.textContent = '✖ config error';
- statusEl.className = 'text-xs text-red-400';
+ setCfgStatus('✖ config error', 'text-red-400');
throw new Error(msg || 'Invalid config');
}
- statusEl.textContent = '✓ parsed';
- statusEl.className = 'text-xs text-green-400';
-
+ setCfgStatus('✓ validated', 'text-green-400');
return await res.json();
}
@@ -686,12 +704,12 @@
async function runCrawl() {
const endpoint = document.getElementById('endpoint').value;
const urls = document.getElementById('urls').value.trim().split(/\n/).filter(u => u);
- // 1) grab python from CodeMirror, validate via /config/dump.
+ // 1) grab the params from CodeMirror, validate via /config/dump.
// Advanced Config is /crawl-only, so md/llm skip this entirely.
let advConfig = {};
if (endpoint === 'crawl' || endpoint === 'crawl_stream') {
try {
- const cfgJson = await pyConfigToJson(); // may throw
+ const cfgJson = await validateConfig(); // may throw
if (Object.keys(cfgJson).length) {
const cfgType = document.getElementById('cfg-type').value;
advConfig = cfgType === 'CrawlerRunConfig'
@@ -699,18 +717,13 @@
: { browser_config: cfgJson };
}
} catch (err) {
- const codeText = cm.getValue();
- const streamFlag = /stream\s*=\s*True/i.test(codeText);
- if (streamFlag) {
- // Fallback: minimal dump-shaped config so shouldUseStream + server load agree
- advConfig = { crawler_config: { type: 'CrawlerRunConfig', params: { stream: true } } };
- } else {
- updateStatus('error');
- document.querySelector('#response-content code').textContent =
- JSON.stringify({ error: err.message }, null, 2);
- forceHighlightElement(document.querySelector('#response-content code'));
- return; // stop run
- }
+ // No fallback: a rejected config used to be swapped for a
+ // stream-only one, which silently discarded everything else (#2260).
+ updateStatus('error');
+ document.querySelector('#response-content code').textContent =
+ JSON.stringify({ error: err.message }, null, 2);
+ forceHighlightElement(document.querySelector('#response-content code'));
+ return; // stop run
}
}
diff --git a/deploy/docker/tests/test_playground_config_protocol.py b/deploy/docker/tests/test_playground_config_protocol.py
new file mode 100644
index 00000000..dcb330e2
--- /dev/null
+++ b/deploy/docker/tests/test_playground_config_protocol.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""The playground's Advanced Config panel must speak the protocol the server
+still accepts.
+
+Issue #2260: the panel posted {type, code} to /config/dump long after the 0.8.x
+security work dropped the eval-based `code` path. `code` is forbidden under the
+untrusted trust boundary, so the pre-flight always 400'd; a regex fallback in
+runCrawl() then swapped the user's config for a stream-only one, so everything
+else they typed was silently discarded.
+
+These tests read the shipped page as text -- no browser, no server boot -- and
+check the templates against the real untrusted gate.
+"""
+
+import json
+import os
+import re
+import unittest
+
+from crawl4ai.async_configs import (
+ UNTRUSTED_FIELD_ALLOWLIST,
+ BrowserConfig,
+ CrawlerRunConfig,
+ Provenance,
+)
+
+PLAYGROUND = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "..", "static", "playground", "index.html"
+)
+CONFIG_CLASSES = {"CrawlerRunConfig": CrawlerRunConfig, "BrowserConfig": BrowserConfig}
+
+
+def read_page() -> str:
+ with open(PLAYGROUND, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def read_templates() -> dict:
+ """Pull the TEMPLATES literal out of the page."""
+ block = re.search(r"const TEMPLATES = \{(.*?)\n \};", read_page(), re.S)
+ assert block, "TEMPLATES literal not found in the playground"
+ return dict(re.findall(r"(\w+): `(\{.*?\})`", block.group(1), re.S))
+
+
+class TestConfigProtocol(unittest.TestCase):
+ def test_config_dump_is_not_sent_a_code_field(self):
+ """`code` is forbidden under UNTRUSTED, so sending it always 400s."""
+ page = read_page()
+ call = re.search(r"authFetch\('/config/dump'.*?\}\);", page, re.S)
+ self.assertIsNotNone(call, "/config/dump call not found")
+ self.assertNotIn("code", call.group(0).split("body:")[1],
+ "playground must not post a `code` field to /config/dump")
+ self.assertIn("params", call.group(0),
+ "playground must post the {type, params} shape /config/dump accepts")
+
+ def test_rejected_config_aborts_instead_of_falling_back(self):
+ """The stream=True regex fallback is what made the config loss silent."""
+ page = read_page()
+ self.assertNotIn("streamFlag", page, "the regex fallback must be gone")
+ self.assertNotRegex(page, r"stream\\s\*=\\s\*True",
+ "the editor text must not be regex-scraped for stream")
+
+
+class TestTemplatesSurviveTheUntrustedGate(unittest.TestCase):
+ def test_templates_are_valid_json_objects(self):
+ templates = read_templates()
+ self.assertEqual(set(templates), set(CONFIG_CLASSES))
+ for name, body in templates.items():
+ with self.subTest(config=name):
+ params = json.loads(body)
+ self.assertIsInstance(params, dict)
+ self.assertTrue(params, "template must not be empty")
+
+ def test_every_template_field_is_allowlisted(self):
+ """A template field outside the allowlist would 400 on every run."""
+ for name, body in read_templates().items():
+ with self.subTest(config=name):
+ unknown = set(json.loads(body)) - UNTRUSTED_FIELD_ALLOWLIST[name]
+ self.assertFalse(unknown, f"{name} template uses forbidden field(s): {unknown}")
+
+ def test_every_template_field_reaches_the_config_object(self):
+ """Load each template the way /config/dump does and read the result back.
+
+ dump() omits values equal to the class default, so assert on the object,
+ not on the echoed payload.
+ """
+ for name, body in read_templates().items():
+ with self.subTest(config=name):
+ params = json.loads(body)
+ obj = CONFIG_CLASSES[name].load(
+ {"type": name, "params": params}, provenance=Provenance.UNTRUSTED
+ )
+ applied = {key: getattr(obj, key) for key in params}
+ self.assertEqual(applied, params)
+
+ def test_stream_reaches_the_shape_should_use_stream_reads(self):
+ """runCrawl picks the streaming endpoint off crawler_config.params.stream."""
+ params = json.loads(read_templates()["CrawlerRunConfig"])
+ dumped = CrawlerRunConfig.load(
+ {"type": "CrawlerRunConfig", "params": params}, provenance=Provenance.UNTRUSTED
+ ).dump()
+ self.assertIs(dumped["params"].get("stream"), True)
+
+ def test_the_old_code_protocol_is_still_refused(self):
+ """Guards the reason for this change: {type, code} cannot come back."""
+ for name, cls in CONFIG_CLASSES.items():
+ with self.subTest(config=name):
+ with self.assertRaises(Exception) as ctx:
+ cls.load({"type": name, "code": f"{name}()"}, provenance=Provenance.UNTRUSTED)
+ self.assertIn("code", str(ctx.exception))
+
+
+class TestConfigStatusIsReset(unittest.TestCase):
+ def test_status_is_cleared_when_the_endpoint_changes(self):
+ """md/llm skip validation, so a stale '✖ config error' used to linger."""
+ page = read_page()
+ handler = re.search(
+ r"getElementById\('endpoint'\)\.addEventListener\('change'.*?\n \}\);",
+ page, re.S,
+ )
+ self.assertIsNotNone(handler, "endpoint change handler not found")
+ self.assertIn("setCfgStatus('',", handler.group(0),
+ "switching endpoint must clear #cfg-status")
+
+
+if __name__ == "__main__":
+ unittest.main()