Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(cli): fail early when -o json has no extraction strategy

Soham Kukreti1e54fa7c86f9
diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py
index 4473cd54..759f41bc 100644
--- a/crawl4ai/cli.py
+++ b/crawl4ai/cli.py
@@ -175,7 +175,7 @@ def show_examples():
crwl https://example.com -o markdown
# Verbose JSON output with cache bypass
- crwl https://example.com -o json -v --bypass-cache
+ crwl https://example.com -o all -v --bypass-cache
2️⃣ Using Config Files:
# Using browser and crawler configs
@@ -516,7 +516,7 @@ async def crawl_with_profile_cli(profile_path, url):
# Ask for output format
output_format = Prompt.ask(
"[cyan]Output format[/cyan]",
- choices=["all", "json", "markdown", "md", "title"],
+ choices=["all", "markdown", "md", "title"],
default="markdown"
)
@@ -1204,6 +1204,12 @@ Always return valid, properly formatted JSON."""
else:
ensure_ascii = config.get("JSON_ENSURE_ASCII", USER_SETTINGS["JSON_ENSURE_ASCII"]["default"])
+ if output == "json" and not question and crawler_cfg.extraction_strategy is None:
+ raise click.UsageError(
+ "-o json needs an extraction strategy (-j or -e). "
+ "Use -o all for the full crawl result."
+ )
+
# Run crawler
result : CrawlResult = anyio.run(
run_crawler,
@@ -1242,7 +1248,6 @@ Always return valid, properly formatted JSON."""
else:
click.echo(json.dumps(main_result.model_dump(), indent=2, ensure_ascii=ensure_ascii))
elif output == "json":
- print(main_result.extracted_content)
extracted_items = json.loads(main_result.extracted_content)
click.echo(json.dumps(extracted_items, indent=2, ensure_ascii=ensure_ascii))
@@ -1292,6 +1297,8 @@ Always return valid, properly formatted JSON."""
else:
f.write(main_result.markdown.fit_markdown)
+ except click.ClickException:
+ raise
except Exception as e:
raise click.ClickException(str(e))
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index ed8f6d71..1cf4b166 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -253,5 +253,20 @@ class TestDeepCrawlOutput:
assert 'Content here' in result.output
+def test_json_output_without_extractor_fails_early(runner):
+ with patch("crawl4ai.cli.anyio.run") as run:
+ result = runner.invoke(cli, ["crawl", "https://example.com", "-o", "json"])
+ assert result.exit_code == 2
+ assert "-o json needs an extraction strategy" in result.output
+ run.assert_not_called() # browser never started
+
+
+def test_json_output_with_question_is_allowed(runner):
+ with patch("crawl4ai.cli.anyio.run") as run, \
+ patch("crawl4ai.cli.setup_llm_config", return_value=("openai/gpt-4o", "x")):
+ runner.invoke(cli, ["crawl", "https://example.com", "-o", "json", "-q", "hi"])
+ run.assert_called() # no early error
+
+
if __name__ == '__main__':
pytest.main(['-v', '-s', '--tb=native', __file__])