Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

chore: tidy the repository root

ntohidi5b8fd5b64e9e
diff --git a/.env.txt b/.env.example
similarity index 100%
rename from .env.txt
rename to .env.example
diff --git a/JOURNAL.md b/docs/deprecated/JOURNAL.md
similarity index 100%
rename from JOURNAL.md
rename to docs/deprecated/JOURNAL.md
diff --git a/test_llm_webhook_feature.py b/test_llm_webhook_feature.py
deleted file mode 100644
index 98133e82..00000000
--- a/test_llm_webhook_feature.py
+++ /dev/null
@@ -1,401 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script to validate webhook implementation for /llm/job endpoint.
-
-This tests that the /llm/job endpoint now supports webhooks
-following the same pattern as /crawl/job.
-"""
-
-import sys
-import os
-
-# Add deploy/docker to path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'deploy', 'docker'))
-
-def test_llm_job_payload_model():
- """Test that LlmJobPayload includes webhook_config field"""
- print("=" * 60)
- print("TEST 1: LlmJobPayload Model")
- print("=" * 60)
-
- try:
- from job import LlmJobPayload
- from schemas import WebhookConfig
- from pydantic import ValidationError
-
- # Test with webhook_config
- payload_dict = {
- "url": "https://example.com",
- "q": "Extract main content",
- "schema": None,
- "cache": False,
- "provider": None,
- "webhook_config": {
- "webhook_url": "https://myapp.com/webhook",
- "webhook_data_in_payload": True,
- "webhook_headers": {"X-Secret": "token"}
- }
- }
-
- payload = LlmJobPayload(**payload_dict)
-
- print(f"✅ LlmJobPayload accepts webhook_config")
- print(f" - URL: {payload.url}")
- print(f" - Query: {payload.q}")
- print(f" - Webhook URL: {payload.webhook_config.webhook_url}")
- print(f" - Data in payload: {payload.webhook_config.webhook_data_in_payload}")
-
- # Test without webhook_config (should be optional)
- minimal_payload = {
- "url": "https://example.com",
- "q": "Extract content"
- }
-
- payload2 = LlmJobPayload(**minimal_payload)
- assert payload2.webhook_config is None, "webhook_config should be optional"
- print(f"✅ LlmJobPayload works without webhook_config (optional)")
-
- return True
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_handle_llm_request_signature():
- """Test that handle_llm_request accepts webhook_config parameter"""
- print("\n" + "=" * 60)
- print("TEST 2: handle_llm_request Function Signature")
- print("=" * 60)
-
- try:
- from api import handle_llm_request
- import inspect
-
- sig = inspect.signature(handle_llm_request)
- params = list(sig.parameters.keys())
-
- print(f"Function parameters: {params}")
-
- if 'webhook_config' in params:
- print(f"✅ handle_llm_request has webhook_config parameter")
-
- # Check that it's optional with default None
- webhook_param = sig.parameters['webhook_config']
- if webhook_param.default is None or webhook_param.default == inspect.Parameter.empty:
- print(f"✅ webhook_config is optional (default: {webhook_param.default})")
- else:
- print(f"⚠️ webhook_config default is: {webhook_param.default}")
-
- return True
- else:
- print(f"❌ handle_llm_request missing webhook_config parameter")
- return False
-
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_process_llm_extraction_signature():
- """Test that process_llm_extraction accepts webhook_config parameter"""
- print("\n" + "=" * 60)
- print("TEST 3: process_llm_extraction Function Signature")
- print("=" * 60)
-
- try:
- from api import process_llm_extraction
- import inspect
-
- sig = inspect.signature(process_llm_extraction)
- params = list(sig.parameters.keys())
-
- print(f"Function parameters: {params}")
-
- if 'webhook_config' in params:
- print(f"✅ process_llm_extraction has webhook_config parameter")
-
- webhook_param = sig.parameters['webhook_config']
- if webhook_param.default is None or webhook_param.default == inspect.Parameter.empty:
- print(f"✅ webhook_config is optional (default: {webhook_param.default})")
- else:
- print(f"⚠️ webhook_config default is: {webhook_param.default}")
-
- return True
- else:
- print(f"❌ process_llm_extraction missing webhook_config parameter")
- return False
-
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_webhook_integration_in_api():
- """Test that api.py properly integrates webhook notifications"""
- print("\n" + "=" * 60)
- print("TEST 4: Webhook Integration in process_llm_extraction")
- print("=" * 60)
-
- try:
- api_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
-
- with open(api_file, 'r') as f:
- api_content = f.read()
-
- # Check for WebhookDeliveryService initialization
- if 'webhook_service = WebhookDeliveryService(config)' in api_content:
- print("✅ process_llm_extraction initializes WebhookDeliveryService")
- else:
- print("❌ Missing WebhookDeliveryService initialization in process_llm_extraction")
- return False
-
- # Check for notify_job_completion calls with llm_extraction
- if 'task_type="llm_extraction"' in api_content:
- print("✅ Uses correct task_type='llm_extraction' for notifications")
- else:
- print("❌ Missing task_type='llm_extraction' in webhook notifications")
- return False
-
- # Count webhook notification calls (should have at least 3: success + 2 failure paths)
- notification_count = api_content.count('await webhook_service.notify_job_completion')
- # Find only in process_llm_extraction function
- llm_func_start = api_content.find('async def process_llm_extraction')
- llm_func_end = api_content.find('\nasync def ', llm_func_start + 1)
- if llm_func_end == -1:
- llm_func_end = len(api_content)
-
- llm_func_content = api_content[llm_func_start:llm_func_end]
- llm_notification_count = llm_func_content.count('await webhook_service.notify_job_completion')
-
- print(f"✅ Found {llm_notification_count} webhook notification calls in process_llm_extraction")
-
- if llm_notification_count >= 3:
- print(f"✅ Sufficient notification points (success + failure paths)")
- else:
- print(f"⚠️ Expected at least 3 notification calls, found {llm_notification_count}")
-
- return True
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_job_endpoint_integration():
- """Test that /llm/job endpoint extracts and passes webhook_config"""
- print("\n" + "=" * 60)
- print("TEST 5: /llm/job Endpoint Integration")
- print("=" * 60)
-
- try:
- job_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'job.py')
-
- with open(job_file, 'r') as f:
- job_content = f.read()
-
- # Find the llm_job_enqueue function
- llm_job_start = job_content.find('async def llm_job_enqueue')
- llm_job_end = job_content.find('\n\n@router', llm_job_start + 1)
- if llm_job_end == -1:
- llm_job_end = job_content.find('\n\nasync def', llm_job_start + 1)
-
- llm_job_func = job_content[llm_job_start:llm_job_end]
-
- # Check for webhook_config extraction
- if 'webhook_config = None' in llm_job_func:
- print("✅ llm_job_enqueue initializes webhook_config variable")
- else:
- print("❌ Missing webhook_config initialization")
- return False
-
- if 'if payload.webhook_config:' in llm_job_func:
- print("✅ llm_job_enqueue checks for payload.webhook_config")
- else:
- print("❌ Missing webhook_config check")
- return False
-
- if 'webhook_config = payload.webhook_config.model_dump(mode=\'json\')' in llm_job_func:
- print("✅ llm_job_enqueue converts webhook_config to dict")
- else:
- print("❌ Missing webhook_config.model_dump conversion")
- return False
-
- if 'webhook_config=webhook_config' in llm_job_func:
- print("✅ llm_job_enqueue passes webhook_config to handle_llm_request")
- else:
- print("❌ Missing webhook_config parameter in handle_llm_request call")
- return False
-
- return True
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_create_new_task_integration():
- """Test that create_new_task stores webhook_config in Redis"""
- print("\n" + "=" * 60)
- print("TEST 6: create_new_task Webhook Storage")
- print("=" * 60)
-
- try:
- api_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
-
- with open(api_file, 'r') as f:
- api_content = f.read()
-
- # Find create_new_task function
- create_task_start = api_content.find('async def create_new_task')
- create_task_end = api_content.find('\nasync def ', create_task_start + 1)
- if create_task_end == -1:
- create_task_end = len(api_content)
-
- create_task_func = api_content[create_task_start:create_task_end]
-
- # Check for webhook_config storage
- if 'if webhook_config:' in create_task_func:
- print("✅ create_new_task checks for webhook_config")
- else:
- print("❌ Missing webhook_config check in create_new_task")
- return False
-
- if 'task_data["webhook_config"] = json.dumps(webhook_config)' in create_task_func:
- print("✅ create_new_task stores webhook_config in Redis task data")
- else:
- print("❌ Missing webhook_config storage in task_data")
- return False
-
- # Check that webhook_config is passed to process_llm_extraction
- if 'webhook_config' in create_task_func and 'background_tasks.add_task' in create_task_func:
- print("✅ create_new_task passes webhook_config to background task")
- else:
- print("⚠️ Could not verify webhook_config passed to background task")
-
- return True
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_pattern_consistency():
- """Test that /llm/job follows the same pattern as /crawl/job"""
- print("\n" + "=" * 60)
- print("TEST 7: Pattern Consistency with /crawl/job")
- print("=" * 60)
-
- try:
- api_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
-
- with open(api_file, 'r') as f:
- api_content = f.read()
-
- # Find handle_crawl_job to compare pattern
- crawl_job_start = api_content.find('async def handle_crawl_job')
- crawl_job_end = api_content.find('\nasync def ', crawl_job_start + 1)
- if crawl_job_end == -1:
- crawl_job_end = len(api_content)
- crawl_job_func = api_content[crawl_job_start:crawl_job_end]
-
- # Find process_llm_extraction
- llm_extract_start = api_content.find('async def process_llm_extraction')
- llm_extract_end = api_content.find('\nasync def ', llm_extract_start + 1)
- if llm_extract_end == -1:
- llm_extract_end = len(api_content)
- llm_extract_func = api_content[llm_extract_start:llm_extract_end]
-
- print("Checking pattern consistency...")
-
- # Both should initialize WebhookDeliveryService
- crawl_has_service = 'webhook_service = WebhookDeliveryService(config)' in crawl_job_func
- llm_has_service = 'webhook_service = WebhookDeliveryService(config)' in llm_extract_func
-
- if crawl_has_service and llm_has_service:
- print("✅ Both initialize WebhookDeliveryService")
- else:
- print(f"❌ Service initialization mismatch (crawl: {crawl_has_service}, llm: {llm_has_service})")
- return False
-
- # Both should call notify_job_completion on success
- crawl_notifies_success = 'status="completed"' in crawl_job_func and 'notify_job_completion' in crawl_job_func
- llm_notifies_success = 'status="completed"' in llm_extract_func and 'notify_job_completion' in llm_extract_func
-
- if crawl_notifies_success and llm_notifies_success:
- print("✅ Both notify on success")
- else:
- print(f"❌ Success notification mismatch (crawl: {crawl_notifies_success}, llm: {llm_notifies_success})")
- return False
-
- # Both should call notify_job_completion on failure
- crawl_notifies_failure = 'status="failed"' in crawl_job_func and 'error=' in crawl_job_func
- llm_notifies_failure = 'status="failed"' in llm_extract_func and 'error=' in llm_extract_func
-
- if crawl_notifies_failure and llm_notifies_failure:
- print("✅ Both notify on failure")
- else:
- print(f"❌ Failure notification mismatch (crawl: {crawl_notifies_failure}, llm: {llm_notifies_failure})")
- return False
-
- print("✅ /llm/job follows the same pattern as /crawl/job")
- return True
-
- except Exception as e:
- print(f"❌ Failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def main():
- """Run all tests"""
- print("\n🧪 LLM Job Webhook Feature Validation")
- print("=" * 60)
- print("Testing that /llm/job now supports webhooks like /crawl/job")
- print("=" * 60 + "\n")
-
- results = []
-
- # Run all tests
- results.append(("LlmJobPayload Model", test_llm_job_payload_model()))
- results.append(("handle_llm_request Signature", test_handle_llm_request_signature()))
- results.append(("process_llm_extraction Signature", test_process_llm_extraction_signature()))
- results.append(("Webhook Integration", test_webhook_integration_in_api()))
- results.append(("/llm/job Endpoint", test_job_endpoint_integration()))
- results.append(("create_new_task Storage", test_create_new_task_integration()))
- results.append(("Pattern Consistency", test_pattern_consistency()))
-
- # Print summary
- print("\n" + "=" * 60)
- print("TEST SUMMARY")
- print("=" * 60)
-
- passed = sum(1 for _, result in results if result)
- total = len(results)
-
- for test_name, result in results:
- status = "✅ PASS" if result else "❌ FAIL"
- print(f"{status} - {test_name}")
-
- print(f"\n{'=' * 60}")
- print(f"Results: {passed}/{total} tests passed")
- print(f"{'=' * 60}")
-
- if passed == total:
- print("\n🎉 All tests passed! /llm/job webhook feature is correctly implemented.")
- print("\n📝 Summary of changes:")
- print(" 1. LlmJobPayload model includes webhook_config field")
- print(" 2. /llm/job endpoint extracts and passes webhook_config")
- print(" 3. handle_llm_request accepts webhook_config parameter")
- print(" 4. create_new_task stores webhook_config in Redis")
- print(" 5. process_llm_extraction sends webhook notifications")
- print(" 6. Follows the same pattern as /crawl/job")
- return 0
- else:
- print(f"\n⚠️ {total - passed} test(s) failed. Please review the output above.")
- return 1
-
-if __name__ == "__main__":
- exit(main())
diff --git a/test_webhook_implementation.py b/test_webhook_implementation.py
deleted file mode 100644
index 072db8b3..00000000
--- a/test_webhook_implementation.py
+++ /dev/null
@@ -1,307 +0,0 @@
-"""
-Simple test script to validate webhook implementation without running full server.
-
-This script tests:
-1. Webhook module imports and syntax
-2. WebhookDeliveryService initialization
-3. Payload construction logic
-4. Configuration parsing
-"""
-
-import sys
-import os
-import json
-from datetime import datetime, timezone
-
-# Add deploy/docker to path to import modules
-# sys.path.insert(0, '/home/user/crawl4ai/deploy/docker')
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'deploy', 'docker'))
-
-def test_imports():
- """Test that all webhook-related modules can be imported"""
- print("=" * 60)
- print("TEST 1: Module Imports")
- print("=" * 60)
-
- try:
- from webhook import WebhookDeliveryService
- print("✅ webhook.WebhookDeliveryService imported successfully")
- except Exception as e:
- print(f"❌ Failed to import webhook module: {e}")
- return False
-
- try:
- from schemas import WebhookConfig, WebhookPayload
- print("✅ schemas.WebhookConfig imported successfully")
- print("✅ schemas.WebhookPayload imported successfully")
- except Exception as e:
- print(f"❌ Failed to import schemas: {e}")
- return False
-
- return True
-
-def test_webhook_service_init():
- """Test WebhookDeliveryService initialization"""
- print("\n" + "=" * 60)
- print("TEST 2: WebhookDeliveryService Initialization")
- print("=" * 60)
-
- try:
- from webhook import WebhookDeliveryService
-
- # Test with default config
- config = {
- "webhooks": {
- "enabled": True,
- "default_url": None,
- "data_in_payload": False,
- "retry": {
- "max_attempts": 5,
- "initial_delay_ms": 1000,
- "max_delay_ms": 32000,
- "timeout_ms": 30000
- },
- "headers": {
- "User-Agent": "Crawl4AI-Webhook/1.0"
- }
- }
- }
-
- service = WebhookDeliveryService(config)
-
- print(f"✅ Service initialized successfully")
- print(f" - Max attempts: {service.max_attempts}")
- print(f" - Initial delay: {service.initial_delay}s")
- print(f" - Max delay: {service.max_delay}s")
- print(f" - Timeout: {service.timeout}s")
-
- # Verify calculations
- assert service.max_attempts == 5, "Max attempts should be 5"
- assert service.initial_delay == 1.0, "Initial delay should be 1.0s"
- assert service.max_delay == 32.0, "Max delay should be 32.0s"
- assert service.timeout == 30.0, "Timeout should be 30.0s"
-
- print("✅ All configuration values correct")
-
- return True
- except Exception as e:
- print(f"❌ Service initialization failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_webhook_config_model():
- """Test WebhookConfig Pydantic model"""
- print("\n" + "=" * 60)
- print("TEST 3: WebhookConfig Model Validation")
- print("=" * 60)
-
- try:
- from schemas import WebhookConfig
- from pydantic import ValidationError
-
- # Test valid config
- valid_config = {
- "webhook_url": "https://example.com/webhook",
- "webhook_data_in_payload": True,
- "webhook_headers": {"X-Secret": "token123"}
- }
-
- config = WebhookConfig(**valid_config)
- print(f"✅ Valid config accepted:")
- print(f" - URL: {config.webhook_url}")
- print(f" - Data in payload: {config.webhook_data_in_payload}")
- print(f" - Headers: {config.webhook_headers}")
-
- # Test minimal config
- minimal_config = {
- "webhook_url": "https://example.com/webhook"
- }
-
- config2 = WebhookConfig(**minimal_config)
- print(f"✅ Minimal config accepted (defaults applied):")
- print(f" - URL: {config2.webhook_url}")
- print(f" - Data in payload: {config2.webhook_data_in_payload}")
- print(f" - Headers: {config2.webhook_headers}")
-
- # Test invalid URL
- try:
- invalid_config = {
- "webhook_url": "not-a-url"
- }
- config3 = WebhookConfig(**invalid_config)
- print(f"❌ Invalid URL should have been rejected")
- return False
- except ValidationError as e:
- print(f"✅ Invalid URL correctly rejected")
-
- return True
- except Exception as e:
- print(f"❌ Model validation test failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_payload_construction():
- """Test webhook payload construction logic"""
- print("\n" + "=" * 60)
- print("TEST 4: Payload Construction")
- print("=" * 60)
-
- try:
- # Simulate payload construction from notify_job_completion
- task_id = "crawl_abc123"
- task_type = "crawl"
- status = "completed"
- urls = ["https://example.com"]
-
- payload = {
- "task_id": task_id,
- "task_type": task_type,
- "status": status,
- "timestamp": datetime.now(timezone.utc).isoformat(),
- "urls": urls
- }
-
- print(f"✅ Basic payload constructed:")
- print(json.dumps(payload, indent=2))
-
- # Test with error
- error_payload = {
- "task_id": "crawl_xyz789",
- "task_type": "crawl",
- "status": "failed",
- "timestamp": datetime.now(timezone.utc).isoformat(),
- "urls": ["https://example.com"],
- "error": "Connection timeout"
- }
-
- print(f"\n✅ Error payload constructed:")
- print(json.dumps(error_payload, indent=2))
-
- # Test with data
- data_payload = {
- "task_id": "crawl_def456",
- "task_type": "crawl",
- "status": "completed",
- "timestamp": datetime.now(timezone.utc).isoformat(),
- "urls": ["https://example.com"],
- "data": {
- "results": [
- {"url": "https://example.com", "markdown": "# Example"}
- ]
- }
- }
-
- print(f"\n✅ Data payload constructed:")
- print(json.dumps(data_payload, indent=2))
-
- return True
- except Exception as e:
- print(f"❌ Payload construction failed: {e}")
- import traceback
- traceback.print_exc()
- return False
-
-def test_exponential_backoff():
- """Test exponential backoff calculation"""
- print("\n" + "=" * 60)
- print("TEST 5: Exponential Backoff Calculation")
- print("=" * 60)
-
- try:
- initial_delay = 1.0 # 1 second
- max_delay = 32.0 # 32 seconds
-
- print("Backoff delays for 5 attempts:")
- for attempt in range(5):
- delay = min(initial_delay * (2 ** attempt), max_delay)
- print(f" Attempt {attempt + 1}: {delay}s")
-
- # Verify the sequence: 1s, 2s, 4s, 8s, 16s
- expected = [1.0, 2.0, 4.0, 8.0, 16.0]
- actual = [min(initial_delay * (2 ** i), max_delay) for i in range(5)]
-
- assert actual == expected, f"Expected {expected}, got {actual}"
- print("✅ Exponential backoff sequence correct")
-
- return True
- except Exception as e:
- print(f"❌ Backoff calculation failed: {e}")
- return False
-
-def test_api_integration():
- """Test that api.py imports webhook module correctly"""
- print("\n" + "=" * 60)
- print("TEST 6: API Integration")
- print("=" * 60)
-
- try:
- # Check if api.py can import webhook module
- api_path = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
- with open(api_path, 'r') as f:
- api_content = f.read()
-
- if 'from webhook import WebhookDeliveryService' in api_content:
- print("✅ api.py imports WebhookDeliveryService")
- else:
- print("❌ api.py missing webhook import")
- return False
-
- if 'WebhookDeliveryService(config)' in api_content:
- print("✅ api.py initializes WebhookDeliveryService")
- else:
- print("❌ api.py doesn't initialize WebhookDeliveryService")
- return False
-
- if 'notify_job_completion' in api_content:
- print("✅ api.py calls notify_job_completion")
- else:
- print("❌ api.py doesn't call notify_job_completion")
- return False
-
- return True
- except Exception as e:
- print(f"❌ API integration check failed: {e}")
- return False
-
-def main():
- """Run all tests"""
- print("\n🧪 Webhook Implementation Validation Tests")
- print("=" * 60)
-
- results = []
-
- # Run tests
- results.append(("Module Imports", test_imports()))
- results.append(("Service Initialization", test_webhook_service_init()))
- results.append(("Config Model", test_webhook_config_model()))
- results.append(("Payload Construction", test_payload_construction()))
- results.append(("Exponential Backoff", test_exponential_backoff()))
- results.append(("API Integration", test_api_integration()))
-
- # Print summary
- print("\n" + "=" * 60)
- print("TEST SUMMARY")
- print("=" * 60)
-
- passed = sum(1 for _, result in results if result)
- total = len(results)
-
- for test_name, result in results:
- status = "✅ PASS" if result else "❌ FAIL"
- print(f"{status} - {test_name}")
-
- print(f"\n{'=' * 60}")
- print(f"Results: {passed}/{total} tests passed")
- print(f"{'=' * 60}")
-
- if passed == total:
- print("\n🎉 All tests passed! Webhook implementation is valid.")
- return 0
- else:
- print(f"\n⚠️ {total - passed} test(s) failed. Please review the output above.")
- return 1
-
-if __name__ == "__main__":
- exit(main())