Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

1.1 KB
Raw Blame
import asyncio
from crawl4ai.docker_client import Crawl4aiDockerClient
from crawl4ai import (
    BrowserConfig,
    CrawlerRunConfig
)

async def main():
    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
        # If jwt is enabled, authenticate first
        # await client.authenticate("[email protected]")
        
        # Non-streaming crawl
        results = await client.crawl(
            ["https://example.com", "https://python.org"],
            browser_config=BrowserConfig(headless=True),
            crawler_config=CrawlerRunConfig()
        )
        print(f"Non-streaming results: {results}")
        
        # Streaming crawl
        crawler_config = CrawlerRunConfig(stream=True)
        async for result in await client.crawl(
            ["https://example.com", "https://python.org"],
            browser_config=BrowserConfig(headless=True),
            crawler_config=crawler_config
        ):
            print(f"Streamed result: {result}")
        
        # Get schema
        schema = await client.get_schema()
        print(f"Schema: {schema}")

if __name__ == "__main__":
    asyncio.run(main())