Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: examples
Nicolas16439b1c7e19
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 705d2e0c7..ded241cdf 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -1,27 +1,27 @@-from firecrawl.firecrawl import ExtractConfig, FirecrawlApp+from firecrawl import JsonConfig, FirecrawlAppfrom pydantic import BaseModel, Fieldfrom typing import Listimport timeapp = FirecrawlApp(api_url="https://api.firecrawl.dev")-# # Scrape a website:+# Scrape a website:scrape_result = app.scrape_url('example.com', formats=["markdown", "html"])print(scrape_result.markdown)-# # Test batch scrapeq+# # # Test batch scrapeurls = ['https://example.com', 'https://docs.firecrawl.dev']-# Synchronous batch scrape+# # Synchronous batch scrapebatch_result = app.batch_scrape_urls(urls, formats=["markdown", "html"])print("Synchronous Batch Scrape Result:")print(batch_result.data[0].markdown)-# # Asynchronous batch scrape+# # # Asynchronous batch scrapeasync_batch_result = app.async_batch_scrape_urls(urls, formats=["markdown", "html"])print("\nAsynchronous Batch Scrape Result:")print(async_batch_result)-# Crawl a website:+# # Crawl a website:crawl_result = app.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])print(crawl_result.data[0].markdown)@@ -53,13 +53,13 @@ class ArticleSchema(BaseModel):class TopArticlesSchema(BaseModel):top: List[ArticleSchema] = Field(..., description="Top 5 stories")-extract_config = ExtractConfig(schema=TopArticlesSchema.model_json_schema())+extract_config = JsonConfig(schema=TopArticlesSchema.model_json_schema())llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)print(llm_extraction_result.extract)-# # Define schema to extract contents into using json schema+# Define schema to extract contents into using json schemajson_schema = {"type": "object","properties": {@@ -75,20 +75,18 @@ json_schema = {},"required": ["title", "points", "by", "commentsURL"]},- "minItems": 5,- "maxItems": 5,"description": "Top 5 stories on Hacker News"}},"required": ["top"]}-extract_config = ExtractConfig(extractionSchema=json_schema, mode="llm-extraction", pageOptions={"onlyMainContent": True})-llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)+extract_config = JsonConfig(schema=json_schema)+llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)-print(llm_extraction_result.extract)+print(llm_extraction_result.json)-# print(llm_extraction_result['llm_extraction'])+print(llm_extraction_result['llm_extraction'])# Map a website:@@ -108,6 +106,20 @@ extract_schema = ExtractSchema.schema()extract_result = app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)print(extract_result)++# Deep research example+research_result = app.deep_research(+ "What are the latest developments in large language models?",+ max_urls=4+)+print("Research Results:", research_result)++# Generate LLMs.txt example+llms_result = app.generate_llms_text(+ "https://firecrawl.dev")+print("LLMs.txt Results:", llms_result)++# Crawl a website with WebSockets:# inside an async function...import nest_asyncio@@ -134,4 +146,15 @@ async def start_crawl_and_watch():watcher.add_event_listener("done", on_done)# Start the watcher- await watcher.connect()\ No newline at end of file+ await watcher.connect()+++class ExtractSchema(BaseModel):+ company_mission: str+ supports_sso: bool+ is_open_source: bool+ is_in_yc: bool++extract_config = JsonConfig(schema=ExtractSchema.model_json_schema())+data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)+print(data.json)\ No newline at end of file