Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update o3-mini_web_crawler.py
Eric Ciarlab0f9e73376cb
diff --git a/examples/o3-mini_web_crawler/o3-mini_web_crawler.py b/examples/o3-mini_web_crawler/o3-mini_web_crawler.pyindex 3c5964f3a..0a1ee1946 100644--- a/examples/o3-mini_web_crawler/o3-mini_web_crawler.py+++ b/examples/o3-mini_web_crawler/o3-mini_web_crawler.py@@ -74,6 +74,77 @@ def find_relevant_page_via_map(objective, url, app, client):links = []else:links = map_website if isinstance(map_website, list) else []++ if not links:+ print(f"{Colors.RED}No links found in map response.{Colors.RESET}")+ return None++ rank_prompt = f"""+ Given this list of URLs and the objective: {objective}+ Analyze each URL and rank the top 3 most relevant ones that are most likely to contain the information we need.+ Return your response as a JSON array with exactly 3 objects, each containing:+ - "url": the full URL+ - "relevance_score": number between 0-100 indicating relevance to objective+ - "reason": brief explanation of why this URL is relevant++ Example output:+ [+ {{+ "url": "https://example.com/about",+ "relevance_score": 95,+ "reason": "Main about page containing company information"+ }},+ {{+ "url": "https://example.com/team",+ "relevance_score": 80,+ "reason": "Team page with leadership details"+ }},+ {{+ "url": "https://example.com/contact",+ "relevance_score": 70,+ "reason": "Contact page with location information"+ }}+ ]++ URLs to analyze:+ {json.dumps(links, indent=2)}+ """++ print(f"{Colors.YELLOW}Ranking URLs by relevance to objective...{Colors.RESET}")+ completion = client.chat.completions.create(+ model="o3-mini",+ messages=[+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": rank_prompt+ }+ ]+ }+ ]+ )++ try:+ ranked_results = json.loads(completion.choices[0].message.content)+ links = [result["url"] for result in ranked_results]++ # Print detailed ranking info+ print(f"{Colors.CYAN}Top 3 ranked URLs:{Colors.RESET}")+ for result in ranked_results:+ print(f"{Colors.GREEN}URL: {result['url']}{Colors.RESET}")+ print(f"{Colors.YELLOW}Relevance Score: {result['relevance_score']}{Colors.RESET}")+ print(f"{Colors.BLUE}Reason: {result['reason']}{Colors.RESET}")+ print("---")++ if not links:+ print(f"{Colors.RED}No relevant links identified.{Colors.RESET}")+ return None++ except (json.JSONDecodeError, KeyError) as e:+ print(f"{Colors.RED}Error parsing ranked results: {str(e)}{Colors.RESET}")+ return Noneprint(f"{Colors.GREEN}Located {len(links)} relevant links.{Colors.RESET}")return links