Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Handled JSON format structure

Aparup Gangulyd5d3df9d1056
diff --git a/examples/o3-mini_web_crawler/o3-mini_web_crawler.py b/examples/o3-mini_web_crawler/o3-mini_web_crawler.py
index 8e1fa4b90..59c1db67f 100644
--- a/examples/o3-mini_web_crawler/o3-mini_web_crawler.py
+++ b/examples/o3-mini_web_crawler/o3-mini_web_crawler.py
@@ -37,7 +37,7 @@ def find_relevant_page_via_map(objective, url, app, client):
print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}")
completion = client.chat.completions.create(
- model="o1-preview",
+ model="o3-mini",
messages=[
{
"role": "user",
@@ -56,9 +56,28 @@ def find_relevant_page_via_map(objective, url, app, client):
print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}")
map_website = app.map_url(url, params={"search": map_search_parameter})
+
+ # Debug print to see the response structure
+ print(f"{Colors.MAGENTA}Debug - Map response structure: {json.dumps(map_website, indent=2)}{Colors.RESET}")
+
print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}")
- print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}")
- return map_website
+
+ # Handle the response based on its structure
+ if isinstance(map_website, dict):
+ # Assuming the links are in a 'urls' or similar key
+ links = map_website.get('urls', []) or map_website.get('links', [])
+ elif isinstance(map_website, str):
+ try:
+ parsed = json.loads(map_website)
+ links = parsed.get('urls', []) or parsed.get('links', [])
+ except json.JSONDecodeError:
+ links = []
+ else:
+ links = map_website if isinstance(map_website, list) else []
+
+ print(f"{Colors.GREEN}Located {len(links)} relevant links.{Colors.RESET}")
+ return links
+
except Exception as e:
print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}")
return None
@@ -67,7 +86,11 @@ def find_relevant_page_via_map(objective, url, app, client):
def find_objective_in_top_pages(map_website, objective, app, client):
try:
# Get top 3 links from the map result
- top_links = map_website[:3] if isinstance(map_website, list) else []
+ if not map_website:
+ print(f"{Colors.RED}No links found to analyze.{Colors.RESET}")
+ return None
+
+ top_links = map_website[:3]
print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}")
for link in top_links:
@@ -93,18 +116,17 @@ def find_objective_in_top_pages(map_website, objective, app, client):
"""
completion = client.chat.completions.create(
- model="o3-mini",
- reasoning_effort="medium",
- messages=[
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": check_prompt
- }
- ]
- }
+ model="o3-mini",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": check_prompt
+ }
+ ]
+ }
]
)
@@ -129,10 +151,10 @@ def find_objective_in_top_pages(map_website, objective, app, client):
# Main function to execute the process
def main():
# Get user input
- url = input(f"{Colors.BLUE}Enter the website to crawl using o3-mini : {Colors.RESET}")
+ url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}")
objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")
- print(f"{Colors.YELLOW}Initiating web crawling process with o3 mini...{Colors.RESET}")
+ print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}")
# Find the relevant page
map_website = find_relevant_page_via_map(objective, url, app, client)