Skip to content

armgit / fara

Public Imported from GitHub
0

Fara1.5 – A family of frontier computer use agent models

107 KB
Raw Blame
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "152e9a51",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from collections import defaultdict\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "from webeval.post_eval_analysis import aggregate_post_eval_errors, extract_last_error, extract_action_timing_stats\n",
    "pd.set_option('display.max_rows', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f482b22e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from artifacts import EvalLogs\n",
    "### path to eval logs\n",
    "el = EvalLogs(az_folder = \"/path/to/Fara/eval/runs/WebSurfer-fara-100-max_n_images-3/fara-7b/<username>/WebVoyager_WebVoyager_data_08312025.jsonl/<run-id>\")\n",
    "print(el)\n",
    "el.mount()\n",
    "\n",
    "# output_folder.map_folders\n",
    "folders = el.map_folders(lambda traj: {'name': traj, 'files': list(traj.iterdir())}, 'traj')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "19de826b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test 1 - Error type: 'Error parsing thoughts and action', Description: 'Error parsing thoughts and action'\n",
      "Test 2 - Error type: 'Web surfing error: playwright._impl._errors.TargetClosedError: Page.bring_to_front: Target page, context or browser has been closed', Description: 'Web surfing error: playwright._impl._errors.TargetClosedError: Page.bring_to_front: Target page, context or browser has been closed'\n",
      "Test 3 - Error type: 'Error running task: some detailed error message', Description: 'Error running task: some detailed error message'\n",
      "Test 4 - Sample log analysis:\n",
      "\tNumber of WebSurfer actions: 2\n",
      "\tAverage time between actions: 43.66 seconds\n",
      "\tSession duration: 969.67 seconds (16.16 minutes)\n",
      "\tLong session (>30 min): No\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# Test the function with the examples\n",
    "test_log1 = '''2025-08-06 18:10:11,637 [INFO] webeval.core.ulta_user_query_840 - [Execution ulta_user_query_840] Start\n",
    "2025-08-06 18:10:11,685 [INFO] webeval.core.ulta_user_query_840 - [Execution ulta_user_query_840] Trajectory is not complete. Reexecuting...\n",
    "2025-08-06 18:10:48,619 [INFO] webeval.core.ulta_user_query_840 - WebSurferEvent(source='WebSurfer', message='computer_use( {\"action\": \"web_search\", \"query\": \"Nude Mood eyeshadow palette site:ulta.com\", \"thoughts\": \"To purchase the Nude Mood eyeshadow palette from Ulta, I need to locate the product on their website. Performing a web search will help me find the specific product page quickly.\"} )', url='https://www.bing.com/', action='computer_use', arguments={'action': 'web_search', 'query': 'Nude Mood eyeshadow palette site:ulta.com', 'thoughts': 'To purchase the Nude Mood eyeshadow palette from Ulta, I need to locate the product on their website. Performing a web search will help me find the specific product page quickly.'})\n",
    "2025-08-06 18:11:11,355 [ERROR] webeval.core.ulta_user_query_840 - Invalid action text: {\"name\": \"computer_use\", \"arguments\": {\"action\": \"left_click\", \"coordinate\": 136, 99]}}\n",
    "2025-08-06 18:11:11,355 [ERROR] webeval.core.ulta_user_query_840 - Error parsing thoughts and action: Clicking on the first relevant link in the search results will likely lead directly to the product page for the Nude Mood eyeshadow palette on Ulta's website, allowing us to proceed with the purchase process.'''\n",
    "\n",
    "error_type1, error_desc1 = extract_last_error(test_log1)\n",
    "print(f\"Test 1 - Error type: '{error_type1}', Description: '{error_desc1}'\")\n",
    "\n",
    "test_log2 = '''2025-08-06 20:22:51,954 [INFO] webeval.core.petsmart_user_query_2557 - [Execution petsmart_user_query_2557] Start\n",
    "2025-08-06 20:23:34,363 [ERROR] webeval.core.petsmart_user_query_2557 - Web surfing error\n",
    "Traceback (most recent call last):\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 150, in _recover_page\n",
    "    await page.goto(current_url, timeout=timeout_secs * 1000)\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/async_api/_generated.py\", line 8992, in goto\n",
    "    await self._impl_obj.goto(\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_page.py\", line 556, in goto\n",
    "    return await self._main_frame.goto(**locals_to_params(locals()))\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_frame.py\", line 153, in goto\n",
    "    await self._channel.send(\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_connection.py\", line 69, in send\n",
    "    return await self._connection.wrap_api_call(\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_connection.py\", line 558, in wrap_api_call\n",
    "    raise rewrite_error(error, f\"{parsed_st['apiName']}: {error}\") from None\n",
    "playwright._impl._errors.TargetClosedError: Page.goto: Target page, context or browser has been closed\n",
    "\n",
    "During handling of the above exception, another exception occurred:\n",
    "\n",
    "Traceback (most recent call last):\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 99, in wrapper\n",
    "    await _recover_page(page, timeout_secs, logger)\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 156, in _recover_page\n",
    "    raise Exception(f\"playwright_controller._recover_page(): All recovery methods failed. Reload error: {e}, Goto error: {goto_error}\")\n",
    "Exception: playwright_controller._recover_page(): All recovery methods failed. Reload error: Page.reload: Target page, context or browser has been closed, Goto error: Page.goto: Target page, context or browser has been closed\n",
    "\n",
    "The above exception was the direct cause of the following exception:\n",
    "\n",
    "Traceback (most recent call last):\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/orca_web_surfer/orca_qwen25vl_history_web_surfer.py\", line 35, in _generate_reply\n",
    "    request_halt, content = await self.__generate_reply(cancellation_token)\n",
    "                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/orca_web_surfer/orca_qwen25vl_history_web_surfer.py\", line 195, in __generate_reply\n",
    "    result = await self._execute_tool(\n",
    "             ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/orca_web_surfer/orca_qwen25vl_web_surfer.py\", line 190, in _execute_tool\n",
    "    url=await self.playwright_controller.get_page_url(self.page),\n",
    "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 105, in wrapper\n",
    "    raise e from recovery_error\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 74, in wrapper\n",
    "    return await func(*args, **kwargs)\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 1000, in get_page_url\n",
    "    await self._ensure_page_ready(page)\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 105, in wrapper\n",
    "    raise e from recovery_error\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 74, in wrapper\n",
    "    return await func(*args, **kwargs)\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 354, in _ensure_page_ready\n",
    "    await self.on_new_page(page)\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 105, in wrapper\n",
    "    raise e from recovery_error\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 74, in wrapper\n",
    "    return await func(*args, **kwargs)\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/mnt/users/wangzhe/code/agento/agento/src/agento/agents/web_surfer/playwright_controller.py\", line 338, in on_new_page\n",
    "    await page.bring_to_front()\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/async_api/_generated.py\", line 9383, in bring_to_front\n",
    "    return mapping.from_maybe_impl(await self._impl_obj.bring_to_front())\n",
    "                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_page.py\", line 664, in bring_to_front\n",
    "    await self._channel.send(\"bringToFront\", None)\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_connection.py\", line 69, in send\n",
    "    return await self._connection.wrap_api_call(\n",
    "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
    "  File \"/home/corbyrosset/anaconda3/envs/spencers_webeval/lib/python3.12/site-packages/playwright/_impl/_connection.py\", line 558, in wrap_api_call\n",
    "    raise rewrite_error(error, f\"{parsed_st['apiName']}: {error}\") from None\n",
    "playwright._impl._errors.TargetClosedError: Page.bring_to_front: Target page, context or browser has been closed\n",
    "2025-08-06 20:23:35,319 [INFO] webeval.core.petsmart_user_query_2557 - [Execution petsmart_user_query_2557] Completed'''\n",
    "\n",
    "error_type2, error_desc2 = extract_last_error(test_log2)\n",
    "print(f\"Test 2 - Error type: '{error_type2}', Description: '{error_desc2}'\")\n",
    "\n",
    "# Test with an \"Error running task\" example\n",
    "test_log3 = '''2025-08-06 20:23:35,319 [ERROR] webeval.core.nordstrom_user_query_9755 - [Execution nordstrom_user_query_9755] Error running task: some detailed error message'''\n",
    "\n",
    "error_type3, error_desc3 = extract_last_error(test_log3)\n",
    "print(f\"Test 3 - Error type: '{error_type3}', Description: '{error_desc3}'\")\n",
    "\n",
    "# Test case for extracting average time between actions\n",
    "# Test the timing extraction with the sample core.log provided\n",
    "sample_log = '''2025-08-13 08:30:14,504 [INFO] webeval.core.costco_user_query_4323 - [Execution costco_user_query_4323] Start\n",
    "2025-08-13 08:30:14,553 [INFO] webeval.core.costco_user_query_4323 - [Execution costco_user_query_4323] Trajectory is not complete. Reexecuting...\n",
    "2025-08-13 08:30:15,995 [INFO] webeval.core.costco_user_query_4323 - Initializing BrowserBase session...\n",
    "2025-08-13 08:30:19,919 [INFO] webeval.core.costco_user_query_4323 - Connected to BrowserBase session: https://browserbase.com/sessions/21ff0446-84cc-4083-85f8-72654b51c7ce\n",
    "2025-08-13 08:30:40,031 [INFO] webeval.core.costco_user_query_4323 - WebSurferEvent(source='WebSurfer-SummarizedAction', message='Action 1: URL=https://www.bing.com/, input_text({\\'state_description\\': \\'Bing.com homepage showing search box focused\\', \\'reasoning\\': \"Search for the Zayne Leather Power Loveseat specifically on Costco\\'s site\", \\'input_field_id\\': 10, \\'text_value\\': \\'Zayne Leather Power Loveseat site:costco.com\\', \\'press_enter\\': True, \\'delete_existing_text\\': True})', url='https://www.bing.com/', action=None, arguments=None)\n",
    "2025-08-13 08:30:40,722 [INFO] webeval.core.costco_user_query_4323 - WebSurferEvent(source='WebSurfer', message='input_text( {\"state_description\": \"Bing.com homepage showing search box focused\", \"reasoning\": \"Search for the Zayne Leather Power Loveseat specifically on Costco\\'s site\", \"input_field_id\": 10, \"text_value\": \"Zayne Leather Power Loveseat site:costco.com\", \"press_enter\": true, \"delete_existing_text\": true} )', url='https://www.bing.com/', action='input_text', arguments={'state_description': 'Bing.com homepage showing search box focused', 'reasoning': \"Search for the Zayne Leather Power Loveseat specifically on Costco's site\", 'input_field_id': 10, 'text_value': 'Zayne Leather Power Loveseat site:costco.com', 'press_enter': True, 'delete_existing_text': True})\n",
    "2025-08-13 08:31:23,685 [INFO] webeval.core.costco_user_query_4323 - WebSurferEvent(source='WebSurfer-SummarizedAction', message='Action 2: URL=https://www.bing.com/search?q=Zayne+Leather+Power+Loveseat+site%3Acostco.com&form=QBLH&sp=-1&lq=0&pq=zayne+leather+power+loveseat+site%3Acostco.co&sc=10-43&qs=n&sk=&cvid=144A236AC9334CCA815928977A36C847, click({\\'state_description\\': \"Bing search results page showing a Costco result for \\'Zayne 2-piece Leather Power Reclining Set with …\\'\", \\'reasoning\\': \\'To navigate to the Costco product page that includes the Zayne Leather Power Loveseat for purchase\\', \\'target_id\\': 34})', url='https://www.bing.com/search?q=Zayne+Leather+Power+Loveseat+site%3Acostco.com&form=QBLH&sp=-1&lq=0&pq=zayne+leather+power+loveseat+site%3Acostco.co&sc=10-43&qs=n&sk=&cvid=144A236AC9334CCA815928977A36C847', action=None, arguments=None)\n",
    "2025-08-13 08:31:24,382 [INFO] webeval.core.costco_user_query_4323 - WebSurferEvent(source='WebSurfer', message='click( {\"state_description\": \"Bing search results page showing a Costco result for \\'Zayne 2-piece Leather Power Reclining Set with \\\\u2026\\'\", \"reasoning\": \"To navigate to the Costco product page that includes the Zayne Leather Power Loveseat for purchase\", \"target_id\": 34} )', url='https://www.bing.com/search?q=Zayne+Leather+Power+Loveseat+site%3Acostco.com&form=QBLH&sp=-1&lq=0&pq=zayne+leather+power+loveseat+site%3Acostco.co&sc=10-43&qs=n&sk=&cvid=144A236AC9334CCA815928977A36C847', action='click', arguments={'state_description': \"Bing search results page showing a Costco result for 'Zayne 2-piece Leather Power Reclining Set with …'\", 'reasoning': 'To navigate to the Costco product page that includes the Zayne Leather Power Loveseat for purchase', 'target_id': 34})\n",
    "2025-08-13 08:46:24,171 [INFO] webeval.core.costco_user_query_4323 - [Execution costco_user_query_4323] Completed'''\n",
    "\n",
    "avg_delta, session_duration, num_actions = extract_action_timing_stats(sample_log)\n",
    "print(f\"Test 4 - Sample log analysis:\")\n",
    "print(f\"\\tNumber of WebSurfer actions: {num_actions}\")\n",
    "print(f\"\\tAverage time between actions: {avg_delta:.2f} seconds\" if avg_delta else \"Average time between actions: N/A (need at least 2 actions)\")\n",
    "print(f\"\\tSession duration: {session_duration:.2f} seconds ({session_duration/60:.2f} minutes)\" if session_duration else \"Session duration: N/A\")\n",
    "print(f\"\\tLong session (>30 min): {'Yes' if session_duration and session_duration > 1800 else 'No'}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5nbu8qthi49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 449 trajectories to extract errors...\n",
      "Re-processing 449 trajectories with updated error extraction...\n",
      "Processed 0/449 trajectories...\n",
      "Processed 100/449 trajectories...\n",
      "Processed 200/449 trajectories...\n",
      "Processed 300/449 trajectories...\n",
      "Processed 400/449 trajectories...\n",
      "Completed processing 449 trajectories!\n",
      "Trajectories with no errors: 219\n",
      "Trajectories with errors: 230\n",
      "Average score: 0.701 (from 351 trajectories with scores)\n",
      "\n",
      "No detailed success metrics found (no 0.8-5-3.json files processed)\n",
      "Final answer statistics:\n",
      "  final_answer_file_not_found: 98\n",
      "  From 351 trajectories with final_answer files:\n",
      "    is_aborted == True: 0\n",
      "    is_aborted == False: 351\n",
      "    empty_answer_below_action_budget_but_not_aborted: 0\n",
      "\n",
      "Token usage statistics:\n",
      "  No token usage data found in final_answer files\n",
      "A) Average time between actions: N/A (no sessions with multiple actions found)\n",
      "B) Average session duration: -65697.96 ± 37333.07 seconds (-1094.97 ± 622.22 minutes) (across 351 sessions)\n",
      "C) Average number of actions per trajectory (all): 0.00 ± 0.00 actions (across 449 trajectories)\n",
      "D) Average number of actions per trajectory (non-aborted): 0.00 ± 0.00 actions (across 351 non-aborted trajectories)\n",
      "\n",
      "========================================================================================================================\n",
      "ERROR ANALYSIS SUMMARY (UPDATED)\n",
      "========================================================================================================================\n",
      "\n",
      "Total trajectories analyzed: 449\n",
      "Trajectories with errors: 230 (51.2%)\n",
      "Trajectories without errors: 219 (48.8%)\n",
      "\n",
      "Error Breakdown:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Error Type</th>\n",
       "      <th>Count</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Sample Trajectories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Page recovery failed</td>\n",
       "      <td>26</td>\n",
       "      <td>5.8%</td>\n",
       "      <td>ArXiv--21, GitHub--9, ESPN--5, ... (+23 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Error running task: Page.goto: Timeout 60000ms exceeded.</td>\n",
       "      <td>17</td>\n",
       "      <td>3.8%</td>\n",
       "      <td>Google Map--20, Cambridge Dictionary--25, Cambridge Dictionary--27, ... (+14 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Error running task: Page.set_viewport_size: Target page, context or browser has been closed</td>\n",
       "      <td>11</td>\n",
       "      <td>2.4%</td>\n",
       "      <td>ESPN--22, Amazon--13, GitHub--0, ... (+8 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Error running task: Timeout 30000ms exceeded.</td>\n",
       "      <td>8</td>\n",
       "      <td>1.8%</td>\n",
       "      <td>Booking--17, Cambridge Dictionary--38, Cambridge Dictionary--18, ... (+5 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WARNING</td>\n",
       "      <td>7</td>\n",
       "      <td>1.6%</td>\n",
       "      <td>Cambridge Dictionary--40, Cambridge Dictionary--34, Cambridge Dictionary--39, ... (+4 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>playwright_controller._recover_page()</td>\n",
       "      <td>5</td>\n",
       "      <td>1.1%</td>\n",
       "      <td>BBC News--1, ArXiv--38, BBC News--20, ... (+2 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Error running task: Page.screenshot: Timeout 15000ms exceeded.</td>\n",
       "      <td>5</td>\n",
       "      <td>1.1%</td>\n",
       "      <td>Cambridge Dictionary--35, Apple--39, Booking--43, ... (+2 more)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Error running task: Page.evaluate: Execution context was destroyed, most likely because of a nav...</td>\n",
       "      <td>2</td>\n",
       "      <td>0.4%</td>\n",
       "      <td>Booking--34, Booking--25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[Execution GitHub--26] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[Execution ESPN--24] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>[Execution Coursera--2] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>[Execution Google Map--9] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>[Execution Coursera--15] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Error running task: [Errno 23] Too many open files in system: '/data/data/Fara/eval/runs/WebSurf...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>[Execution GitHub--36] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>[Execution Google Map--15] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[Execution Coursera--14] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>[Execution Coursera--8] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>[Evaluation Cambridge Dictionary--6] Error evaluating example</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Cambridge Dictionary--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>[Execution Google Map--8] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>[Execution ESPN--23] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>[Execution GitHub--40] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>[Execution Google Flights--17] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>[Execution GitHub--2] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>[Execution Google Flights--15] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>[Execution Coursera--21] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>[Execution ESPN--15] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>[Execution Coursera--41] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>[Execution Google Flights--13] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>[Execution Google Map--6] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>[Execution Google Flights--14] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>[Execution ESPN--39] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>[Execution GitHub--6] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>[Execution GitHub--17] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>[Execution Coursera--33] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>[Execution Google Flights--26] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>[Execution ESPN--31] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>[Execution Coursera--20] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>[Execution GitHub--11] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>[Execution Google Flights--0] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>[Execution Google Map--12] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>[Execution GitHub--28] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>[Execution Google Flights--40] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>[Execution ESPN--41] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>[Execution GitHub--18] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>[Execution ESPN--0] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>[Execution Coursera--1] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>[Execution Coursera--34] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>[Execution Google Flights--11] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>[Execution Google Flights--36] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>[Execution GitHub--35] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>[Execution ESPN--20] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>[Execution ESPN--7] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>[Execution Google Flights--16] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>[Execution ESPN--18] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>[Execution GitHub--22] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>[Execution Coursera--4] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>[Execution Coursera--5] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>[Execution Google Flights--10] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>[Execution Google Flights--21] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>[Execution Google Map--10] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>[Execution Google Map--13] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>[Execution Google Flights--6] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>[Execution Google Flights--30] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>[Execution Coursera--12] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Error running task: invalid syntax (&lt;unknown&gt;, line 2)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Booking--36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>[Execution Google Map--3] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>[Execution Google Map--5] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>[Execution Coursera--22] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>[Evaluation Booking--42] Error evaluating example</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Booking--42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>[Execution Google Flights--19] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>[Execution Google Flights--37] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>[Execution Google Flights--31] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>[Execution Coursera--39] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>[Execution GitHub--15] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>[Execution Google Flights--12] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>[Execution Coursera--18] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>[Execution Google Flights--22] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>[Execution ESPN--27] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>[Execution GitHub--27] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>[Execution GitHub--31] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>[Execution Google Flights--33] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>[Execution Cambridge Dictionary--41] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Cambridge Dictionary--41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>[Execution GitHub--16] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>[Execution Coursera--3] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>[Execution Google Map--14] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>[Execution Coursera--16] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>[Execution GitHub--20] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>[Execution Coursera--13] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>[Execution Google Map--7] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>[Execution GitHub--30] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>[Execution Coursera--10] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>[Execution Coursera--24] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>[Execution Google Map--1] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>[Evaluation Booking--37] Error evaluating example</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Booking--37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>[Execution Coursera--6] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>Error running task: Page.goto: Download is starting</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ArXiv--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>[Execution ESPN--28] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>[Execution Coursera--9] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>[Execution Coursera--32] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>[Execution ESPN--12] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>[Execution ESPN--35] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>[Execution Google Map--4] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>[Execution ESPN--10] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>[Execution Coursera--35] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>[Execution GitHub--24] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>Error running task: Unknown tool:  openFileDialog</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ArXiv--18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>[Execution Google Flights--9] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>[Execution Coursera--27] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>[Execution GitHub--10] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>[Execution GitHub--21] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>[Execution Google Flights--25] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>[Execution GitHub--39] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>[Execution Google Flights--24] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>[Execution GitHub--7] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>[Execution Coursera--25] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>[Execution GitHub--3] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>[Execution ESPN--3] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>[Execution GitHub--38] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>[Execution Google Flights--34] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>[Execution Google Flights--18] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>[Execution Google Flights--20] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>[Execution Google Flights--27] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>[Execution Google Flights--39] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>[Execution Google Flights--35] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>[Execution Coursera--30] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>[Execution ESPN--8] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>[Execution Coursera--31] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>[Execution Google Flights--8] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>[Execution ESPN--34] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>[Execution Google Flights--28] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>[Execution GitHub--32] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>[Execution Google Map--0] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>[Execution GitHub--19] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>[Execution GitHub--13] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>[Execution GitHub--25] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>[Execution Coursera--37] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>[Execution Google Map--17] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>[Execution Google Map--16] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>[Execution Google Flights--41] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>[Execution GitHub--29] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>[Execution Google Flights--32] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>[Execution GitHub--37] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>[Execution Google Map--11] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>[Execution Coursera--40] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>[Execution Coursera--7] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>[Execution Google Map--2] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Map--2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>[Execution Coursera--23] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Coursera--23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>[Execution ESPN--36] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>[Execution GitHub--23] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>[Execution GitHub--34] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>[Execution Google Flights--29] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>[Execution Google Flights--38] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>[Execution Google Flights--23] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>Google Flights--23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>[Execution ESPN--6] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>ESPN--6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>[Execution GitHub--14] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>[Execution GitHub--33] Max retries (5) reached. Task failed with stage=0.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2%</td>\n",
       "      <td>GitHub--33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>Average Score for 351/449</td>\n",
       "      <td>0.701</td>\n",
       "      <td>78.2%</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>final_answer_file_not_found</td>\n",
       "      <td>98</td>\n",
       "      <td>21.8%</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>is_aborted == True</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0%</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>is_aborted == False</td>\n",
       "      <td>351</td>\n",
       "      <td>78.2%</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>empty_answer_below_action_budget_but_not_aborted</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0%</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>average_num_actions_all: 0.00 ± 0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>449/449</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>average_num_actions_non_aborted: 0.00 ± 0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>351/449</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                              Error Type  \\\n",
       "0                                                                                   Page recovery failed   \n",
       "1                                               Error running task: Page.goto: Timeout 60000ms exceeded.   \n",
       "2            Error running task: Page.set_viewport_size: Target page, context or browser has been closed   \n",
       "3                                                          Error running task: Timeout 30000ms exceeded.   \n",
       "4                                                                                                WARNING   \n",
       "5                                                                  playwright_controller._recover_page()   \n",
       "6                                         Error running task: Page.screenshot: Timeout 15000ms exceeded.   \n",
       "7    Error running task: Page.evaluate: Execution context was destroyed, most likely because of a nav...   \n",
       "8                              [Execution GitHub--26] Max retries (5) reached. Task failed with stage=0.   \n",
       "9                                [Execution ESPN--24] Max retries (5) reached. Task failed with stage=0.   \n",
       "10                            [Execution Coursera--2] Max retries (5) reached. Task failed with stage=0.   \n",
       "11                          [Execution Google Map--9] Max retries (5) reached. Task failed with stage=0.   \n",
       "12                           [Execution Coursera--15] Max retries (5) reached. Task failed with stage=0.   \n",
       "13   Error running task: [Errno 23] Too many open files in system: '/data/data/Fara/eval/runs/WebSurf...   \n",
       "14                             [Execution GitHub--36] Max retries (5) reached. Task failed with stage=0.   \n",
       "15                         [Execution Google Map--15] Max retries (5) reached. Task failed with stage=0.   \n",
       "16                           [Execution Coursera--14] Max retries (5) reached. Task failed with stage=0.   \n",
       "17                            [Execution Coursera--8] Max retries (5) reached. Task failed with stage=0.   \n",
       "18                                         [Evaluation Cambridge Dictionary--6] Error evaluating example   \n",
       "19                          [Execution Google Map--8] Max retries (5) reached. Task failed with stage=0.   \n",
       "20                               [Execution ESPN--23] Max retries (5) reached. Task failed with stage=0.   \n",
       "21                             [Execution GitHub--40] Max retries (5) reached. Task failed with stage=0.   \n",
       "22                     [Execution Google Flights--17] Max retries (5) reached. Task failed with stage=0.   \n",
       "23                              [Execution GitHub--2] Max retries (5) reached. Task failed with stage=0.   \n",
       "24                     [Execution Google Flights--15] Max retries (5) reached. Task failed with stage=0.   \n",
       "25                           [Execution Coursera--21] Max retries (5) reached. Task failed with stage=0.   \n",
       "26                               [Execution ESPN--15] Max retries (5) reached. Task failed with stage=0.   \n",
       "27                           [Execution Coursera--41] Max retries (5) reached. Task failed with stage=0.   \n",
       "28                     [Execution Google Flights--13] Max retries (5) reached. Task failed with stage=0.   \n",
       "29                          [Execution Google Map--6] Max retries (5) reached. Task failed with stage=0.   \n",
       "30                     [Execution Google Flights--14] Max retries (5) reached. Task failed with stage=0.   \n",
       "31                               [Execution ESPN--39] Max retries (5) reached. Task failed with stage=0.   \n",
       "32                              [Execution GitHub--6] Max retries (5) reached. Task failed with stage=0.   \n",
       "33                             [Execution GitHub--17] Max retries (5) reached. Task failed with stage=0.   \n",
       "34                           [Execution Coursera--33] Max retries (5) reached. Task failed with stage=0.   \n",
       "35                     [Execution Google Flights--26] Max retries (5) reached. Task failed with stage=0.   \n",
       "36                               [Execution ESPN--31] Max retries (5) reached. Task failed with stage=0.   \n",
       "37                           [Execution Coursera--20] Max retries (5) reached. Task failed with stage=0.   \n",
       "38                             [Execution GitHub--11] Max retries (5) reached. Task failed with stage=0.   \n",
       "39                      [Execution Google Flights--0] Max retries (5) reached. Task failed with stage=0.   \n",
       "40                         [Execution Google Map--12] Max retries (5) reached. Task failed with stage=0.   \n",
       "41                             [Execution GitHub--28] Max retries (5) reached. Task failed with stage=0.   \n",
       "42                     [Execution Google Flights--40] Max retries (5) reached. Task failed with stage=0.   \n",
       "43                               [Execution ESPN--41] Max retries (5) reached. Task failed with stage=0.   \n",
       "44                             [Execution GitHub--18] Max retries (5) reached. Task failed with stage=0.   \n",
       "45                                [Execution ESPN--0] Max retries (5) reached. Task failed with stage=0.   \n",
       "46                            [Execution Coursera--1] Max retries (5) reached. Task failed with stage=0.   \n",
       "47                           [Execution Coursera--34] Max retries (5) reached. Task failed with stage=0.   \n",
       "48                     [Execution Google Flights--11] Max retries (5) reached. Task failed with stage=0.   \n",
       "49                     [Execution Google Flights--36] Max retries (5) reached. Task failed with stage=0.   \n",
       "50                             [Execution GitHub--35] Max retries (5) reached. Task failed with stage=0.   \n",
       "51                               [Execution ESPN--20] Max retries (5) reached. Task failed with stage=0.   \n",
       "52                                [Execution ESPN--7] Max retries (5) reached. Task failed with stage=0.   \n",
       "53                     [Execution Google Flights--16] Max retries (5) reached. Task failed with stage=0.   \n",
       "54                               [Execution ESPN--18] Max retries (5) reached. Task failed with stage=0.   \n",
       "55                             [Execution GitHub--22] Max retries (5) reached. Task failed with stage=0.   \n",
       "56                            [Execution Coursera--4] Max retries (5) reached. Task failed with stage=0.   \n",
       "57                            [Execution Coursera--5] Max retries (5) reached. Task failed with stage=0.   \n",
       "58                     [Execution Google Flights--10] Max retries (5) reached. Task failed with stage=0.   \n",
       "59                     [Execution Google Flights--21] Max retries (5) reached. Task failed with stage=0.   \n",
       "60                         [Execution Google Map--10] Max retries (5) reached. Task failed with stage=0.   \n",
       "61                         [Execution Google Map--13] Max retries (5) reached. Task failed with stage=0.   \n",
       "62                      [Execution Google Flights--6] Max retries (5) reached. Task failed with stage=0.   \n",
       "63                     [Execution Google Flights--30] Max retries (5) reached. Task failed with stage=0.   \n",
       "64                           [Execution Coursera--12] Max retries (5) reached. Task failed with stage=0.   \n",
       "65                                                Error running task: invalid syntax (<unknown>, line 2)   \n",
       "66                          [Execution Google Map--3] Max retries (5) reached. Task failed with stage=0.   \n",
       "67                          [Execution Google Map--5] Max retries (5) reached. Task failed with stage=0.   \n",
       "68                           [Execution Coursera--22] Max retries (5) reached. Task failed with stage=0.   \n",
       "69                                                     [Evaluation Booking--42] Error evaluating example   \n",
       "70                     [Execution Google Flights--19] Max retries (5) reached. Task failed with stage=0.   \n",
       "71                     [Execution Google Flights--37] Max retries (5) reached. Task failed with stage=0.   \n",
       "72                     [Execution Google Flights--31] Max retries (5) reached. Task failed with stage=0.   \n",
       "73                           [Execution Coursera--39] Max retries (5) reached. Task failed with stage=0.   \n",
       "74                             [Execution GitHub--15] Max retries (5) reached. Task failed with stage=0.   \n",
       "75                     [Execution Google Flights--12] Max retries (5) reached. Task failed with stage=0.   \n",
       "76                           [Execution Coursera--18] Max retries (5) reached. Task failed with stage=0.   \n",
       "77                     [Execution Google Flights--22] Max retries (5) reached. Task failed with stage=0.   \n",
       "78                               [Execution ESPN--27] Max retries (5) reached. Task failed with stage=0.   \n",
       "79                             [Execution GitHub--27] Max retries (5) reached. Task failed with stage=0.   \n",
       "80                             [Execution GitHub--31] Max retries (5) reached. Task failed with stage=0.   \n",
       "81                     [Execution Google Flights--33] Max retries (5) reached. Task failed with stage=0.   \n",
       "82               [Execution Cambridge Dictionary--41] Max retries (5) reached. Task failed with stage=0.   \n",
       "83                             [Execution GitHub--16] Max retries (5) reached. Task failed with stage=0.   \n",
       "84                            [Execution Coursera--3] Max retries (5) reached. Task failed with stage=0.   \n",
       "85                         [Execution Google Map--14] Max retries (5) reached. Task failed with stage=0.   \n",
       "86                           [Execution Coursera--16] Max retries (5) reached. Task failed with stage=0.   \n",
       "87                             [Execution GitHub--20] Max retries (5) reached. Task failed with stage=0.   \n",
       "88                           [Execution Coursera--13] Max retries (5) reached. Task failed with stage=0.   \n",
       "89                          [Execution Google Map--7] Max retries (5) reached. Task failed with stage=0.   \n",
       "90                             [Execution GitHub--30] Max retries (5) reached. Task failed with stage=0.   \n",
       "91                           [Execution Coursera--10] Max retries (5) reached. Task failed with stage=0.   \n",
       "92                           [Execution Coursera--24] Max retries (5) reached. Task failed with stage=0.   \n",
       "93                          [Execution Google Map--1] Max retries (5) reached. Task failed with stage=0.   \n",
       "94                                                     [Evaluation Booking--37] Error evaluating example   \n",
       "95                            [Execution Coursera--6] Max retries (5) reached. Task failed with stage=0.   \n",
       "96                                                   Error running task: Page.goto: Download is starting   \n",
       "97                               [Execution ESPN--28] Max retries (5) reached. Task failed with stage=0.   \n",
       "98                            [Execution Coursera--9] Max retries (5) reached. Task failed with stage=0.   \n",
       "99                           [Execution Coursera--32] Max retries (5) reached. Task failed with stage=0.   \n",
       "100                              [Execution ESPN--12] Max retries (5) reached. Task failed with stage=0.   \n",
       "101                              [Execution ESPN--35] Max retries (5) reached. Task failed with stage=0.   \n",
       "102                         [Execution Google Map--4] Max retries (5) reached. Task failed with stage=0.   \n",
       "103                              [Execution ESPN--10] Max retries (5) reached. Task failed with stage=0.   \n",
       "104                          [Execution Coursera--35] Max retries (5) reached. Task failed with stage=0.   \n",
       "105                            [Execution GitHub--24] Max retries (5) reached. Task failed with stage=0.   \n",
       "106                                                    Error running task: Unknown tool:  openFileDialog   \n",
       "107                     [Execution Google Flights--9] Max retries (5) reached. Task failed with stage=0.   \n",
       "108                          [Execution Coursera--27] Max retries (5) reached. Task failed with stage=0.   \n",
       "109                            [Execution GitHub--10] Max retries (5) reached. Task failed with stage=0.   \n",
       "110                            [Execution GitHub--21] Max retries (5) reached. Task failed with stage=0.   \n",
       "111                    [Execution Google Flights--25] Max retries (5) reached. Task failed with stage=0.   \n",
       "112                            [Execution GitHub--39] Max retries (5) reached. Task failed with stage=0.   \n",
       "113                    [Execution Google Flights--24] Max retries (5) reached. Task failed with stage=0.   \n",
       "114                             [Execution GitHub--7] Max retries (5) reached. Task failed with stage=0.   \n",
       "115                          [Execution Coursera--25] Max retries (5) reached. Task failed with stage=0.   \n",
       "116                             [Execution GitHub--3] Max retries (5) reached. Task failed with stage=0.   \n",
       "117                               [Execution ESPN--3] Max retries (5) reached. Task failed with stage=0.   \n",
       "118                            [Execution GitHub--38] Max retries (5) reached. Task failed with stage=0.   \n",
       "119                    [Execution Google Flights--34] Max retries (5) reached. Task failed with stage=0.   \n",
       "120                    [Execution Google Flights--18] Max retries (5) reached. Task failed with stage=0.   \n",
       "121                    [Execution Google Flights--20] Max retries (5) reached. Task failed with stage=0.   \n",
       "122                    [Execution Google Flights--27] Max retries (5) reached. Task failed with stage=0.   \n",
       "123                    [Execution Google Flights--39] Max retries (5) reached. Task failed with stage=0.   \n",
       "124                    [Execution Google Flights--35] Max retries (5) reached. Task failed with stage=0.   \n",
       "125                          [Execution Coursera--30] Max retries (5) reached. Task failed with stage=0.   \n",
       "126                               [Execution ESPN--8] Max retries (5) reached. Task failed with stage=0.   \n",
       "127                          [Execution Coursera--31] Max retries (5) reached. Task failed with stage=0.   \n",
       "128                     [Execution Google Flights--8] Max retries (5) reached. Task failed with stage=0.   \n",
       "129                              [Execution ESPN--34] Max retries (5) reached. Task failed with stage=0.   \n",
       "130                    [Execution Google Flights--28] Max retries (5) reached. Task failed with stage=0.   \n",
       "131                            [Execution GitHub--32] Max retries (5) reached. Task failed with stage=0.   \n",
       "132                         [Execution Google Map--0] Max retries (5) reached. Task failed with stage=0.   \n",
       "133                            [Execution GitHub--19] Max retries (5) reached. Task failed with stage=0.   \n",
       "134                            [Execution GitHub--13] Max retries (5) reached. Task failed with stage=0.   \n",
       "135                            [Execution GitHub--25] Max retries (5) reached. Task failed with stage=0.   \n",
       "136                          [Execution Coursera--37] Max retries (5) reached. Task failed with stage=0.   \n",
       "137                        [Execution Google Map--17] Max retries (5) reached. Task failed with stage=0.   \n",
       "138                        [Execution Google Map--16] Max retries (5) reached. Task failed with stage=0.   \n",
       "139                    [Execution Google Flights--41] Max retries (5) reached. Task failed with stage=0.   \n",
       "140                            [Execution GitHub--29] Max retries (5) reached. Task failed with stage=0.   \n",
       "141                    [Execution Google Flights--32] Max retries (5) reached. Task failed with stage=0.   \n",
       "142                            [Execution GitHub--37] Max retries (5) reached. Task failed with stage=0.   \n",
       "143                        [Execution Google Map--11] Max retries (5) reached. Task failed with stage=0.   \n",
       "144                          [Execution Coursera--40] Max retries (5) reached. Task failed with stage=0.   \n",
       "145                           [Execution Coursera--7] Max retries (5) reached. Task failed with stage=0.   \n",
       "146                         [Execution Google Map--2] Max retries (5) reached. Task failed with stage=0.   \n",
       "147                          [Execution Coursera--23] Max retries (5) reached. Task failed with stage=0.   \n",
       "148                              [Execution ESPN--36] Max retries (5) reached. Task failed with stage=0.   \n",
       "149                            [Execution GitHub--23] Max retries (5) reached. Task failed with stage=0.   \n",
       "150                            [Execution GitHub--34] Max retries (5) reached. Task failed with stage=0.   \n",
       "151                    [Execution Google Flights--29] Max retries (5) reached. Task failed with stage=0.   \n",
       "152                    [Execution Google Flights--38] Max retries (5) reached. Task failed with stage=0.   \n",
       "153                    [Execution Google Flights--23] Max retries (5) reached. Task failed with stage=0.   \n",
       "154                               [Execution ESPN--6] Max retries (5) reached. Task failed with stage=0.   \n",
       "155                            [Execution GitHub--14] Max retries (5) reached. Task failed with stage=0.   \n",
       "156                            [Execution GitHub--33] Max retries (5) reached. Task failed with stage=0.   \n",
       "157                                                                            Average Score for 351/449   \n",
       "158                                                                          final_answer_file_not_found   \n",
       "159                                                                                   is_aborted == True   \n",
       "160                                                                                  is_aborted == False   \n",
       "161                                                     empty_answer_below_action_budget_but_not_aborted   \n",
       "162                                                                 average_num_actions_all: 0.00 ± 0.00   \n",
       "163                                                         average_num_actions_non_aborted: 0.00 ± 0.00   \n",
       "\n",
       "     Count Percentage  \\\n",
       "0       26       5.8%   \n",
       "1       17       3.8%   \n",
       "2       11       2.4%   \n",
       "3        8       1.8%   \n",
       "4        7       1.6%   \n",
       "5        5       1.1%   \n",
       "6        5       1.1%   \n",
       "7        2       0.4%   \n",
       "8        1       0.2%   \n",
       "9        1       0.2%   \n",
       "10       1       0.2%   \n",
       "11       1       0.2%   \n",
       "12       1       0.2%   \n",
       "13       1       0.2%   \n",
       "14       1       0.2%   \n",
       "15       1       0.2%   \n",
       "16       1       0.2%   \n",
       "17       1       0.2%   \n",
       "18       1       0.2%   \n",
       "19       1       0.2%   \n",
       "20       1       0.2%   \n",
       "21       1       0.2%   \n",
       "22       1       0.2%   \n",
       "23       1       0.2%   \n",
       "24       1       0.2%   \n",
       "25       1       0.2%   \n",
       "26       1       0.2%   \n",
       "27       1       0.2%   \n",
       "28       1       0.2%   \n",
       "29       1       0.2%   \n",
       "30       1       0.2%   \n",
       "31       1       0.2%   \n",
       "32       1       0.2%   \n",
       "33       1       0.2%   \n",
       "34       1       0.2%   \n",
       "35       1       0.2%   \n",
       "36       1       0.2%   \n",
       "37       1       0.2%   \n",
       "38       1       0.2%   \n",
       "39       1       0.2%   \n",
       "40       1       0.2%   \n",
       "41       1       0.2%   \n",
       "42       1       0.2%   \n",
       "43       1       0.2%   \n",
       "44       1       0.2%   \n",
       "45       1       0.2%   \n",
       "46       1       0.2%   \n",
       "47       1       0.2%   \n",
       "48       1       0.2%   \n",
       "49       1       0.2%   \n",
       "50       1       0.2%   \n",
       "51       1       0.2%   \n",
       "52       1       0.2%   \n",
       "53       1       0.2%   \n",
       "54       1       0.2%   \n",
       "55       1       0.2%   \n",
       "56       1       0.2%   \n",
       "57       1       0.2%   \n",
       "58       1       0.2%   \n",
       "59       1       0.2%   \n",
       "60       1       0.2%   \n",
       "61       1       0.2%   \n",
       "62       1       0.2%   \n",
       "63       1       0.2%   \n",
       "64       1       0.2%   \n",
       "65       1       0.2%   \n",
       "66       1       0.2%   \n",
       "67       1       0.2%   \n",
       "68       1       0.2%   \n",
       "69       1       0.2%   \n",
       "70       1       0.2%   \n",
       "71       1       0.2%   \n",
       "72       1       0.2%   \n",
       "73       1       0.2%   \n",
       "74       1       0.2%   \n",
       "75       1       0.2%   \n",
       "76       1       0.2%   \n",
       "77       1       0.2%   \n",
       "78       1       0.2%   \n",
       "79       1       0.2%   \n",
       "80       1       0.2%   \n",
       "81       1       0.2%   \n",
       "82       1       0.2%   \n",
       "83       1       0.2%   \n",
       "84       1       0.2%   \n",
       "85       1       0.2%   \n",
       "86       1       0.2%   \n",
       "87       1       0.2%   \n",
       "88       1       0.2%   \n",
       "89       1       0.2%   \n",
       "90       1       0.2%   \n",
       "91       1       0.2%   \n",
       "92       1       0.2%   \n",
       "93       1       0.2%   \n",
       "94       1       0.2%   \n",
       "95       1       0.2%   \n",
       "96       1       0.2%   \n",
       "97       1       0.2%   \n",
       "98       1       0.2%   \n",
       "99       1       0.2%   \n",
       "100      1       0.2%   \n",
       "101      1       0.2%   \n",
       "102      1       0.2%   \n",
       "103      1       0.2%   \n",
       "104      1       0.2%   \n",
       "105      1       0.2%   \n",
       "106      1       0.2%   \n",
       "107      1       0.2%   \n",
       "108      1       0.2%   \n",
       "109      1       0.2%   \n",
       "110      1       0.2%   \n",
       "111      1       0.2%   \n",
       "112      1       0.2%   \n",
       "113      1       0.2%   \n",
       "114      1       0.2%   \n",
       "115      1       0.2%   \n",
       "116      1       0.2%   \n",
       "117      1       0.2%   \n",
       "118      1       0.2%   \n",
       "119      1       0.2%   \n",
       "120      1       0.2%   \n",
       "121      1       0.2%   \n",
       "122      1       0.2%   \n",
       "123      1       0.2%   \n",
       "124      1       0.2%   \n",
       "125      1       0.2%   \n",
       "126      1       0.2%   \n",
       "127      1       0.2%   \n",
       "128      1       0.2%   \n",
       "129      1       0.2%   \n",
       "130      1       0.2%   \n",
       "131      1       0.2%   \n",
       "132      1       0.2%   \n",
       "133      1       0.2%   \n",
       "134      1       0.2%   \n",
       "135      1       0.2%   \n",
       "136      1       0.2%   \n",
       "137      1       0.2%   \n",
       "138      1       0.2%   \n",
       "139      1       0.2%   \n",
       "140      1       0.2%   \n",
       "141      1       0.2%   \n",
       "142      1       0.2%   \n",
       "143      1       0.2%   \n",
       "144      1       0.2%   \n",
       "145      1       0.2%   \n",
       "146      1       0.2%   \n",
       "147      1       0.2%   \n",
       "148      1       0.2%   \n",
       "149      1       0.2%   \n",
       "150      1       0.2%   \n",
       "151      1       0.2%   \n",
       "152      1       0.2%   \n",
       "153      1       0.2%   \n",
       "154      1       0.2%   \n",
       "155      1       0.2%   \n",
       "156      1       0.2%   \n",
       "157  0.701      78.2%   \n",
       "158     98      21.8%   \n",
       "159      0       0.0%   \n",
       "160    351      78.2%   \n",
       "161      0       0.0%   \n",
       "162    0.0    449/449   \n",
       "163    0.0    351/449   \n",
       "\n",
       "                                                                             Sample Trajectories  \n",
       "0                                                  ArXiv--21, GitHub--9, ESPN--5, ... (+23 more)  \n",
       "1             Google Map--20, Cambridge Dictionary--25, Cambridge Dictionary--27, ... (+14 more)  \n",
       "2                                                 ESPN--22, Amazon--13, GitHub--0, ... (+8 more)  \n",
       "3                 Booking--17, Cambridge Dictionary--38, Cambridge Dictionary--18, ... (+5 more)  \n",
       "4    Cambridge Dictionary--40, Cambridge Dictionary--34, Cambridge Dictionary--39, ... (+4 more)  \n",
       "5                                            BBC News--1, ArXiv--38, BBC News--20, ... (+2 more)  \n",
       "6                                Cambridge Dictionary--35, Apple--39, Booking--43, ... (+2 more)  \n",
       "7                                                                       Booking--34, Booking--25  \n",
       "8                                                                                     GitHub--26  \n",
       "9                                                                                       ESPN--24  \n",
       "10                                                                                   Coursera--2  \n",
       "11                                                                                 Google Map--9  \n",
       "12                                                                                  Coursera--15  \n",
       "13                                                                             Google Flights--3  \n",
       "14                                                                                    GitHub--36  \n",
       "15                                                                                Google Map--15  \n",
       "16                                                                                  Coursera--14  \n",
       "17                                                                                   Coursera--8  \n",
       "18                                                                       Cambridge Dictionary--6  \n",
       "19                                                                                 Google Map--8  \n",
       "20                                                                                      ESPN--23  \n",
       "21                                                                                    GitHub--40  \n",
       "22                                                                            Google Flights--17  \n",
       "23                                                                                     GitHub--2  \n",
       "24                                                                            Google Flights--15  \n",
       "25                                                                                  Coursera--21  \n",
       "26                                                                                      ESPN--15  \n",
       "27                                                                                  Coursera--41  \n",
       "28                                                                            Google Flights--13  \n",
       "29                                                                                 Google Map--6  \n",
       "30                                                                            Google Flights--14  \n",
       "31                                                                                      ESPN--39  \n",
       "32                                                                                     GitHub--6  \n",
       "33                                                                                    GitHub--17  \n",
       "34                                                                                  Coursera--33  \n",
       "35                                                                            Google Flights--26  \n",
       "36                                                                                      ESPN--31  \n",
       "37                                                                                  Coursera--20  \n",
       "38                                                                                    GitHub--11  \n",
       "39                                                                             Google Flights--0  \n",
       "40                                                                                Google Map--12  \n",
       "41                                                                                    GitHub--28  \n",
       "42                                                                            Google Flights--40  \n",
       "43                                                                                      ESPN--41  \n",
       "44                                                                                    GitHub--18  \n",
       "45                                                                                       ESPN--0  \n",
       "46                                                                                   Coursera--1  \n",
       "47                                                                                  Coursera--34  \n",
       "48                                                                            Google Flights--11  \n",
       "49                                                                            Google Flights--36  \n",
       "50                                                                                    GitHub--35  \n",
       "51                                                                                      ESPN--20  \n",
       "52                                                                                       ESPN--7  \n",
       "53                                                                            Google Flights--16  \n",
       "54                                                                                      ESPN--18  \n",
       "55                                                                                    GitHub--22  \n",
       "56                                                                                   Coursera--4  \n",
       "57                                                                                   Coursera--5  \n",
       "58                                                                            Google Flights--10  \n",
       "59                                                                            Google Flights--21  \n",
       "60                                                                                Google Map--10  \n",
       "61                                                                                Google Map--13  \n",
       "62                                                                             Google Flights--6  \n",
       "63                                                                            Google Flights--30  \n",
       "64                                                                                  Coursera--12  \n",
       "65                                                                                   Booking--36  \n",
       "66                                                                                 Google Map--3  \n",
       "67                                                                                 Google Map--5  \n",
       "68                                                                                  Coursera--22  \n",
       "69                                                                                   Booking--42  \n",
       "70                                                                            Google Flights--19  \n",
       "71                                                                            Google Flights--37  \n",
       "72                                                                            Google Flights--31  \n",
       "73                                                                                  Coursera--39  \n",
       "74                                                                                    GitHub--15  \n",
       "75                                                                            Google Flights--12  \n",
       "76                                                                                  Coursera--18  \n",
       "77                                                                            Google Flights--22  \n",
       "78                                                                                      ESPN--27  \n",
       "79                                                                                    GitHub--27  \n",
       "80                                                                                    GitHub--31  \n",
       "81                                                                            Google Flights--33  \n",
       "82                                                                      Cambridge Dictionary--41  \n",
       "83                                                                                    GitHub--16  \n",
       "84                                                                                   Coursera--3  \n",
       "85                                                                                Google Map--14  \n",
       "86                                                                                  Coursera--16  \n",
       "87                                                                                    GitHub--20  \n",
       "88                                                                                  Coursera--13  \n",
       "89                                                                                 Google Map--7  \n",
       "90                                                                                    GitHub--30  \n",
       "91                                                                                  Coursera--10  \n",
       "92                                                                                  Coursera--24  \n",
       "93                                                                                 Google Map--1  \n",
       "94                                                                                   Booking--37  \n",
       "95                                                                                   Coursera--6  \n",
       "96                                                                                      ArXiv--6  \n",
       "97                                                                                      ESPN--28  \n",
       "98                                                                                   Coursera--9  \n",
       "99                                                                                  Coursera--32  \n",
       "100                                                                                     ESPN--12  \n",
       "101                                                                                     ESPN--35  \n",
       "102                                                                                Google Map--4  \n",
       "103                                                                                     ESPN--10  \n",
       "104                                                                                 Coursera--35  \n",
       "105                                                                                   GitHub--24  \n",
       "106                                                                                    ArXiv--18  \n",
       "107                                                                            Google Flights--9  \n",
       "108                                                                                 Coursera--27  \n",
       "109                                                                                   GitHub--10  \n",
       "110                                                                                   GitHub--21  \n",
       "111                                                                           Google Flights--25  \n",
       "112                                                                                   GitHub--39  \n",
       "113                                                                           Google Flights--24  \n",
       "114                                                                                    GitHub--7  \n",
       "115                                                                                 Coursera--25  \n",
       "116                                                                                    GitHub--3  \n",
       "117                                                                                      ESPN--3  \n",
       "118                                                                                   GitHub--38  \n",
       "119                                                                           Google Flights--34  \n",
       "120                                                                           Google Flights--18  \n",
       "121                                                                           Google Flights--20  \n",
       "122                                                                           Google Flights--27  \n",
       "123                                                                           Google Flights--39  \n",
       "124                                                                           Google Flights--35  \n",
       "125                                                                                 Coursera--30  \n",
       "126                                                                                      ESPN--8  \n",
       "127                                                                                 Coursera--31  \n",
       "128                                                                            Google Flights--8  \n",
       "129                                                                                     ESPN--34  \n",
       "130                                                                           Google Flights--28  \n",
       "131                                                                                   GitHub--32  \n",
       "132                                                                                Google Map--0  \n",
       "133                                                                                   GitHub--19  \n",
       "134                                                                                   GitHub--13  \n",
       "135                                                                                   GitHub--25  \n",
       "136                                                                                 Coursera--37  \n",
       "137                                                                               Google Map--17  \n",
       "138                                                                               Google Map--16  \n",
       "139                                                                           Google Flights--41  \n",
       "140                                                                                   GitHub--29  \n",
       "141                                                                           Google Flights--32  \n",
       "142                                                                                   GitHub--37  \n",
       "143                                                                               Google Map--11  \n",
       "144                                                                                 Coursera--40  \n",
       "145                                                                                  Coursera--7  \n",
       "146                                                                                Google Map--2  \n",
       "147                                                                                 Coursera--23  \n",
       "148                                                                                     ESPN--36  \n",
       "149                                                                                   GitHub--23  \n",
       "150                                                                                   GitHub--34  \n",
       "151                                                                           Google Flights--29  \n",
       "152                                                                           Google Flights--38  \n",
       "153                                                                           Google Flights--23  \n",
       "154                                                                                      ESPN--6  \n",
       "155                                                                                   GitHub--14  \n",
       "156                                                                                   GitHub--33  \n",
       "157                                                                                               \n",
       "158                                                                                               \n",
       "159                                                                                               \n",
       "160                                                                                               \n",
       "161                                                                                               \n",
       "162                                                                                               \n",
       "163                                                                                               "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Re-initialize error tracking dictionaries with the updated function\n",
    "error_counts = defaultdict(int)\n",
    "error_trajectories = defaultdict(list)\n",
    "no_error_count = 0\n",
    "total_trajectories = len(folders)\n",
    "\n",
    "print(f\"processing {total_trajectories} trajectories to extract errors...\")\n",
    "\n",
    "# Aggregate errors from the folders\n",
    "df = aggregate_post_eval_errors(folders)\n",
    "\n",
    "# Configure pandas display options for better formatting\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', 100)\n",
    "pd.set_option('display.width', None)\n",
    "\n",
    "# Display the DataFrame\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fara_webeval",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}