Fara1.5 – A family of frontier computer use agent models
online mind2web works
corby1d6d1d64a6dc
diff --git a/README.md b/README.mdindex 3f6c844..e229fa1 100644--- a/README.md+++ b/README.md@@ -286,6 +286,8 @@ Make sure you set a valid OpenAI GPT-4o endpoint in `endpoint_configs_gpt4o/dev````bashpython webvoyager.py --model_url /path/where/you/want/to/download/model/ --model_port 5000 --eval_oai_config ../endpoint_configs_gpt4o/dev/ --out_url /path/to/save/eval/files --device_id 0,1 --processes 1 --run_id 1 --max_rounds 100+python om2w.py --model_url /path/where/you/want/to/download/model/ --model_port 5000 --eval_oai_config ../endpoint_configs_o4/dev/ --eval_model o4-mini --out_url /path/to/save/eval/files --device_id 0,1 --processes 1 --run_id 1 --max_rounds 100+```**Option 2: Azure Foundry Deployment**@@ -294,6 +296,7 @@ Deploy [Fara-7B on Foundry endpoint(s)](https://ai.azure.com/explore/models/Fara```bashpython webvoyager.py --model_endpoint ../../endpoint_configs/ --eval_oai_config ../endpoint_configs_gpt4o/dev/ --out_url /path/to/save/eval/files --processes 1 --run_id 1_endpoint --max_rounds 100+python om2w.py --model_endpoint ../../endpoint_configs/ --eval_oai_config ../endpoint_configs_o4/dev/ --eval_model o4-mini --out_url /path/to/save/eval/files --processes 1 --run_id 1_endpoint --max_rounds 100```### Notes@@ -324,10 +327,12 @@ Example path:Each evaluation folder contains:- `gpt_eval/` - LLM-as-a-judge evaluation results- `traj/` - Per-task trajectory subdirectories containing:- - `final_answer.json` (e.g., `Amazon--1_final_answer.json`) - `<no_answer>` indicates abortion or step budget exceeded- - `scores/gpt_eval.json` - LLM judge scores+ - `*-final_answer.json` (e.g., `Amazon--1_final_answer.json`) - `<no_answer>` indicates abortion or step budget exceeded+ - `scores/*_eval.json` - LLM judge scores (`gpt_eval.json` for WebVoyager, `WebJudge_Online_Mind2Web_eval-3.json` for Online-Mind2Web)- `web_surfer.log` - Action history and errors- `screenshot_X.png` - Screenshots captured before each action X+ - `times.json` - contains start and endtimes of task+ - `core.log` - contains high level logs like if the trajectory needs to be started or is already cached/done, the evaluation score, duration, and errors encountered### Running Analysis@@ -357,6 +362,18 @@ To re-run failed tasks, execute the evaluation script again with the same `run_i</details>+<details>+<summary>Example of web_surfer.log (action history)</summary>++```json+{"message": "\nThought #1: To begin locating the closest GameStop store, I will click on the 'Trade-In' button, which typically leads to a store locator feature on retail websites.\nAction #1: executing tool 'left_click' with arguments {\"action\": \"left_click\", \"coordinate\": [1074, 37]}", "timestamp": "2025-11-25T22:23:42.788320", "type": "OtherEvent"}+{"message": "Observation#1: I clicked at coordinates (1083.0252100840337, 37.16517857142858).", "timestamp": "2025-11-25T22:23:47.616613", "type": "OtherEvent"}+{"message": "\nThought #2: Since access to Gamestop's website has been blocked, I will search for alternative sources of store location and hours for GameStop locations near zip code 90028. This could include cached or third-party website information.\nAction #2: executing tool 'web_search' with arguments {\"action\": \"web_search\", \"query\": \"GameStop store locator zip code 90028\"}", "timestamp": "2025-11-25T22:23:50.579702", "type": "OtherEvent"}+{"message": "Observation#2: I typed 'GameStop store locator zip code 90028' into the browser search bar.", "timestamp": "2025-11-25T22:23:55.119495", "type": "OtherEvent"}+{"message": "\nThought #3: The closest GameStop store to zip code 90028 is located at 5533 W Sunset Blvd, Los Angeles, CA 90028. It is currently closed and will open tomorrow at 11 AM. The contact number is (323) 466-3714.\nAction #3: executing tool 'terminate' with arguments {\"action\": \"terminate\", \"status\": \"success\"}", "timestamp": "2025-11-25T22:23:58.729038", "type": "OtherEvent"}+{"message": "Observation#3: The closest GameStop store to zip code 90028 is located at 5533 W Sunset Blvd, Los Angeles, CA 90028. It is currently closed and will open tomorrow at 11 AM. The contact number is (323) 466-3714.", "timestamp": "2025-11-25T22:24:02.379069", "type": "OtherEvent"}+```+---## Citationdiff --git a/webeval/data/om2w/Online_Mind2Web_06042025.json b/webeval/data/om2w/Online_Mind2Web_06042025.jsonnew file mode 100644index 0000000..cfd2bc4--- /dev/null+++ b/webeval/data/om2w/Online_Mind2Web_06042025.json@@ -0,0 +1,2102 @@+[+ {+ "task_id": "b7258ee05d75e6c50673a59914db412e",+ "confirmed_task": "Find the store location and hours of the closest Gamestop to zip code 90028 and set it as the home store on Gamestop.",+ "website": "https://www.gamestop.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "ade4c09ad3fdb1607209750924cd232f",+ "confirmed_task": "Compare available plans for the AeroAPI on Flightaware.",+ "website": "https://www.flightaware.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "fb7b4f784cfde003e2548fdf4e8d6b4f",+ "confirmed_task": "Open the page with an overview of the submission of releases on Discogs.",+ "website": "https://www.discogs.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "824eb7bb0ef1ce40bfd49c12182d9428",+ "confirmed_task": "Get the lowest priced women's plus size one piece swimsuit in color black with a customer rating of at least 5 on Kohls.",+ "website": "https://www.kohls.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "046138801a05ddf56ad94e8672942496",+ "confirmed_task": "Find discussions of the community and open one with the most replies on Flightaware.",+ "website": "https://www.flightaware.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "92a3d4236f167af4afdc08876a902ba6",+ "confirmed_task": "Find a 2022 Tesla Model 3 on CarMax.",+ "website": "https://www.carmax.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "48c73f3f53e2611c4a1052457c1033db",+ "confirmed_task": "Get the report from the final environmental impact statement for the Jamaica Bus Depot expansion on new.mta.info.",+ "website": "https://new.mta.info/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "8f2611047de227a2ca8bda13f6e2e5fb",+ "confirmed_task": "Find the used 2012-2013 Honda Crosstour with the lowest mileage for under $25,000 near zip code 49102 on CarGurus.",+ "website": "https://www.cargurus.com/",+ "reference_length": 17,+ "level": "hard"+ },+ {+ "task_id": "b320c68bffc1f3c7f2a8dc9d5478fb27",+ "confirmed_task": "Find a walkthrough for the game \"The Legend of Zelda: Breath of the Wild\" on ign.",+ "website": "https://www.ign.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "aa4b5cb7114fcc138ade82b4b9716d24",+ "confirmed_task": "Find an editor's choice review with a score of 10 in the boardgame category on ign.",+ "website": "https://www.ign.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "005be9dd91c95669d6ddde9ae667125c",+ "confirmed_task": "Find the weight of baggage allowance for economy class on Qatar Airways.",+ "website": "https://www.qatarairways.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "323bd85e3559655d89e5496b951a25e8",+ "confirmed_task": "Tell me information about what identification I need to bring on my trip on Amtrak.",+ "website": "https://www.amtrak.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "123e8c2fc453f55fadd1d0b9aaf94df4",+ "confirmed_task": "Browse used Audi cars made before 2015 and sort by lowest price on KBB.",+ "website": "https://www.kbb.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "56f8890a837c49f7df766b9c981646f3",+ "confirmed_task": "Show crazy credits for the movie \" Prometheus\" on IMDb.",+ "website": "https://www.imdb.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "644a856c3897665e475e0dce50bf217d",+ "confirmed_task": "Find a pair of wireless headphones on Amazon with active noise canceling for $100 or less and add them to the cart.",+ "website": "https://www.amazon.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "62f1626ce249c31098854f8b38bdd6cf",+ "confirmed_task": "Find Playstation 5 digital edition on gamestop.",+ "website": "https://www.gamestop.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "561693d6eec7bbfba3fefe9e4b26decb",+ "confirmed_task": "Browse Marriott Bonvoy credit cards on Marriott.",+ "website": "https://www.marriott.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "b7a9a6b5d451164c09bbd27b670bc2ae",+ "confirmed_task": "Show me the list of Men's Blazers, Black, Size M on Uniqlo.",+ "website": "https://www.uniqlo.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "bfa2de159be6978acf2702be31a2eeeb",+ "confirmed_task": "Show me the options for a roundtrip leaving from Las Vegas on flexible dates on the interactive map on united.",+ "website": "https://www.united.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "4091bdd3fa64a5b0d912bc08eaf9c824",+ "confirmed_task": "Find the list of neighborhood maps for Brooklyn on new.mta.info.",+ "website": "https://new.mta.info/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "79f0bd7df6e685f30f20025cc6755c0a",+ "confirmed_task": "Find me the cheapest external Hard Drive for an Xbox One on GameStop.",+ "website": "https://www.gamestop.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "6ebde509dca8f15c0fa1bd74f071e8d6",+ "confirmed_task": "Search for a job in Miami, Florida, in Human Resources on target.",+ "website": "https://www.target.com/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "34ccd15a8ea8fd3895af83f5ccf62369",+ "confirmed_task": "Find out what to do when I lose an item on a bus on us.megabus.",+ "website": "https://us.megabus.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "c698ff3fc0f6cbce39947c597ab5749b",+ "confirmed_task": "Browse the page with event planning tips on Eventbrite.",+ "website": "https://www.eventbrite.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "b6d10e9bd19b4009a02dea0e98f4e1ae",+ "confirmed_task": "Check the current standings for MLS on Fox Sports.",+ "website": "https://www.foxsports.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "d71be72aa25c3eab8eea47a0e60382e2",+ "confirmed_task": "Find technical specs for the latest Macbook Air on Apple.",+ "website": "https://www.apple.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "0b51b4fa0295ae80ccd176ebdad6fff6",+ "confirmed_task": "Search for a red Toyota Corolla from model years 2018 to 2023 on CarMax.",+ "website": "https://www.carmax.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "3f312ae3efc3c3e90ababe050dd4e7ae",+ "confirmed_task": "Find the current NFL standings for the AFC East division on NFL.com and go to the page on which team is in first place.",+ "website": "https://www.nfl.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "95cad96f2e43f3c0d8efad1331c77c8c",+ "confirmed_task": "View the list of the Most Popular TV on rotten tomatoes.",+ "website": "https://www.rottentomatoes.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "bf3b311cc8dce16d3de844f4b5875dfd",+ "confirmed_task": "Compare Apple watches and learn more about the ultra version on apple.",+ "website": "https://www.apple.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "b64f938af842f6a1b4489d0e49a785a7",+ "confirmed_task": "Get the frozen vegan cheese pizza between 5 to 10 USD on Target.",+ "website": "https://www.target.com/",+ "reference_length": 17,+ "level": "hard"+ },+ {+ "task_id": "5e1b8254c123c80178cc28e0afdb14f0",+ "confirmed_task": "Find a help page about buying tickets on seatgeek.",+ "website": "https://seatgeek.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "f27c0a7b8b0bb33d37698dff227fc8d7",+ "confirmed_task": "Browse used Mercedes-Benz cars from model years 2004 to 2012 on KBB and sort by highest price.",+ "website": "https://www.kbb.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "8fdec8eeffd3491e6526cc78c028120b",+ "confirmed_task": "See Nissan and Honda cars for sale near Kentwood, MI 49512 on CarMax.",+ "website": "https://www.carmax.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "7b182a5087347d494b48a29dbc0f1d3e",+ "confirmed_task": "Find a shelter or rescue group near zip code 90011.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "828c2d98616a9478d5864d847d5a1b28",+ "confirmed_task": "Browse the list of Civil Division forms.",+ "website": "https://www.justice.gov/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "608c595eec271fa5dc03506923519994",+ "confirmed_task": "Calculate a FedEx Ground shipping rate for a 3-pound package from zip code 10019 to zip code 90028.",+ "website": "https://www.fedex.com/en-us/home.html",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "a7a73c8fa75441fc76df9746c327bdd6",+ "confirmed_task": "Estimate the cost of a photographer in 07055 for a 4-hour project.",+ "website": "https://www.thumbtack.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "cfafe3771369d1d261e9f7ecd44c296d",+ "confirmed_task": "Find the highest-rated dealer for Cadillac with a rating above 4 stars within 20 miles of zip 60606.",+ "website": "https://www.cars.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "bbbc243b4f18a7a897f0bc84e11d293f",+ "confirmed_task": "Find out how many assists Chris Paul has been averaging in the current season.",+ "website": "https://www.nba.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "816851ff92ff0219acf4364dcc2c4692",+ "confirmed_task": "Search for boys' infant pajamas below $40.",+ "website": "https://www.macys.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "8244409b2c82043f966cad05f9afe132",+ "confirmed_task": "Find the best Audiologist within 50 miles of New York, NY, with a rating of 4 and above.",+ "website": "https://doctor.webmd.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "e7301bb694871429bf2eb36c3a72186c",+ "confirmed_task": "Find baby shoes priced under $20 with a 5-star rating.",+ "website": "https://www.macys.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "905cb53061c33aa2d77e485fe1fca516",+ "confirmed_task": "Browse dermatologists within 10 miles of zip code 10019 and filter by only those who accept Blue Medicare Advantage.",+ "website": "https://www.healthgrades.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "fcf4952d2a1d80ea505c555c3c3b54e7",+ "confirmed_task": "Find the cheapest used 8-cylinder bmw made between 2005-2015 and priced from 25,000 to 50,000 dollars with mileage less than 50,000 miles or less.",+ "website": "https://www.cars.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "3c1ffc3f494e423b3c434c79e35da8f3",+ "confirmed_task": "Find 12 Monkeys community and view the latest posts mentioning James Cole.",+ "website": "https://www.reddit.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "26a0e5c21c145dd8448aa92f35bec5ea",+ "confirmed_task": "Browse optometrists who offer telehealth services in Columbus, OH.",+ "website": "https://www.healthgrades.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "070c907d34a4ce71dfdbea38f9c5d4d8",+ "confirmed_task": "Find a dentist who specializes in pediatric dentistry and is located near zip code 90210 (within 5-mile distance).",+ "website": "https://www.healthgrades.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "43a1ca251f11c6b0bdd0379766cc49e6",+ "confirmed_task": "Find a neurosurgeon who is over 50 years old and has an appointment available tomorrow.",+ "website": "https://www.healthgrades.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "b3f8bd9198d9d157e0848109563c4b23",+ "confirmed_task": "Find a permanent job in Logistics within 20 miles of New York, zip 11005, in the middle-income range for a high school diploma holder.",+ "website": "https://ohiomeansjobs.ohio.gov/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "20a460a8fe1971b84411c5b1e6ac4186",+ "confirmed_task": "Show theatre events for Las Vegas and select one.",+ "website": "https://www.stubhub.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "db1ffb5e60578597d1c3aa3c389ac7b1",+ "confirmed_task": "Search for smart TVs with a screen size of 55 to 65 inches and filter the results to show only those that have an LED display.",+ "website": "https://www.google.com/shopping?udm=28",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "7be8cd8dba885cddd9af5320f49bc41b",+ "confirmed_task": "Find roofing contractors within 5 miles of zip code 10002.",+ "website": "https://www.bbb.org/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "239a29bde438fe44fe17fe1390ef1634",+ "confirmed_task": "Find me a gluten-free diet to lose weight for a pregnant woman.",+ "website": "https://www.healthline.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "9f1cba613830ca1c6a58f9498c06e679",+ "confirmed_task": "Find a premier real estate agent in St Augustine, FL.",+ "website": "https://www.redfin.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "75146b7b67388b9244e0f21a1527c022",+ "confirmed_task": "Find a male senior boxer near zip code 90028.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "59b7b990b4828bc305ab0d7ed6071b55",+ "confirmed_task": "Get owner-financing homesite land for sale in New Mexico, Luna County, listed in the last 30 days, and contact the cheapest per acre land seller.",+ "website": "https://www.landwatch.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "9c97bab9c2abfb90a426cbe9addae8d0",+ "confirmed_task": "Check the details of order 12345 with email [email protected].",+ "website": "https://www.macys.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "871e7771cecb989972f138ecc373107b",+ "confirmed_task": "Find the weather for Vancouver, British Columbia for the next seven days.",+ "website": "https://www.theweathernetwork.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "b69eb4de621e9e265676daac44938f3f",+ "confirmed_task": "Find an adult husky near zip code 10019.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "9bb63ad0e38d5691a618932a8b31c05a",+ "confirmed_task": "Look for reviews of a Nest Hello Video Doorbell and filter by 1-star ratings.",+ "website": "https://www.google.com/shopping?udm=28",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "8ae510355d978424f490798f900bfa2c",+ "confirmed_task": "Show me the shared rooms in any university in Melbourne that has a private bathroom wifi, and gas included in the bills.",+ "website": "https://www.student.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "4c186c6ed888d0c8d4cf4adb39443080",+ "confirmed_task": "Find a medium Devin Booker jersey and add it to the shopping cart.",+ "website": "https://www.nba.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "2fc51dd3febd447f0fdcdabca8d944ce",+ "confirmed_task": "Locate a self-storage unit near zip code 60538 that can fit about a dorm room full of items and is climate-controlled.",+ "website": "https://www.extraspace.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "eb323dc584156d0eb3a2b90bb8c4b791",+ "confirmed_task": "Find the latest 2 bed and 1.5+ bath apartment listing for rent in New York.",+ "website": "https://www.redfin.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "87f4c5128e36cdb9366a138a7b61bb00",+ "confirmed_task": "View the speakers that are bluetooth and wireless and filter the results to only show models that are on sale and cost less than $50.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "354b4ddf048815f8fd4163d0d7e1aaa3",+ "confirmed_task": "Browse marketing jobs and filter by Bachelor's Degree education level.",+ "website": "https://ohiomeansjobs.ohio.gov/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "e4e097222d13a2560db6f6892612dab6",+ "confirmed_task": "Search for a young spayed male dog cared for by a private owner within 50 miles of zip 33109.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "f389398d2eeb29e5571e00439c57eb76",+ "confirmed_task": "Find the latest climate news.",+ "website": "https://www.theweathernetwork.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "8ea6c3a2ea3f59150619935261a76d19",+ "confirmed_task": "Find a staffed FedEx location near zip code 10019 to return a package.",+ "website": "https://www.fedex.com/en-us/home.html",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "c1d6ea6f2196d25782cc3646ff3090db",+ "confirmed_task": "Create a list of drip coffee makers that are on sale and within $25-60 and have a black finish.",+ "website": "https://www.google.com/shopping?udm=28",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "2dd41b1d0e8f389d0683f4a4627abfe6",+ "confirmed_task": "Show houses for sale in Maryland with a maximum price of $60,000.",+ "website": "https://www.landwatch.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "f2097f92a10d42a842c14179f422311e",+ "confirmed_task": "Add a $50 Uber gift card to the cart.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "85b284c18d7e78c9b5a9e074e7aa3b98",+ "confirmed_task": "View the cheapest apartment available for students at the University of Leeds with bills that include WIFI and cleaning services.",+ "website": "https://www.student.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "853afd530c72f4b00ffc32ae854efaf8",+ "confirmed_task": "Show me the wind flow map for Belo Horizonte.",+ "website": "https://www.accuweather.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "c09721cc937d4dcfb391a0bc2c574b28",+ "confirmed_task": "Find the next available date for Albion Basin.",+ "website": "https://www.recreation.gov/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "4c572a627b53b0f9a734ab37f21819b8",+ "confirmed_task": "Browse apartments with at least 2 bedrooms and 2 bathrooms and a max price of $4000 per month.",+ "website": "https://craigslist.org/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "301f267f421b93045874726183e8f722",+ "confirmed_task": "Find healthy savory vegan snack recipes which can be cooked within 5 minutes and contain a high level of protein.",+ "website": "https://www.healthline.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "4f903626f632586fe4728d6664947bab",+ "confirmed_task": "Find press releases by the antitrust division in 2022.",+ "website": "https://www.justice.gov/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "3ec0f6138d37fadcb989347a6088ec45",+ "confirmed_task": "Open the page to learn more about how to get accredited.",+ "website": "https://www.bbb.org/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "2207bb4f21786690cfed20b37253fb8b",+ "confirmed_task": "Check the current wind speed in Calgary, Alberta.",+ "website": "https://www.theweathernetwork.com/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "9c04b71bb8db6cf8e743b2290cbc8797",+ "confirmed_task": "Find a UPS drop-off point near Miami Florida.",+ "website": "https://www.ups.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "4e0f5561a76478da87995dee00b09572",+ "confirmed_task": "Show me the monthly weather forecast for Florida City.",+ "website": "https://www.accuweather.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "7562d9b4e4829a44245aafce2e1f62db",+ "confirmed_task": "Find the nearest location to zip code 54620 that offers size 4 P.O. Boxes.",+ "website": "https://www.usps.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "bd1e3770b7181f6fce9c35e18caa9785",+ "confirmed_task": "Browse service listings for a solar panel installer and hide duplicates.",+ "website": "https://craigslist.org/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "330cd04c773ac498f51afa4665461ec8",+ "confirmed_task": "Browse couches for sale, sort by cheapest, and search in titles only.",+ "website": "https://craigslist.org/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "ec78d3a635e417bc2a80d03ca93d7165",+ "confirmed_task": "What are the benefits and financial support a single person living in England, over the state pension age, unemployed, with no health conditions, or caring for someone with one, can get?",+ "website": "https://www.gov.uk/",+ "reference_length": 16,+ "level": "hard"+ },+ {+ "task_id": "a0a18ca6a3529f3e97c771aadd42d3a0",+ "confirmed_task": "Add a men's T-shirt that is in large size with a stripe pattern, short sleeve, and under the Best Sellers group to the cart.",+ "website": "https://www.macys.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "82eb3bfedd78456a0230b389f4e7a938",+ "confirmed_task": "Open the XRP yearly chart.",+ "website": "https://coinmarketcap.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "e7f6cca9a8875f98fee3b711ead3a444",+ "confirmed_task": "Find the comments made by the user Separate-Camp7202.",+ "website": "https://www.reddit.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "75a1b5dcd2c28508a971d98d51fe5767",+ "confirmed_task": "Open the reviews of a recipe with beef sirloin.",+ "website": "https://www.allrecipes.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "c03ee2be3d73556ab789c0ad1cbd3451",+ "confirmed_task": "Find a dog groomer for nail trimming within 100 miles of zip code 10005 and check the detailed service prices of the first one.",+ "website": "https://www.akc.org/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "05483c50cc9b04c8ac44c574758fb2bd",+ "confirmed_task": "Look for the best rated BBB accredited charity near 12023.",+ "website": "https://www.bbb.org/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "a172a5d9ffaf5ef02bd550ec4fe24e6d",+ "confirmed_task": "Browse the natural products database.",+ "website": "https://www.drugs.com/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "7e1047f4803237f319c004f7a7f6bccb",+ "confirmed_task": "Discover the trade-in value of my Intel 7th generation i3 Windows 10, HP laptop in fair condition, which has 8 GB memory and can be powered on, proceed for the in-store trade-in.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "f2be37a9a60fbc25b6b11cf622d17352",+ "confirmed_task": "Find obedience trials in state of New York during the month of May.",+ "website": "https://www.akc.org/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "e24662008c3be5d56f986f232fcec447",+ "confirmed_task": "Find the stock price for WWE over the last month.",+ "website": "https://www.google.com/finance/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "0170ca95038b05fa58d463fe627ac605",+ "confirmed_task": "Check if a visa is required to work in the UK for longer than 6 months in Healthcare as an American citizen.",+ "website": "https://www.gov.uk/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "b3a7da968de13bbdcaed12ffe4993df6",+ "confirmed_task": "Compare the breeds Afghan Hound, Akita and Azawakh.",+ "website": "https://www.akc.org/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "515f2e5811cfdd5e0e669e40f17886d8",+ "confirmed_task": "Search for a new internal M2 Samsung SSD drive between $25 and $200.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "4d3157aab34b54e5f0c4b965dfe930f3",+ "confirmed_task": "Show me community posts about pregnancy fever from the past 30 days.",+ "website": "https://www.babycenter.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "e4e19e04286f644d747d8c5a79d17fac",+ "confirmed_task": "Find the Drug Interaction Report for Viagra and alcohol.",+ "website": "https://www.drugs.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "cad62d2be0c53f08a416457486b3db23",+ "confirmed_task": "Search for adoptable dogs near 21122 zip code.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "9ef1a8972f375db59c0e6329e11b7939",+ "confirmed_task": "Find Farms land in Wilkes County, NC with the lowest price.",+ "website": "https://www.landwatch.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "a11ecdff735b51372d536c866011af6f",+ "confirmed_task": "Explore courses related to Psychology.",+ "website": "https://www.coursera.org/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "7fff82864f21ddeccf4104a220892824",+ "confirmed_task": "Find the lowest 27\"-32\" Samsung or LG computer monitors nearby which have 4k, IPS display.",+ "website": "https://www.google.com/shopping?udm=28",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "50d91eabde542906937ab4c5b6f8f23a",+ "confirmed_task": "Calculate Pregnancy Weight Gain for a 5-week pregnancy with a 169lb weight before pregnancy and a 175lb after pregnancy with a 5.6ft height.",+ "website": "https://www.babycenter.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "dcd26e662a616d373ddd339747c6ce5b",+ "confirmed_task": "Take a weight management quiz to find a motivating article for a non-exercising, mostly eating out and can't control portions and cravings, and who has a strong support system, enjoys traveling, loves family time and cooking.",+ "website": "https://www.healthline.com/",+ "reference_length": 22,+ "level": "hard"+ },+ {+ "task_id": "eb2db4b769c145dbe6ba4f74f3e0de98",+ "confirmed_task": "Find an energetic hairless dog with medium barking.",+ "website": "https://www.akc.org/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "c0fa2c0e622971955cabf5bcf7b777e8",+ "confirmed_task": "Search for rentals in Corning, CA with a maximum price of $1500.",+ "website": "https://www.apartments.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "ce616721ce9aeda69890fbccb29677a6",+ "confirmed_task": "Calculate the price to ship a large flat-rate box from 77449 to 77084 at the first available date and time.",+ "website": "https://www.usps.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "9d09bc948462db032bac98968b11b008",+ "confirmed_task": "Find NHL events occurring in Boston.",+ "website": "https://www.stubhub.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "29526b17a32485742b5ab63507e99417",+ "confirmed_task": "Browse Humira dosage information.",+ "website": "https://www.drugs.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "d7c955b47af68e01766fa86d0bee08a7",+ "confirmed_task": "Add Elevate at Chicago, IL, to favorites and show a virtual tour.",+ "website": "https://www.apartments.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "9d090a15c214eb070d9caa8a034d03c1",+ "confirmed_task": "Find the lowest-priced Student housing near Liverpool International College which has been priced between 100 to 300 pounds and has a private bathroom.",+ "website": "https://www.student.com/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "5916018d1cad999881018cac1216a692",+ "confirmed_task": "Find a personal trainer service at 10040 for a 25-year-old client aiming to build muscle.",+ "website": "https://www.thumbtack.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "0059adc6b12a3822305deb68929b2de8",+ "confirmed_task": "Find support services jobs in Bentonville, in the state of Arkansas.",+ "website": "https://careers.walmart.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "07bdc595306729a028ba06cc7451a80a",+ "confirmed_task": "Select a high speed train ticket with a departure time before 23:00 from Shanghai to Beijing.",+ "website": "https://us.trip.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "64b76158720a69e4a5c31a55d54928bf",+ "confirmed_task": "Compare two pescatarian diets for eating healthier.",+ "website": "https://www.healthline.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "e3ab665e01e7632ce33ac1aeca14aff6",+ "confirmed_task": "Find the next available dates for Alley Creek Camp.",+ "website": "https://www.recreation.gov/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "2d5a7f95f951a26838289dfd629ae850",+ "confirmed_task": "Find a list of houses for sale in zip code 85747 with a private pool.",+ "website": "https://www.redfin.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "26810ed9c123a62992e3eed31db3c5ee",+ "confirmed_task": "Show daily weather for New York City.",+ "website": "https://www.accuweather.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "c181f903ec1107b850032c17cad88393",+ "confirmed_task": "Help me identify a pink round pill with 150 written on it.",+ "website": "https://www.webmd.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "ef289e34a2f59a707cb07e2a6229ff03",+ "confirmed_task": "Compare the Acura CL 2003 with the ILX 2022.",+ "website": "https://www.cars.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "84f806c7fc15576673915f195efa72df",+ "confirmed_task": "Find a nationwide nearest animal shelter for birds around zip 10012.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "be9e7dca1222714571ef3d7d59d2a41c",+ "confirmed_task": "Find out the cold and flu forecast and today's air quality in Champaign, IL.",+ "website": "https://weather.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "11abb668c751dd56bb41f296a8bb3a13",+ "confirmed_task": "Find a store near zip 30010 that provides authorized Apple services for imacs and make this one my store.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "207e933d1bba815bcb58664b5d82c085",+ "confirmed_task": "Find Ohio City apartments with parking, a fitness center, and an elevator.",+ "website": "https://www.apartments.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "29b7372d5a3884a2ba831af2d117af3c",+ "confirmed_task": "Browse the first top news of Microsoft stock on Google Finance.",+ "website": "https://www.google.com/finance/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "5c00e9561eae94789443f405525a5869",+ "confirmed_task": "Find the recommended dosage for Vivitrol.",+ "website": "https://www.healthline.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "2532fd402d3c741b79894e6ff2269f53",+ "confirmed_task": "find electricians near 10203.",+ "website": "https://www.thumbtack.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "9829f3087ab1f9c8eba6b6dd2b831d25",+ "confirmed_task": "Play the latest video from NBA TV.",+ "website": "https://www.nba.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "783ce6a3499fa7cf25bc12f8f0ecbbbb",+ "confirmed_task": "Find Florida internship programs in the Mayo Clinic College of Medicine and Science.",+ "website": "https://www.mayoclinic.org/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "6db4a0e346976f2729ba9afcd3208941",+ "confirmed_task": "Look up tracking information for shipment #3023858502.",+ "website": "https://www.fedex.com/en-us/home.html",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "1fc28d91d25ccd1c6ba268101326a654",+ "confirmed_task": "Find the 5-day price chart for Bitcoin.",+ "website": "https://www.google.com/finance/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "255bf27c43fd3f9254d6b81a5f36d3a9",+ "confirmed_task": "Look for the largest hunting land for auction in Kansas high plain region with mineral rights posted in the last seven days.",+ "website": "https://www.landwatch.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "a8b9edd598561d2de901864d5f40fe67",+ "confirmed_task": "Calculate the shipping cost for 4 pound package from Texas to New York.",+ "website": "https://www.fedex.com/en-us/home.html",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "a6f0434ce6aff5f9b03681241b03ad82",+ "confirmed_task": "Find the closing stock price for Tesla on March 17, 2023.",+ "website": "https://finance.yahoo.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "415bf9da6f3db3a735ecbba3b0c76c15",+ "confirmed_task": "Find the nearest vet within 50 miles of zip 75228.",+ "website": "https://www.akc.org/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "8103786e0e5976ebf961bd062d5f39cd",+ "confirmed_task": "Find possible causes for the symptoms of chest pain which is sharp which is accompanied by anxiety.",+ "website": "https://www.mayoclinic.org/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "92160852a6bbbc165cee4e14ab0b1d59",+ "confirmed_task": "Find the shipping cost of a Common medium-sized box in flat-rate shipping and compare it with other parcel services.",+ "website": "https://www.ups.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "502e864440283214e0180645015f568b",+ "confirmed_task": "Check permit availability for a group of 4 in Brooks Camp, Katmai National Park on May 22.",+ "website": "https://www.recreation.gov/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "7680a920359cb1a508fbddb001b98167",+ "confirmed_task": "See the prediction about the girl child's height, whose current height at seven years is 4 feet and whose weight is 55 lbs, her mother is 5 feet 2, and her father is 5 feet 8.",+ "website": "https://www.babycenter.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "07ec4a12cba8090e2dc524d558ac7675",+ "confirmed_task": "Check drug interaction for melatonin and Folate Forte.",+ "website": "https://www.drugs.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "987bad7c6d4726d64232a8a1c3386888",+ "confirmed_task": "Find the seller info and seller's notes about the used car model 2011 BMW 135 with a max price of $30000.",+ "website": "https://www.cars.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "15be05973fba714e490cd9c884e4f072",+ "confirmed_task": "Find the procedure to get the license for Athletic Trainer.",+ "website": "https://ohio.gov/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "3adeea7627f4343069f38adae40f73d0",+ "confirmed_task": "Within 25 Miles of 96817, find a nursing home that accepts medicare.",+ "website": "https://health.usnews.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "c94551d2b18f9ad0ab31b0bd98ca42e3",+ "confirmed_task": "Find cats available for adoption within 10 miles of zip code 94587, Young or adult-age cats, sorted by Oldest Addition.",+ "website": "https://www.petfinder.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "4e801ba102dfaf22c7cf7a126b107609",+ "confirmed_task": "Find Linux platform software developers in 10080 who master the Python language and Java language with web interface project type.",+ "website": "https://www.thumbtack.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "39c388cdc468688c8139cc2bb5157c13",+ "confirmed_task": "Calculate the estimated car loan payment amount for an average credit-rated person for a 15,000-dollar car with a down payment of 2000 dollars and loan tenure of 48 months in zip 65215 and shop for the lowest-priced car.",+ "website": "https://www.cars.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "c8d7f2aa7eb5dd074c48c9f76f8659ad",+ "confirmed_task": "Show Teen Driver Safety program information.",+ "website": "https://www.dmv.virginia.gov/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "fd787623166785d84093565bf945fd24",+ "confirmed_task": "Check the interaction between Novolin N and Novolin R.",+ "website": "https://www.drugs.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "c3307a70bb12ebf56cc9ec926b368f15",+ "confirmed_task": "Find the interactions between Eulexin and hepatic dysfunction.",+ "website": "https://www.drugs.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "9586827ad04ee2362f4f0076bf0f0468",+ "confirmed_task": "Find the side effects of taking Montelukast.",+ "website": "https://www.drugs.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "34992feb69eb8e788faa06868b365c49",+ "confirmed_task": "Submit a request for vehicle registration renewal with title number X123456 and last 4 digits of VIN is 1234.",+ "website": "https://www.dmv.virginia.gov/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "47b93b9e649eadeb8d96a6e3df715c2d",+ "confirmed_task": "Show me Diagnoses & Treatment for Female infertility.",+ "website": "https://www.mayoclinic.org/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "3443e9c3151fef19a3c3a45eb2c13640",+ "confirmed_task": "Search for the ovulation calculator and enter Mar 1 as the first date of the period and calculate the date of ovulation and pregnancy test day.",+ "website": "https://www.webmd.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "6b5be1764692d1dc8f17dc4375b2daa8",+ "confirmed_task": "Show me historical data for EUR/USD.",+ "website": "https://finance.yahoo.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "16200f51d63f0a47a58fa17acd49e368",+ "confirmed_task": "Find a recipe that includes eggplant and mushrooms.",+ "website": "https://cookpad.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "c2153fc053112e89c2f103869c4d6890",+ "confirmed_task": "Find a house cleaning service in 10001 on a weekly basis.",+ "website": "https://www.thumbtack.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "5e4e89c9b6fdaee7a41aca5601b82e04",+ "confirmed_task": "Identify a pill with a pink color and oval shape with 894 5 number on it.",+ "website": "https://www.drugs.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "60cbbbd58eb9d28b053aef945f464228",+ "confirmed_task": "Look up if the phone number 555555555 is a scam.",+ "website": "https://www.bbb.org/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "8f80e64e44e1fada018997b2fe869683",+ "confirmed_task": "What are the top posts of all time on Reddit?",+ "website": "https://www.reddit.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "65c4030f22fb6eb101acfee4825f1318",+ "confirmed_task": "Find a female MD Cardiologist in Jacksonville, Florida.",+ "website": "https://www.mayoclinic.org/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "6ca20f1da01edeb49a7a42c816d8c6fe",+ "confirmed_task": "Find the Eligibility to get the child benefit and How it works and how to claim",+ "website": "https://www.gov.uk/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "2e4e21cf1449c6894b17d571c47b77ea",+ "confirmed_task": "Find an English bulldog near zip code 90028 that was cared for by a private owner.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "1df24ec81137386d6476bcf343a79012",+ "confirmed_task": "Search for NordicTrack with the lowest price.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "4639a54f3ab549864fd8d60b7398b1e1",+ "confirmed_task": "Find a white female kitten within 35 miles of zip 77494.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "9af05e392cf3f5a8ff17aa764ba5bda6",+ "confirmed_task": "Get a quote from C and above-rated solar energy equipment company within 10 miles of Miami, Florida.",+ "website": "https://www.bbb.org/",+ "reference_length": 16,+ "level": "hard"+ },+ {+ "task_id": "627f7a18d85f29a687234f1ade4585c2",+ "confirmed_task": "Find the current league leader in total blocked shots.",+ "website": "https://www.nba.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "0b838cd54f826c59c71f600c56b89a11",+ "confirmed_task": "Find all the locations for the second-best-rated used car dealer less than 5 miles from New York.",+ "website": "https://www.bbb.org/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "5dec0e6620849459f29e6465982c597e",+ "confirmed_task": "Search for 33 to 49inch Qled gaming monitor with a 240hz refresh rate that is within $1000 to $2000.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "52efbab520734ef9bf7c09ba0f62cdc8",+ "confirmed_task": "Find the app for iOS.",+ "website": "https://www.recreation.gov/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "b1ce968a361e1088ce8d2ade6c2c9af0",+ "confirmed_task": "Find young cats in Seattle and show off the newest additions.",+ "website": "https://www.petfinder.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "23204728192da9f73197a613d9681c18",+ "confirmed_task": "What are the Symptoms and causes of fever?",+ "website": "https://www.mayoclinic.org/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "a69d2934fe54fef165490a5a2d95bf38",+ "confirmed_task": "Show me recipes for pancakes with wheat and without beetroot.",+ "website": "https://cookpad.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "e9f4dfc67e0e6aa37f05f7cc5aa7428c",+ "confirmed_task": "Browse pediatricians near zip code 90028 who specialize in Internal Medicine and have a rating of at least 4 stars.",+ "website": "https://www.healthgrades.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "2218042362d8fae73756eb309848c2b2",+ "confirmed_task": "Compare Audi A7 with Audi A6, both made in 2023, and hide similarities.",+ "website": "https://www.cars.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "ba2a469af584f16da93ce6a7430cf7e5",+ "confirmed_task": "Search for a beginner\u2019s course in computer science that includes advertisement skills.",+ "website": "https://www.coursera.org/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "26784156ae9859a0dd6c5920eb106f91",+ "confirmed_task": "calculate and search rent for a $6000 monthly income with 30% rent budget near 90012 area.",+ "website": "https://www.apartments.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "47e314cc452c540524ffb7cf520285a3",+ "confirmed_task": "Find the park that offers the cheapest paddling permits.",+ "website": "https://www.recreation.gov/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "271b36efd4346721b5542488ff997042",+ "confirmed_task": "Browse 8K Samsung TVs that are open box.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "6b2cfae0ef25c73d1224b6ab74cb8b63",+ "confirmed_task": "Find Devin Booker's highest-scoring points per game playoff run.",+ "website": "https://www.nba.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "0a54069a0ef542e571d1fee7f39c93d5",+ "confirmed_task": "Browse senior spayed/neutered dogs near zip code 90028.",+ "website": "https://www.adoptapet.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "4e3f6a538cc1f7321cfc50260db9545d",+ "confirmed_task": "Look up the current temperature for zip code 10019.",+ "website": "https://www.theweathernetwork.com/",+ "reference_length": 2,+ "level": "easy"+ },+ {+ "task_id": "f00e7accfb4a5e09680bdb326e6274ad",+ "confirmed_task": "Check the hourly forecast for Boston.",+ "website": "https://www.accuweather.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "6174e5ddd40cfbdc33ee1502f40bac39",+ "confirmed_task": "Find a day-use park that offers horseback riding near Nashville.",+ "website": "https://www.recreation.gov/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "547f5729c59d5d12a457a3ebb74c31c6",+ "confirmed_task": "Search for 3 bedroom condos with 2 bathrooms within $1500- $2500 range in NYC.",+ "website": "https://www.apartments.com/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "0b2623e9fa5cea997f76490bcbc5220f",+ "confirmed_task": "Find a list of shorthaired dogs available for adoption within 100 miles of zip code 94587 that are good with kids and cats, and have been on Petfinder for over 30 days.",+ "website": "https://www.petfinder.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "3ae28b3c440efe87dc700480b78ac608",+ "confirmed_task": "Find the closest 5-star rated dentist to zip code 98011.",+ "website": "https://www.healthgrades.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "0632e496d37badee0350dad358f047c5",+ "confirmed_task": "Browse recipes for gluten-free chocolate chip cookies that can be made without nuts.",+ "website": "https://cookpad.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "aafd1fddea1558466ac6133934d35156",+ "confirmed_task": "Find a Single-Family House for Rent in Houston, TX with 1 bed.",+ "website": "https://www.apartments.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "246d654fab7c31d9651007e39e75f74f",+ "confirmed_task": "Open the most helpful 5-star reviews of Alpine Ridge.",+ "website": "https://www.recreation.gov/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "690d7b4a285fdb1e9dabf973bf46ae4d",+ "confirmed_task": "Browse iPhone X for sale that is in good condition, has a max price of 400, and searches in titles only.",+ "website": "https://craigslist.org/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "c43a7dccf5c44f7b45a821e712dd1970",+ "confirmed_task": "Take a newsletter subscription with my email id ([email protected]) for Allergies and asthma, Anxiety and depression, nutrition, diabetes, breast cancer, and migraine with email id.",+ "website": "https://www.healthline.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "d5c34bf39eb6096ae5d439325cde4d32",+ "confirmed_task": "Find a DMV center in Richmond.",+ "website": "https://www.dmv.virginia.gov/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "180ed2ec377ef3a4af9035a21522091a",+ "confirmed_task": "Find the way to give a gift to UM-Dearborn.",+ "website": "https://umich.edu/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "c521933dad9c0ef9f1dfa2f38b8e4405",+ "confirmed_task": "See the monthly forecast for Atlanta, GA.",+ "website": "https://www.accuweather.com/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "9b5dfe54a1c14c5c6336bae7374c3bb5",+ "confirmed_task": "Find a UPS Access Point near SPRING, TX and services provided by them.",+ "website": "https://www.ups.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "c073ac1bcf40f84c599affc97edbc396",+ "confirmed_task": "Search for the cheapest apartment in Detroit for a student.",+ "website": "https://www.apartments.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "73d08420706ae205a9c5be28b6d4e80f",+ "confirmed_task": "Show me the rules and cancellation for Alley Spring.",+ "website": "https://www.recreation.gov/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "0a0fa834ce41b5297c6474293383759d",+ "confirmed_task": "What are the onboard activities of the highest-rated Regent Seven Seas Cruise ship based on Costco member reviews?",+ "website": "https://www.costco.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "a13e4231a3d6a7000c622c56448d97ba",+ "confirmed_task": "Find an Airbnb in Cleveland for three nights. The check-in date is the day after tomorrow. We have 2 adults, 2 kids, and 1 pet. The budget is $100 to $300 per night. Essential amenities include free parking, a washer, and a gym.",+ "website": "https://www.airbnb.com/",+ "reference_length": 19,+ "level": "hard"+ },+ {+ "task_id": "bb518416a786fdb9b9bbf0c78515595e",+ "confirmed_task": "Browse the class schedule of graduate-level computer science courses.",+ "website": "https://www.osu.edu/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "b99c02965196d51e80ac7539e33f335b",+ "confirmed_task": "Please find graduate-level computer science courses scheduled on Tuesdays starting time from 2:00 to 6:00 PM in the Fall 2023 semester.",+ "website": "https://www.berkeley.edu/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "27fa3ac20745d3d35e89fae157f63069",+ "confirmed_task": "Browse the class schedule of graduate-level chemistry courses on Monday afternoons in the winter of 2023.",+ "website": "https://www.stanford.edu/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "b4aa7315e31dfcdc52baf7771be260c9",+ "confirmed_task": "Find the HGX H100 driver for Ubuntu 22.04 on AMD64 CPU.",+ "website": "https://www.nvidia.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "442a450e696a96085257db6297891a4d",+ "confirmed_task": "Using a calculator to determine how much I can have in my 401(k) account at retirement, if I work from age 22 to 65, with an annual rate of return of 3%, annual employee contributions of $8,000, and annual employer contributions of $8,000.",+ "website": "https://www.chase.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "9ed3827266b3b804f485859c3d00401e",+ "confirmed_task": "If I'm 30, plan to retire at 65, and can save $300/month, with a 3% annual return, 13% current tax rate, and 24% retirement tax rate, show the comparison chart between Traditional and Roth IRA.",+ "website": "https://www.chase.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "c801d1c951f59297f526bab84fa86c6e",+ "confirmed_task": "Browse the latest negative reviews from players with over 100 hours of playtime for the game that won the 2023 VR Game of the Year Award.",+ "website": "https://store.steampowered.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "7c09c2c7c87cf6bb1138701eb54284ea",+ "confirmed_task": "Find the comments for the most popular news in the past month under the Quantum Physics topic.",+ "website": "https://phys.org/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "afcebfed28bea091d58f49ea6cb8194b",+ "confirmed_task": "Find the most reviewed gluten-free multivitamins from CVS Health Brand under $15.",+ "website": "https://www.cvs.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "64345c365f544375357c7b67917f08a0",+ "confirmed_task": "Look for the newest refrigerator that is 34-36 inches wide, priced between $1,000 and $2,000, and has a customer review rating of 4 stars or higher.",+ "website": "https://www.costco.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "ab6ee3b83aab6cd283320f5e01003cff",+ "confirmed_task": "Find the tech specs of the MacBook Pro 16-inch introduced in November 2023.",+ "website": "https://www.apple.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "33bd2cdcea4fcc42a09a8a1e4e5841c6",+ "confirmed_task": "Add a 5-piece Tenders Combo to my bag with Sweet Corn as the side, Sweet Tea as the drink, and both Honey BBQ and Honey Mustard sauces. Select the store closest to Zip code 10001 for pick-up tomorrow at 12:00 PM.",+ "website": "https://www.kfc.com/",+ "reference_length": 23,+ "level": "hard"+ },+ {+ "task_id": "47186fac8e7c7277af01144644eb4e0b",+ "confirmed_task": "What is the ownership cost of the first car in the list \"top buys 2025\"?",+ "website": "https://www.parkers.co.uk/",+ "reference_length": 3,+ "level": "easy"+ },+ {+ "task_id": "fa9adb815b85d259f943d81874a052e5",+ "confirmed_task": "Browse a user homepage that reposted the top song from the Top 50 Rock chart.",+ "website": "https://soundcloud.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "b922508886ded315c9835457a6eb43ea",+ "confirmed_task": "Browse tenured/tenure-track faculty positions in Computer Sciences & Technology in California.",+ "website": "https://jobs.chronicle.com",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "5d542a7ec1fa142ba73cc87d970caf39",+ "confirmed_task": "Find the most cited publication at the 2022 CVPR main conference.",+ "website": "https://dblp.org/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "864244b6969e0f8733b0eb1ca06cd51f",+ "confirmed_task": "Find the race time for who wins the first place in the last race of the 2023 Formula 1 (F1).",+ "website": "https://www.espn.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "01abae9608f2d8752a83e08f136f720c",+ "confirmed_task": "Show me the code for the company that is the top mover in the Cboe Europe Technology Sector Index (BEPTEC) as of the latest market close.",+ "website": "https://www.cboe.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "da8f3823a827c7d3a492f383808e7912",+ "confirmed_task": "Find and open the earliest press release.",+ "website": "https://www.instructure.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "8689af4d33ce00bf2cdd8987d3bbfd86",+ "confirmed_task": "Add the cheapest certified refurbished iPad Air with 256GB of storage in any shade of blue to my bag.",+ "website": "https://www.apple.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "78f397336b6fd1cbba0127db7a8cd502",+ "confirmed_task": "Browse the upcoming SuperBike events taking place in Italy.",+ "website": "https://www.redbull.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "3dca7cbe7d086619d837ff9f5312cebc",+ "confirmed_task": "Can you show me products under the category path 'Automotive' -> 'Car Jack', with an additional filter for the color pink?",+ "website": "https://us.shein.com/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "b962927dfe03bf2274a54381127ed433",+ "confirmed_task": "Find the best-selling vinyl record by an artist from New York City in the classical music genre.",+ "website": "https://bandcamp.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "78baf9dbe7c3532f7d7ef4cc22a7f065",+ "confirmed_task": "Find the most popular digital trends report in the Finance & Insurance industry within the region of China.",+ "website": "https://www.statista.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "c7c07ec10c668625a21ba64165d719bb",+ "confirmed_task": "Find the total monthly price for four prepaid unlimited lines without autopay discounts.",+ "website": "https://www.verizon.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "512fd4deab099b8dc0dcfc0ec48a3c63",+ "confirmed_task": "Identify the open issue with the most comments in the first trending open-source repository this week.",+ "website": "https://github.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "d9d8b7d84a3f8d057e368254fe8d65e2",+ "confirmed_task": "Find the first commit submitted by NielsRogge to the official repository of the SAM2 model.",+ "website": "https://github.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "157f4a79d55e8fa3fd55ba772ba40fbc",+ "confirmed_task": "Find the most popular blue Lilo & Stitch toys.",+ "website": "https://www.disney.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "62c8d970b3d13891f355911e5a8f4030",+ "confirmed_task": "Find the top game listed in the Steam Deck's top-played list over the past year. Then, browse reviews for that game from players who have played over 100 hours and primarily use a Steam Deck.",+ "website": "https://store.steampowered.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "11857213ca01510f12813740afd59918",+ "confirmed_task": "Add the most top-selling Adidas men's basketball shoe in red, size 10 to my cart.",+ "website": "https://www.adidas.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "47bfe8a7e0e4e7efc837287b407fbe90",+ "confirmed_task": "Compare the first and second most popular smartphones manufactured by Xiaomi and show the comparison chart.",+ "website": "https://versus.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "bb314cb80f0f8489135cbf59074d11e2",+ "confirmed_task": "Open the page for the first Best Paper Award video recording of talks from ICLR 2016.",+ "website": "https://iclr.cc/",+ "reference_length": 4,+ "level": "easy"+ },+ {+ "task_id": "1aeca99e6a60b0e3aefb3ef212bdce79",+ "confirmed_task": "Find full-time legal occupation jobs in San Diego County with a minimum salary of $4,000+ per month.",+ "website": "https://www.ca.gov/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "d730f4ff450da1bd60a836163736ef6a",+ "confirmed_task": "Find the best-selling GORE-TEX men's hiking shoe priced between $100.00 and $199.99 with a rating of 4 stars or higher, and show its most helpful comment.",+ "website": "https://www.rei.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "fe33894188d20d7469f37a9fd855e7ff",+ "confirmed_task": "Find me Python 3.9 packages on PyPI that are designed for the Web Environment, licensed under MIT, have a stable production status, and are intended for developers.",+ "website": "https://pypi.org/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "71f8de1834599fba443f40dbbfab8edd",+ "confirmed_task": "Search for papers related to reinforcement learning under the topics of computer science and mathematics on arxiv, with recent submission dates between September 2024 and January 2025.",+ "website": "https://arxiv.org/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "c8c1ff115879b3afd14280beb1559b13",+ "confirmed_task": "Find the latest Doraemon video in MP4 format that is over 20 minutes long and has a medium file size.",+ "website": "https://www.4shared.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "d4fb78b7e74508cd3b33f01cf9200997",+ "confirmed_task": "Show the figure comparing Occupational Fatalities Trends between Ohio and New York.",+ "website": "https://www.americashealthrankings.org/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "0e42c3a73f2aece1f854e0ba55b7c8b0",+ "confirmed_task": "Find a gas station in Manhattan, NY with a rating above 4.0, and sort the user reviews by the lowest rating.",+ "website": "https://www.google.com/maps/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "96afb3c51146b0c2a9c55f039a5ea6d6",+ "confirmed_task": "Find the most frequent word that rhymes with \"thought\" and has three syllables.",+ "website": "https://www.merriam-webster.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "59912927c1fddee6ded8a49986896bc2",+ "confirmed_task": "Look for the most useful reviews of the highest-rated anti-reflective TVs with screen sizes from 55\" to 64\" and prices ranging from $300 to $1500.",+ "website": "https://www.samsung.com/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "e43cbc8a0bf9e999884928d11006f894",+ "confirmed_task": "Browse the list of things to do in Miami that have a rating of 9+ (wonderful), last between 1 to 4 hours per session, cost under $100 per person, and are available for booking between next Monday and next Friday.",+ "website": "https://www.expedia.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "1b867afecf072cb877ebfa4069263746",+ "confirmed_task": "Display the figure comparing unemployment trends among women in Illinois and Michigan.",+ "website": "https://www.americashealthrankings.org/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "c3a333968fc3c43d7f2688f425a0d633",+ "confirmed_task": "Find the cheapest certified pre-owned Porsche 911 with a model year of 2019 or newer, within a 200-mile radius of ZIP code 97007.",+ "website": "https://www.porsche.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "bb5d90e6f2fbc0ae146f7c1998c2b4a1",+ "confirmed_task": "Find the most viewed TED talk on the topic of robots that lasts between 12 and 18 minutes.",+ "website": "https://www.ted.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "c577a14301a725e09ccd269a3e0b271e",+ "confirmed_task": "Return the page for the highest-rated red wine from Oregon under $40 that pairs well with either mushrooms or veal.",+ "website": "https://www.vivino.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "c6c9dc6079677cef594cec2fa6b16602",+ "confirmed_task": "Add the cheapest black sofa with at least three seats, a leather finish, and at least four stars to my cart.",+ "website": "https://www.ikea.com/",+ "reference_length": 16,+ "level": "hard"+ },+ {+ "task_id": "c39d6c245f8243993e707d54d2f4acec",+ "confirmed_task": "Browse the final skin in the list for the champion Ahri.",+ "website": "https://www.leagueoflegends.com/",+ "reference_length": 18,+ "level": "hard"+ },+ {+ "task_id": "b2f4fde2fce122a93c7b578086cb0585",+ "confirmed_task": "Find the cheapest hotel + flight + car package from New York to San Francisco, departing tomorrow and returning on the fourth day from departure, for two adults and a six-year-old child. The package should be one room with free breakfast and spa access.",+ "website": "https://www.booking.com/",+ "reference_length": 19,+ "level": "hard"+ },+ {+ "task_id": "d02d236836924919f35f2438d9ed2374",+ "confirmed_task": "Browse the top 250 movies and find one movie that is available on AMC+.",+ "website": "https://www.imdb.com/",+ "reference_length": 22,+ "level": "hard"+ },+ {+ "task_id": "3621b099326c7aebd2e2dac6be3b52d1",+ "confirmed_task": "Open the profile page of the leader of the Nvidia Learning and Perception Lab.",+ "website": "https://www.nvidia.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "f27b393bbd2082f92b566270c4b74fe6",+ "confirmed_task": "Find a large van for sale from the year 2024 or newer with up to 10,000 miles.",+ "website": "https://www.parkers.co.uk/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "ba01ea557b73f864c35ebba0dd6f3cb2",+ "confirmed_task": "Find the top-rated hotel in Manhattan, NY, suitable for 4 guests, and identify the fastest public transportation option from the hotel to LGA airport.",+ "website": "https://www.google.com/maps/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "662ae0f2d3ac851dbcdd245f908277e3",+ "confirmed_task": "What is the second stop among the best stops along the road trip from Yellowstone National Park to Las Vegas?",+ "website": "https://wanderlog.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "461ab9b0c7b20ac5f912704480979c65",+ "confirmed_task": "Find the NYSE Rule 605 Market Center Files data for July 2024.",+ "website": "https://www.nyse.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "a96fca87a17d792644e736d1d10d3cbe",+ "confirmed_task": "View the pricing plan for 'Business'. Specifically, we have 100 users. We need a 1PB storage quota and a 50 TB transfer quota.",+ "website": "https://mega.io/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "2c8ef01a92c71ba9ef2e59bb17eea2b3",+ "confirmed_task": "If there are any discounts on the Apple Mac Studio, add the one with the largest absolute discount to my cart; otherwise, add the cheapest one.",+ "website": "https://www.costco.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "3084bc225219fcb73dc1cb0f97276c1c",+ "confirmed_task": "Get quotes for a package weighing 10 lbs with dimensions of 2 inches in length, width, and height, being shipped from Long Beach, 90802 to Portland, 97201.",+ "website": "https://www.ups.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "949dc965a6c23a95663b3bc2ca2c3a8a",+ "confirmed_task": "Find UA or AA flights from London to New York that arrive between 8:00 PM and 11:00 PM on FlightAware.",+ "website": "https://www.flightaware.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "636b07af4dd97c1793733db1fd1b90b8",+ "confirmed_task": "Filter handbags to evening bags that are blue, and polyester and cost less than $100.",+ "website": "https://www.macys.com/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "38203be65401943aea2179c4c680059a",+ "confirmed_task": "Check the status of bus S92 for any disruptions on new.mta.info.",+ "website": "https://new.mta.info/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "cf757a775fa1224acfc7998489e199a8",+ "confirmed_task": "Find a flight from Dublin to anywhere under $100 tomorrow on Ryanair.",+ "website": "https://www.ryanair.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "d8e2a81fa621ce4737e5ea85671b630e",+ "confirmed_task": "Search for regular weekday jobs around 14810 that I can start within two weeks or three.",+ "website": "https://hiring.amazon.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "63d6866fc000fcb1f153e07604bd1395",+ "confirmed_task": "What are the Nearby Attractions from the most popular attraction in Hong Kong?",+ "website": "https://us.trip.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "199be0b54a436daee74247971fc684ee",+ "confirmed_task": "Add a Macy's Happy Birthday E-Gift Card worth $50 from Shak to my cart, with the birthday wish message \"Happy birthday, wish you many more years to come\", addressed to christene ([email protected]).",+ "website": "https://www.macys.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "c00437fd76a7a83b57f3dc4e5dbc41f8",+ "confirmed_task": "Check the most recent full-time medical health and safety jobs, requiring 1-3 years of industry experience available in the US.",+ "website": "https://www.amazon.jobs/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "fc53ddd3421411a41c1020a3fdc84ec4",+ "confirmed_task": "I want to purchase an open-box Samsung Galaxy S25 Plus in excellent condition and trade in a gray Galaxy S20 5G (Verizon), with a perfect screen, in good condition. How much would it cost?",+ "website": "https://www.bestbuy.com/",+ "reference_length": 17,+ "level": "hard"+ },+ {+ "task_id": "9d46ccb915eff39ee1ae1e7328f5f20d",+ "confirmed_task": "Get a quote for the fastest shipping available for 5 lbs with dimensions of 4 inches in length, width, and height from New York, NY 10001, USA to Truckee, California 96162, USA.",+ "website": "https://www.ups.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "d1970c16271496cbbe166ecbecc0a1d8",+ "confirmed_task": "I'm 25 and located in Texas. Shop for 2020 made dry red wine made in United States priced between 15-20 dollars and add 5 bottles to the cart.",+ "website": "https://macyswineshop.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "7211af65d266402f99499053924262e9",+ "confirmed_task": "View the most recent job posting for a full-time pharmacy position in the US.",+ "website": "https://www.amazon.jobs/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "4464a8421f8bc8786524a499258dfad3",+ "confirmed_task": "Check the specifications of the best-selling HP FHD laptop with 16 GB RAM and core i7 running on Windows 11.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "f707d765bca668830745d20807d7bee6",+ "confirmed_task": "Show me the list of young female English Spot rabbits available for adoption in Chicago, IL, within 50 miles.",+ "website": "https://www.petfinder.com/",+ "reference_length": 14,+ "level": "hard"+ },+ {+ "task_id": "d392e154c1c6ffbb26e2331c3afafc67",+ "confirmed_task": "Add a $100 Best Buy gift card for a birthday to my cart.",+ "website": "https://www.bestbuy.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "a5c87cc1c94a090c9a8dc2c8b6a125d0",+ "confirmed_task": "Find the SO2 air quality over the past hour for Maine North, County Cork, Ireland.",+ "website": "https://www.accuweather.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "367d843c640637745e8fafa741cca13b",+ "confirmed_task": "Find a condo for rent in Houston, TX, with a monthly rent of no more than 30% of an income of $8000. The condo should have a minimum area of 600 square feet, and the move-in date is the 1st of next month.",+ "website": "https://www.apartments.com/",+ "reference_length": 15,+ "level": "hard"+ },+ {+ "task_id": "84ef883a37af638c3bcf7561f28ce80a",+ "confirmed_task": "Find the cheapest used hatchback car listing in Madison which has black interiors with a heated seat option and premium sound system.",+ "website": "https://www.cars.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "d9a8689393effeed75ea0866e44e1def",+ "confirmed_task": "Find the address and phone of the Office of the Inspector General (OIG).",+ "website": "https://www.justice.gov/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "1bc154377120ec15b18dbabdba49c741",+ "confirmed_task": "Book 4 tickets in the upper for any Kevin Hart show in New York in the next three months and view ticket prices with estimated fees.",+ "website": "https://www.stubhub.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "28e7574e7bd6d14f36d2988a5ef2bd23",+ "confirmed_task": "Get a part-time job within 5 miles of Moscow, Idaho in the accommodation and food services industry, as a chef, and show jobs for corporate only.",+ "website": "https://ohiomeansjobs.ohio.gov/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "1c3b747ae12ccee895745f82e3f2ef8a",+ "confirmed_task": "Identify the ongoing competition that offers the highest prize and find the code that received the most votes in that competition.",+ "website": "https://www.kaggle.com/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "d1807551297ac60ecaaabbd2a2ed301a",+ "confirmed_task": "Find the No.1 children's hospital in the California that specializes in Neonatology.",+ "website": "https://health.usnews.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "7abdceee212151f187ee1a1744c57606",+ "confirmed_task": "Can you show me the page with the filing fee for a self-petitioned I-140 application?",+ "website": "https://www.uscis.gov/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "0e5536aaad9d3462b06cf725e6ed535a",+ "confirmed_task": "Show me the page with average wait times for U.S. citizens arriving at Raleigh-Durham International Airport on 2025-03-12.",+ "website": "https://www.cbp.gov/",+ "reference_length": 11,+ "level": "hard"+ },+ {+ "task_id": "bc2ce7f206045dd2d322e5695a947219",+ "confirmed_task": "Estimate the federal income tax I would owe on $158,500 of taxable income in ZIP code 97007, filing as single.",+ "website": "https://smartasset.com/",+ "reference_length": 6,+ "level": "medium"+ },+ {+ "task_id": "7e6993f2c5cd72c44809024f0bc85dc1",+ "confirmed_task": "Create a meme with a frog as the background and leave the only text with \"Enjoy your life\".",+ "website": "https://imgur.com/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "a48e2f1ee8d87eaeea56fe5e730427e6",+ "confirmed_task": "Pass the first trending chess puzzle.",+ "website": "https://www.chess.com/",+ "reference_length": 7,+ "level": "medium"+ },+ {+ "task_id": "dd44c665cec1e9c929a4c5f074e7844a",+ "confirmed_task": "Find parking near the San Francisco Museum of Modern Art from June 18, 1:00 PM to 5:00 PM. I'm driving a Ford F-150 and need a garage that allows in-and-out privileges. If there are multiple options, show me the details of the one with the lowest price.",+ "website": "https://spothero.com/",+ "reference_length": 17,+ "level": "hard"+ },+ {+ "task_id": "99daaed9a83c266341d28aa40067d376",+ "confirmed_task": "Find the most popular board game on the 'The Hotness' list that has a rating above 7.5 and is suitable for 2 players.",+ "website": "https://boardgamegeek.com/",+ "reference_length": 5,+ "level": "easy"+ },+ {+ "task_id": "7072d09436972a5d5fe7476e3e9f1559",+ "confirmed_task": "Show me the comparison of the first two personal credit cards that do not charge foreign transaction fees.",+ "website": "https://www.americanexpress.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "2c20d87a046fadcb6ff07ee877bfbf37",+ "confirmed_task": "Open the form 8843 for tax year 2022.",+ "website": "https://www.irs.gov/",+ "reference_length": 8,+ "level": "medium"+ },+ {+ "task_id": "753f372c189d3b306623cb0c65b50320",+ "confirmed_task": "Compare the U.S. ETP Odd Lot Rate (%) between Quartile 1 and Quartile 4, viewing quartiles by price, and display the chart with a logarithmic scale on the vertical axis.",+ "website": "https://www.sec.gov/",+ "reference_length": 9,+ "level": "medium"+ },+ {+ "task_id": "733f1d8bf79d5bc2240c5357f928ffff",+ "confirmed_task": "Find the cheapest travel deal or discount to Thailand that lasts more than 10 days, departs in next month, and show the total price.",+ "website": "https://www.tourradar.com/",+ "reference_length": 10,+ "level": "medium"+ },+ {+ "task_id": "f05e87c5b92d9869e08806103c1c15a1",+ "confirmed_task": "Find all startup companies from the 2022 and 2023 Y Combinator batches that are based in France and currently have job openings.",+ "website": "https://www.ycombinator.com/",+ "reference_length": 12,+ "level": "hard"+ },+ {+ "task_id": "3ef64f34eae59c9fac7ee9a4f18b4a0c",+ "confirmed_task": "Find and open an animal learning course on YouTube Kids for my 6-year-old without login in. As a parent born in 1992, I would prefer not to enable search.",+ "website": "https://www.youtube.com/",+ "reference_length": 16,+ "level": "hard"+ },+ {+ "task_id": "f158345f8489e0d1d91e28768c39bca1",+ "confirmed_task": "Estimate the total cost (with basic support) of using 5 million input tokens and 5 million output tokens each for GPT-4o and GPT-4o Mini, both deployed in the US/EU Data Zones under Standard (On-Demand) in the East US region.",+ "website": "https://azure.microsoft.com/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "1ab384fb3a791edfb410213cc6b82151",+ "confirmed_task": "Show me the result of a proton emission decay for a Beryllium nucleus with 6 protons and 4 neutrons in the simulation.",+ "website": "https://phet.colorado.edu/",+ "reference_length": 13,+ "level": "hard"+ },+ {+ "task_id": "1223b07536a87e0170ff87cbbebd1d3c",+ "confirmed_task": "Complete a multiplication quiz on https://www.coolmath4kids.com/, covering multiplication facts for 11-12. The quiz should consist of 10 questions, with unlimited time allowed for each. The goal is to achieve a perfect score of 10 out of 10.",+ "website": "https://www.coolmath4kids.com/",+ "reference_length": 24,+ "level": "hard"+ }+]\ No newline at end of filediff --git a/webeval/endpoint_configs_o4/dev/o4mini-0-config.json b/webeval/endpoint_configs_o4/dev/o4mini-0-config.jsonnew file mode 100644index 0000000..7ec9f15--- /dev/null+++ b/webeval/endpoint_configs_o4/dev/o4mini-0-config.json@@ -0,0 +1,16 @@+{+ "CHAT_COMPLETION_PROVIDER": "azure",+ "CHAT_COMPLETION_KWARGS_JSON": {+ "api_version": "2025-04-01-preview",+ "azure_endpoint": "https://<your-endpoint>.openai.azure.com/",+ "model_capabilities": {+ "function_calling": true,+ "json_output": true,+ "vision": true+ },+ "azure_deployment": "<your-deployment-name>",+ "azure_ad_token_provider": "default",+ "model": "o4-mini",+ "max_retries": 5+ }+}diff --git a/webeval/scripts/om2w.py b/webeval/scripts/om2w.pynew file mode 100644index 0000000..915fe0e--- /dev/null+++ b/webeval/scripts/om2w.py@@ -0,0 +1,123 @@+from webeval.systems.websurfer import WebSurferSystem+from webeval.benchmarks import OnlineM2WBenchmark+from pathlib import Path+import numpy as np+import os+import logging+# from aztool.workspace import Workspace, AIF_WORKSPACE+import mlflow+from eval_exp import EvalExp, ModelReference, get_foundry_endpoint_configs+from webeval.oai_clients.graceful_client import GracefulRetryClient+from webeval.eval_result import EvalResult, Stage+from arg_parsing import get_eval_args++DEFAULT_DATA_URL = '../data/om2w/Online_Mind2Web_06042025.json'++class Callback:+ def __init__(self):+ self.scores = []++ def __call__(self, result: EvalResult, mlflow_facade, mlflow_run_id: str):+ if result.stage == Stage.EVALUATED:+ self.scores.append(result.score)+ mlflow_facade.log_metric('score', np.mean(self.scores or [0]), run_id = mlflow_run_id)+++def add_om2w_args(parser):+ parser.add_argument('--eval_data_url', type=str, default=DEFAULT_DATA_URL, help='Azure URI to the evaluation data')+ parser.add_argument('--split', type=str, default='*')+ parser.add_argument('--eval_method', type=str, default='WebJudge_Online_Mind2Web_eval', help='Evaluation method (default: WebJudge_Online_Mind2Web_eval, one of: AgentTrek_eval, Autonomous_eval, WebJudge_general_eval, WebJudge_Online_Mind2Web_eval)')+++def main():+ args = get_eval_args(add_om2w_args)+ assert args.eval_model == "o4-mini", "only o4-mini can be used for om2w eval"++ if args.browserbase:+ assert os.environ.get("BROWSERBASE_API_KEY"), "BROWSERBASE_API_KEY environment variable must be set to use browserbase"+ assert os.environ.get("BROWSERBASE_PROJECT_ID"), "BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables must be set to use browserbase"++ experiment = EvalExp(+ ws = None,+ user = args.user,+ seed = args.seed)++ with experiment.start_run() as run:+ model_ref = ModelReference(args.model_url, args.model_port, args.device_id, args.web_surfer_kwargs.get('max_n_images', 3), args.gpt_solver_model_name, args.dtype, args.enforce_eager, use_external_endpoint=bool(args.model_endpoint))++ logger = logging.getLogger('om2w-eval')+ logger.setLevel(logging.INFO)++ mlflow.log_param('max_rounds', args.max_rounds)+ if args.web_surfer_model_type == "gpt_solver":+ mlflow.log_param('web_surfer_model_type', args.web_surfer_model_type + "/" + args.gpt_solver_model_name)+ else:+ mlflow.log_param('web_surfer_model_type', args.web_surfer_model_type)+ mlflow.log_param('fn_call_template', args.fn_call_template)++ # If using external endpoint, load all endpoint configs into a list of dicts+ if args.model_endpoint:+ websurfer_client_cfg = get_foundry_endpoint_configs(args.model_endpoint)+ logger.info(f"Loaded {len(websurfer_client_cfg)} external endpoint config(s) from {args.model_endpoint}")+ model_ref.model_url_to_log = websurfer_client_cfg[0]['base_url'] # log the first endpoint URL as+ model_ref.model_to_log = websurfer_client_cfg[0]['base_url']+ mlflow.log_param('using_external_endpoint', True)+ mlflow.log_param('endpoint_config_path', ','.join([x['base_url'] for x in websurfer_client_cfg]))+ else:+ # For local VLLM, use a simple flat config structure that FaraAgent expects+ websurfer_client_cfg = {+ "api_key": "NONE",+ "model": "gpt-4o-mini-2024-07-18",+ "base_url": f"http://0.0.0.0:{args.model_port}/v1"+ }+ if args.web_surfer_client_cfg is not None:+ websurfer_client_cfg = args.web_surfer_client_cfg+ if args.web_surfer_kwargs:+ mlflow.log_params({f'web_surfer_kwargs.{k}': v for k, v in args.web_surfer_kwargs.items()})++ system = WebSurferSystem(+ system_name="WebSurfer",+ web_surfer_model_type = args.web_surfer_model_type,+ max_rounds = args.max_rounds,+ websurfer_client_cfg = websurfer_client_cfg,+ start_on_target_url=True,+ browserbase=args.browserbase,+ web_surfer_kwargs=args.web_surfer_kwargs,+ gpt_solver_model_name=args.gpt_solver_model_name,+ fn_call_template=args.fn_call_template,+ step_budgets=args.step_budgets,+ )+++ mlflow.log_param("eval_data", args.eval_data_url)+ mlflow.log_param("eval_model", args.eval_model)+ mlflow.log_param("eval_method", args.eval_method)+ data_dir = Path(__file__).resolve().parent.parent / "data" / "om2w"+ data_dir.mkdir(parents=True, exist_ok=True)+ benchmark = OnlineM2WBenchmark(+ data_dir=data_dir,+ eval_method = args.eval_method,+ data_az_url = args.eval_data_url,+ model_client = GracefulRetryClient.from_path(args.eval_oai_config, logger=logger, eval_model=args.eval_model))++ mlflow.log_param('subsample', args.subsample)+ mlflow.log_param('processes', args.processes)+ mlflow.log_param('split', args.split)+ mlflow.log_param('max_error_task_retries', args.max_error_task_retries)+ experiment.run(+ model_ref = model_ref,+ system = system,+ benchmark = benchmark,+ out_url = args.out_url,+ subsample = args.subsample,+ redo_eval = args.redo_eval,+ run_id = args.run_id,+ split = args.split,+ processes = args.processes,+ callbacks = [Callback()],+ eval_only = args.eval_only,+ max_error_task_retries = args.max_error_task_retries)+++if __name__ == "__main__":+ main()\ No newline at end of filediff --git a/webeval/scripts/test_trajectory_loading.py b/webeval/scripts/test_trajectory_loading.pynew file mode 100644index 0000000..dad8fcd--- /dev/null+++ b/webeval/scripts/test_trajectory_loading.py@@ -0,0 +1,82 @@+#!/usr/bin/env python3+"""+Test script to verify Trajectory class can load text-based log files.+"""++import sys+from pathlib import Path++# Add the src directory to path+sys.path.insert(0, '/mnt/users/wangzhe/code/fara/webeval/src')++from webeval.trajectory import Trajectory++def main():+ traj_path = Path('/data/data/Fara/eval/runs/WebSurfer-fara-100-max_n_images-3/model_checkpoints/corbyrosset/WebVoyager_WebVoyager_data_08312025.jsonl/Nov252025/traj/Allrecipes--0/')++ print("="*80)+ print("Testing Trajectory Loading")+ print("="*80)+ print(f"\nTrajectory path: {traj_path}")+ print(f"Path exists: {traj_path.exists()}\n")++ try:+ # Load the trajectory+ print("Loading trajectory...")+ traj = Trajectory(traj_path)++ print("\n" + "="*80)+ print("✓ Trajectory loaded successfully!")+ print("="*80)++ # Display summary+ print(f"\nTrajectory Summary:")+ print(f" - Representation: {repr(traj)}")+ print(f" - Number of events: {len(traj.events)}")+ print(f" - Number of actions: {len(traj.actions)}")+ print(f" - Number of thoughts: {len(traj.thoughts)}")+ print(f" - Number of screenshots: {len(traj.screenshots)}")+ print(f" - Is aborted: {traj.is_aborted}")++ # Display actions and thoughts+ print("\n" + "="*80)+ print("Actions and Thoughts")+ print("="*80)++ for i, (thought, action) in enumerate(zip(traj.thoughts, traj.actions), 1):+ print(f"\n{'─'*80}")+ print(f"Action {i}:")+ print(f"{'─'*80}")+ print(f"Thought: {thought}")+ print(f"Action: {action}")++ # Display answer information+ print("\n" + "="*80)+ print("Final Answer")+ print("="*80)+ print(f"Answer: {traj.answer.final_answer}")+ print(f"Is aborted: {traj.answer.is_aborted}")++ # Display token usage if available+ if traj.answer.token_usage:+ print("\n" + "="*80)+ print("Token Usage")+ print("="*80)+ for key, usage in traj.answer.token_usage.items():+ print(f" {key}: {usage}")++ print("\n" + "="*80)+ print("✓ Test completed successfully!")+ print("="*80)++ except Exception as e:+ print("\n" + "="*80)+ print("✗ Failed to load trajectory!")+ print("="*80)+ print(f"Error: {type(e).__name__}: {e}")+ import traceback+ traceback.print_exc()+ sys.exit(1)++if __name__ == "__main__":+ main()diff --git a/webeval/scripts/webvoyager.py b/webeval/scripts/webvoyager.pyindex f7146c4..2e0e9db 100644--- a/webeval/scripts/webvoyager.py+++ b/webeval/scripts/webvoyager.py@@ -12,7 +12,7 @@ from webeval.eval_result import EvalResult, Stagefrom arg_parsing import get_eval_args-DEFAULT_DATA_URL = '../data/WebVoyager_data_08312025.jsonl'+DEFAULT_DATA_URL = '../data/webvoyager/WebVoyager_data_08312025.jsonl'class Callback:def __init__(self):diff --git a/webeval/src/webeval/benchmarks/__init__.py b/webeval/src/webeval/benchmarks/__init__.pyindex 5a6bd58..04410ce 100644--- a/webeval/src/webeval/benchmarks/__init__.py+++ b/webeval/src/webeval/benchmarks/__init__.py@@ -4,7 +4,7 @@from .webvoyager.webvoyager import WebVoyagerBenchmark# from .mdm2w.mdm2w import MultimodalM2WBenchmark# from .aurora.aurora import AuroraBenchmark-# from .om2w.om2w import OnlineM2WBenchmark+from .om2w.om2w import OnlineM2WBenchmark# from .holdout.holdout import HoldoutBenchmark# from .deepshop.deepshop import DeepShopBenchmark# from .browsecomp.browsecomp import BrowseCompBenchmark@@ -23,7 +23,7 @@ __all__ = ["WebVoyagerBenchmark",# "MultimodalM2WBenchmark",# "AuroraBenchmark",- # "OnlineM2WBenchmark",+ "OnlineM2WBenchmark",# "HoldoutBenchmark",# "DeepShopBenchmark",# "BrowseCompBenchmark",diff --git a/webeval/src/webeval/benchmarks/om2w/__init__.py b/webeval/src/webeval/benchmarks/om2w/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/webeval/src/webeval/benchmarks/om2w/impl/.gitattributes b/webeval/src/webeval/benchmarks/om2w/impl/.gitattributesnew file mode 100644index 0000000..e69de29diff --git a/webeval/src/webeval/benchmarks/om2w/impl/.gitignore b/webeval/src/webeval/benchmarks/om2w/impl/.gitignorenew file mode 100644index 0000000..e69de29diff --git a/webeval/src/webeval/benchmarks/om2w/impl/README.md b/webeval/src/webeval/benchmarks/om2w/impl/README.mdnew file mode 100644index 0000000..776be96--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/README.md@@ -0,0 +1,228 @@+<h1 align="center"> Online-Mind2Web Benchmark </h1>++<p align="center">+ <a href="https://xuetianci.github.io/" style="text-decoration: none;">Tianci Xue<sup>,1</sup></a>,+ <a href="https://x.com/weijian_qi" style="text-decoration: none;">Weijian Qi<sup>*,1</sup></a>,+ <a href="https://tnshi.com/" style="text-decoration: none;">Tianneng Shi<sup>*2</sup></a>,+ <a href="https://chanh.ee/" style="text-decoration: none;">Chan Hee Song<sup>1</sup></a>,+ <a href="https://boyugou.github.io/" style="text-decoration: none;">Boyu Gou<sup>1</sup></a>,+ <a href="https://dawnsong.io/" style="text-decoration: none;">Dawn Song<sup>,2</sup></a>,+ <a href="https://u.osu.edu/ihudas/people/" style="text-decoration: none;">Huan Sun<sup>†,1</sup></a>+ <a href="https://ysu1989.github.io/" style="text-decoration: none;">Yu Su<sup>†,1</sup></a>+</p>++<p align="center">+ <sup>1</sup>The Ohio State University, <sup>2</sup>University of California, Berkeley </br>+ <sub><sup>*</sup>Equal contribution, <sup>†</sup>Equal advising</sub>+</p>++<p align="center">+<a href="https://arxiv.org/abs/2504.01382">📃 Paper</a>+•+<a href="https://tiancixue.notion.site/An-Illusion-of-Progress-Assessing-the-Current-State-of-Web-Agents-1ac6cd2b9aac80719cd6f68374aaf4b4?pvs=4">📃 Blog</a>+•+<a href="https://huggingface.co/spaces/osunlp/Online_Mind2Web_Leaderboard" >🏆 Leaderboard</a>+•+<a href="https://huggingface.co/datasets/osunlp/Online-Mind2Web" >🤗 Data</a>+</p>+++# Online-Mind2Web benchmark++## News+* [05/11/2025] Check out our updates in the [paper](https://arxiv.org/abs/2504.01382).+ * The performance of Claude Computer Use 3.7.+ * WebJudge(o4-mini) achieves high agreement (86%) with a low success rate gap (3.8%) compared with humans.+ * Release [WebJudge-7B](https://huggingface.co/osunlp/WebJudge-7B), a robust and reliable reward model for Reinforcement learning.++## Tasks+Online-Mind2Web includes 300 diverse tasks from 136 popular websites across various domains. It covers a diverse set of real-world user tasks, such as clothing, food, housing, and transportation, to evaluate web agents' performance in a real-world online environment.++## Update Tasks++We will regularly update Online-Mind2Web by replacing outdated or invalid tasks (e.g., due to website changes) to maintain its value as a rigorous benchmark for web agents. If you find any tasks are outdated, please reach out to us, and we will update them.++To ensure fair comparisons, we will aim to keep the updated tasks on the same websites as before and with a similar reference length. Additionally, once agent performance saturates on Online-Mind2Web, we will also revise simple tasks to preserve its long-term value.++# Automatic Evaluator via LLM-as-a-Judge (WebJudge)+To enhance the reliability and scalability of the evaluation process in online environments, We propose a more reliable automatic evaluation method called **WebJudge**, which consists of three parts. (1) Key Point Identification: The model is prompted to identify several key points necessary for completing the task, based on the given instruction and task description. (2) Key Screenshot Identification: Important screenshots are selected from the agent’s trajectory to retain relevant visual evidence while discarding uninformative frames. (3) Outcome Judgment: Output the judgement result based on the task description, key points, key screenshots, and the action history. Our method preserves critical intermediate screenshots while mitigating the token overload issue.++<p align="center">+ <img src="./images/WebJudge.jpg" width="100%" alt="pipeline">+</p>++# Results++## Comparison against Existing Evaluation Methods on Online-Mind2Web+<table>+<tr>+ <th>Model</th>+ <th>Auto-Eval</th>+ <td>SeeAct</td>+ <td>Agent-E</td>+ <td>Browser Use</td>+ <td>Claude 3.5 </td>+ <td>Claude 3.7</td>+ <td>Operator</td>+ <th>Avg AR</th>+</tr>+<tr>+ <th rowspan="4">GPT-4o</th>+ <td>Autonomous Eval</td>+ <td>84.7</td>+ <td>85.0</td>+ <td>76.0</td>+ <td>83.7</td>+ <td>75.5</td>+ <td>71.7</td>+ <td>79.4</td>+</tr>+<tr>+ <td>AgentTrek Eval</td>+ <td>73.0</td>+ <td>64.3</td>+ <td>63.3</td>+ <td>--</td>+ <td>--</td>+ <td>--</td>+ <td>66.9</td>+</tr>+<tr>+ <td>WebVoyager</td>+ <td>--</td>+ <td>75.3</td>+ <td>71.3</td>+ <td>74.0</td>+ <td>72.0</td>+ <td>76.7</td>+ <td>73.9</td>+</tr>+<tr>+ <td>WebJudge</td>+ <td>86.7</td>+ <td>86.0</td>+ <td>81.4</td>+ <td>86.3</td>+ <td>79.1</td>+ <td>81.8</td>+ <td><b>83.6</b></td>+</tr>++<tr>+ <th rowspan="3">o4-mini</th>+ <td>Autonomous Eval</td>+ <td>79.7</td>+ <td>85.7</td>+ <td>86.0</td>+ <td>84.3</td>+ <td>68.0</td>+ <td>73.3</td>+ <td>79.5</td>+</tr>+<tr>+ <td>WebVoyager</td>+ <td>--</td>+ <td>80.3</td>+ <td>79.0</td>+ <td>81.7</td>+ <td>74.3</td>+ <td>78.3</td>+ <td>78.7</td>+</tr>+<tr>+ <td>WebJudge</td>+ <td>85.3</td>+ <td>86.3</td>+ <td>89.3</td>+ <td>87.0</td>+ <td>82.3</td>+ <td>83.7</td>+ <td><b>85.7</b></td>+</tr>++<tr>+ <th></th>+ <td>WebJudge-7B</td>+ <td>86.0</td>+ <td>87.3</td>+ <td>88.3</td>+ <td>89.7</td>+ <td>84.3</td>+ <td>86.3</td>+ <td><b>87.0</b></td>+</tr>+</table>+WebJudge powered by GPT-4o and o4-mini consistently achieves the highest agreement, with averages of 83.6% and 85.7%, respectively. Meanwhile, WebJudge-7B even outperforms o4-mini, reaching a high agreement with human judgment of 87%.+++## Excellent generalization capabilities on [AgentRewardBench](https://agent-reward-bench.github.io/) (5 OOD benchmarks)+| **Methods** | **AB** | **VWA** | **WA** | **Work** | **Wk++** | **Overall** |+|--------------|--------|--------|--------|----------|----------|--------------|+| *Rule-based** | 25.0 | **85.2** | 79.0 | 100.0 | 83.3 | 83.8 |+| Autonomous Eval* | 83.3 | 61.2 | 67.6 | 96.4 | 59.3 | 67.6 |+| GPT-4o (A11y Tree)* | 77.8 | 63.0 | 70.2 | 94.6 | 63.0 | 69.8 |+| WebJudge (GPT-4o) | 66.7 | 69.8 | 72.6 | 92.3 | 75.0 | 73.7 |+| WebJudge-7B | 80.0 | 66.7 | 77.5 | 100.0 | 70.0 | 75.7 |+| WebJudge (o4-mini) | **100.0** | 74.5 | **81.2** | **100.0** | **90.0** | **82.0** |++WebJudge significantly outperforms existing methods, achieving impressive overall precision of 73.7% 75.7% and 82.0% on WebArena (WA), VisualWebArena (VWA), AssistantBench (AB), WorkArena (Work) and WorkArena++ (Wk++) across 1302 trajectories.++The high precision suggests that WebJudge holds potential as a robust and scalable reward model for downstream applications such as Rejection Sampling Fine-Tuning, Reflection, and Reinforcement Learning.++# Model Release+We have released the fine-tuned [WebJudge-7B](https://huggingface.co/osunlp/WebJudge-7B) weights, which are now available on Hugging Face.++# Setup Environment++Create a conda environment and install dependencies:+```+conda create -n Online_Mind2Web python=3.11+conda activate Online_Mind2Web+pip install -r requirements.txt+```++# Evaluation+You can run the provided example evaluation script directly to perform the evaluation. Adjust the "mode" parameter to choose among various auto-eval methods.+```bash+bash ./script/eval.sh+```++## Important Notes for Reliable Evaluation on Online-Mind2Web:+> [!IMPORTANT]+> - **Start from the specified websites, not Google Search**:To enable fair comparisons, please ensure that each task starts from the specified website in our benchmark. Starting from Google Search or alternative websites can lead agents to use different websites to solve the task, resulting in varying difficulty levels and potentially skewed evaluation results.+> - **Include only factual actions, not agent outputs**: The action history should contain only the factual actions taken by the agent to complete the task (e.g., clicking elements and Typing text). Do not include the final response or any other agent's outputs, as they may contain hallucinated content and result in a high rate of false positives.+> - **Use o4-mini for WebJudge**: WebJudge powered by o4-mini demonstrates a higher alignment with human judgment, achieving an average agreement rate of 85.7% and maintaining a narrow success rate gap of just 3.8%. Therefore, please use o4-mini as the backbone for automatic evaluation.++# Evaluation Results++In certain scenarios, testing on the full Online-Mind2Web dataset may not be feasible due to cost, privacy, or legal constraints. To facilitate fair and apple-to-apple comparisons, we release both our human evaluation labels and auto-eval details.++- **Human Evaluation**: Task-level human evaluation labels are provided in the [file](https://github.com/OSU-NLP-Group/Online-Mind2Web/tree/main/data/evaluation_results/online_mind2web_evaluation_results/human_label.json).+- **Auto-Evaluation**: The results of WebJudge are available in the [folder](https://github.com/OSU-NLP-Group/Online-Mind2Web/tree/main/data/evaluation_results/online_mind2web_evaluation_results).++## 📚 Citation++Note: Online-Mind2Web is derived from the original Mind2Web dataset. We kindly ask that you cite both the original and this work when using or referencing the data.+```+@article{xue2025illusionprogressassessingcurrent,+ title={An Illusion of Progress? Assessing the Current State of Web Agents},+ author={Tianci Xue and Weijian Qi and Tianneng Shi and Chan Hee Song and Boyu Gou and Dawn Song and Huan Sun and Yu Su},+ year={2025},+ eprint={2504.01382},+ archivePrefix={arXiv},+ primaryClass={cs.AI},+ url={https://arxiv.org/abs/2504.01382},+}++@inproceedings{deng2023mind2web,+ author = {Deng, Xiang and Gu, Yu and Zheng, Boyuan and Chen, Shijie and Stevens, Sam and Wang, Boshi and Sun, Huan and Su, Yu},+ booktitle = {Advances in Neural Information Processing Systems},+ editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},+ pages = {28091--28114},+ publisher = {Curran Associates, Inc.},+ title = {Mind2Web: Towards a Generalist Agent for the Web},+ url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf},+ volume = {36},+ year = {2023}+}+```diff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/README.md b/webeval/src/webeval/benchmarks/om2w/impl/data/README.mdnew file mode 100644index 0000000..25ade4a--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/data/README.md@@ -0,0 +1,27 @@+## Example++You can refer to the [example](https://github.com/OSU-NLP-Group/Online-Mind2Web/tree/main/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f) format when conducting auto-eval. For each task, the result should be stored in a folder named as its `task_id`, containing :++- `trajectory/`: Stores screenshots of each step.+- `result.json`: Task metadata and action history.+++## 🧠 Human Labels++The file `human_label.json` provides human evaluation labels for each task across different agents. The meaning of labels represent as following:++- `0`: ❌ **Failure** – The agent did **not** successfully complete the task.+- `1`: ✅ **Success** – The agent **successfully** completed the task.+- `2`: 🚫 **Not Executable** – The agent was **unable to execute** the task due to external or system-related limitations.++## ℹ️ Details on Label 2++The label 2 indicates that the agent could not execute the task at all. The specific reasons vary by agent:++- **SeeAct**: Some tasks raise internal bugs when the agent extracts elements from the web page.+- **Operator**: Certain websites (e.g., Reddit) are inaccessible, preventing task execution.+- **Claude Computer Use 3.7**: Due to updates on the websites, a few tasks became outdated during the testing phase..+- **Agent-E** and **Browser Use**: Label 2 typically corresponds to tasks blocked by **CAPTCHAs**. There are only **two** tasks for Agent-E and **one** task for Browser Use.++## Note+The results on the leaderboard are averaged over three runs, so there may be slight differences compared to the auto-eval results shown here.diff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/result.json b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/result.jsonnew file mode 100644index 0000000..59b5dbb--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/result.json@@ -0,0 +1,17 @@+{+ "task_id": "fb7b4f784cfde003e2548fdf4e8d6b4f",+ "task": "Open the page with an overview of the submission of releases on Discogs.",+ "final_result_response": "The page with an overview of submission guidelines for releases on Discogs is open: https://support.discogs.com/hc/en-us/articles/360004016474-Overview-Of-Submission-Guidelines-For-Releases",+ "action_history": [+ "<div role=\"button\"> -> CLICK",+ "<button> -> CLICK",+ "<button> -> CLICK",+ "<a href=\"https://support.discogs.com/hc/articles/360004016474-Overview-of-Submission-Guidelines-for-Releases\" role=\"menuitem\"> -> CLICK"+ ],+ "thoughts": [+ "Navigate to the section about submission of releases on Discogs.",+ "Accept cookies to ensure smooth navigation, then look for a link related to submissions or guidelines.",+ "Explore the 'Community' section to find information about submission of releases.",+ "Click on 'Database Guidelines' to find information about submission of releases."+ ]+}\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/0_full_screenshot.png b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/0_full_screenshot.pngnew file mode 100644index 0000000..456ac5eBinary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/0_full_screenshot.png differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/1_full_screenshot.png b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/1_full_screenshot.pngnew file mode 100644index 0000000..293a97dBinary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/1_full_screenshot.png differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/2_full_screenshot.png b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/2_full_screenshot.pngnew file mode 100644index 0000000..7b10c3aBinary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/2_full_screenshot.png differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/3_full_screenshot.png b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/3_full_screenshot.pngnew file mode 100644index 0000000..12029dfBinary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/3_full_screenshot.png differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/4_full_screenshot.png b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/4_full_screenshot.pngnew file mode 100644index 0000000..81f495cBinary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f/trajectory/4_full_screenshot.png differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/images/WebJudge.jpg b/webeval/src/webeval/benchmarks/om2w/impl/images/WebJudge.jpgnew file mode 100644index 0000000..d93f0b9Binary files /dev/null and b/webeval/src/webeval/benchmarks/om2w/impl/images/WebJudge.jpg differdiff --git a/webeval/src/webeval/benchmarks/om2w/impl/requirements.txt b/webeval/src/webeval/benchmarks/om2w/impl/requirements.txtnew file mode 100644index 0000000..17fbc1c--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/requirements.txt@@ -0,0 +1,3 @@+backoff==2.2.1+openai==1.68.2+Pillow==11.1.0diff --git a/webeval/src/webeval/benchmarks/om2w/impl/script/eval.sh b/webeval/src/webeval/benchmarks/om2w/impl/script/eval.shnew file mode 100644index 0000000..a66fc09--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/script/eval.sh@@ -0,0 +1,24 @@++api_key=API_KEY+model_name=MODEL_NAME++#Automatic evaluation method+modes=(+ "WebJudge_Online_Mind2Web_eval"+ "WebJudge_general_eval"+ "Autonomous_eval"+ "WebVoyager_eval"+ "AgentTrek_eval"+)++base_dir="./data/example"+for mode in "${modes[@]}"; do+ python ./src/run.py \+ --mode "$mode" \+ --model "${model_name}" \+ --trajectories_dir "$base_dir" \+ --api_key "${api_key}" \+ --output_path ${base_dir}_result \+ --num_worker 1 \+ --score_threshold 3+donediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/clean_html.py b/webeval/src/webeval/benchmarks/om2w/impl/src/clean_html.pynew file mode 100644index 0000000..0069694--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/clean_html.py@@ -0,0 +1,42 @@+from typing import Any, Iterable, List+from bs4 import BeautifulSoup++SALIENT_ATTRIBUTES = (+ "alt",+ "aria-describedby",+ "aria-label",+ "aria-role",+ "aria-controls",+ "input-checked",+ "label",+ "name",+ "option_selected",+ "placeholder",+ "readonly",+ "text-value",+ "title",+ "value",+ "data-gtm-label",+ "href",+ "role",+)++def process_element_tag(element: str, salient_attributes: Iterable[str]) -> str:+ """Clean an HTML element string, keeping only salient_attributes."""+ if not element.endswith(">"):+ element += "'>"++ soup = BeautifulSoup(element, "html.parser")+ for tag in soup.find_all(True):+ # Keep only salient attributes+ filtered_attrs = {k: tag.attrs[k] for k in tag.attrs if k in salient_attributes}+ name_val = filtered_attrs.pop("name", None)+ new_tag = soup.new_tag(tag.name, **filtered_attrs)+ if name_val:+ new_tag["name"] = name_val+ return str(new_tag).split(f"</{tag.name}>")[0]+ return element++if __name__ == "__main__":+ text = '<input type=\"text\" name=\"q\" id=\"mntl-search-form--open__search-input\" class=\"mntl-search-form__input\" placeholder=\"Find a recipe or ingredient\" required=\"required\" value=\"\" autocomplete=\"off\" style=\"\"> -> TYPE beef sirloin'+ print(process_element_tag(text, SALIENT_ATTRIBUTES))\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/methods/agenttrek_eval.py b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/agenttrek_eval.pynew file mode 100644index 0000000..0dcaa52--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/agenttrek_eval.py@@ -0,0 +1,56 @@+from webeval.benchmarks.om2w.impl.src.utils import encode_image+from PIL import Image++def AgentTrek_eval(task, last_actions, thoughts, images_path):+ system_msg = """You are an expert in evaluating the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. Given the user's task goal, the agent's trajectory, your goal is to decide whether the agent's execution is successful or not.++*Evaluation Criteria*+Whether the agent's trajectory is effective and corresponding to the goal++*Instructions* 1. Review the agent's actions and reasoning processes step by step.+2. if the agent is stuck in the very first login stage, which means it fails to log into target website at the beginning, that's a failure.+3. Determine if the agent has achieved the task goal based on the trajectory. A task can be considered successful if most trajectory is effective.+4. the agent sometimes can't stop after finishing a task and continue doing repeated actions. these actions may be some failed attempt after a series of correct actions. the task should be regarded as successful if the correct actions are effective and almost reach the goal.+5. if the agent is stuck in the loop at the early stage of the task, which means they don't even get close to the goal before they get stuck in the loop, that's a failure. for example, the agent begin to get stuck before third step.+6. when the task is to change the google account password, it can't be regarded as successful when agent finish at trying to click "manage your account".+7. if there are over 8 correct action in the trajectory, it can be regard as a successful agent.+8. final saving action is not a must. the task is successful if the agent does most things right and just forget to save the change at last.+9. if the original task has 2 subtasks, the agent only complete one of them, that's still a success. e.g. the task is to update name and birthday, but agent only update name, that's fine.+10. if the task is to post a review, the agent can be considered successful when it finish writing the review and reach the step to post it, don't have to click the post button.+11. Since we don't have a printer, some printing related task can be considered successful if the agent reach the step to click print button.+12. if the task is finished at the initial state and the agent do nothing because of it, it should also be regarded as successful.++*IMPORTANT*+1. in the trajectory, an action always follows a corresponding reasoning, which shows the observation and thought of the agent.+2. your response should be contain:+Thoughts: <your thoughts and reasoning process>+Status: "success" or "failure"+"""+ prompt = """The goal of the task: {task}++Trajectory:+{thoughts_and_actions}++The last snapshot of the web page is shown in the image."""+ thoughts_and_actions = ""+ for idx, (thought, action) in enumerate(zip(thoughts, last_actions)):+ thought = thought.replace("\n\n", " ")+ action = action.replace("\n\n", " ")+ thoughts_and_actions += f"Thought {idx+1}: {thought}\nAction {idx+1}: {action}\n\n"+ text = prompt.format(task=task, thoughts_and_actions=thoughts_and_actions.strip("\n\n"))++ jpg_base64_str = encode_image(Image.open(images_path))+ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{jpg_base64_str}", "detail": "high"},+ },+ ],+ }+ ]+ return messages, text, system_msg\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/methods/automomous_eval.py b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/automomous_eval.pynew file mode 100644index 0000000..a7e9b4f--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/automomous_eval.py@@ -0,0 +1,41 @@+from webeval.benchmarks.om2w.impl.src.utils import encode_image+from PIL import Image++def Autonomous_eval(task, last_actions, images_path):+ system_msg = """You are an expert in evaluating the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. Given the user's intent, the agent's action history, the final state of the webpage, and the agent's response to the user, your goal is to decide whether the agent's execution is successful or not.++There are three types of tasks:+1. Information seeking: The user wants to obtain certain information from the webpage, such as the information of a product, reviews, map info, comparison of map routes, etc. The bot's response must contain the information the user wants, or explicitly state that the information is not available. Otherwise, e.g. the bot encounters an exception and respond with the error content, the task is considered a failure. Besides, be careful about the sufficiency of the agent's actions. For example, when asked to list the top-searched items in a shop, the agent should order the items by the number of searches, and then return the top items. If the ordering action is missing, the task is likely to fail.+2. Site navigation: The user wants to navigate to a specific page. Carefully examine the bot's action history and the final state of the webpage to determine whether the bot successfully completes the task. No need to consider the bot's response.+3. Content modification: The user wants to modify the content of a webpage or configuration. Carefully examine the bot's action history and the final state of the webpage to determine whether the bot successfully completes the task. No need to consider the bot's response.++*IMPORTANT*+Format your response into two lines as shown below:++Thoughts: <your thoughts and reasoning process>+Status: "success" or "failure"+"""+ prompt = """User Intent: {task}++Action History:+{last_actions}++The last snapshot of the web page is shown in the image."""++ text = prompt.format(task=task, last_actions="\n".join(f"{i+1}. {action}" for i, action in enumerate(last_actions)))++ jpg_base64_str = encode_image(Image.open(images_path))+ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{jpg_base64_str}", "detail": "high"},+ },+ ],+ }+ ]+ return messages, text, system_msg\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_general_eval.py b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_general_eval.pynew file mode 100644index 0000000..734277c--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_general_eval.py@@ -0,0 +1,225 @@+from webeval.benchmarks.om2w.impl.src.utils import encode_image+from PIL import Image+import re+import asyncio+MAX_IMAGE =50++async def identify_key_points(task, input_image_paths, model):+ system_msg = """You are an expert tasked with analyzing a given task to identify the key points explicitly stated in the task description.++**Objective**: Carefully analyze the task description and extract the critical elements explicitly mentioned in the task for achieving its goal.++**Instructions**:+1. Read the task description carefully.+2. Identify and extract **key points** directly stated in the task description.+ - A **key point** is a critical element, condition, or step explicitly mentioned in the task description.+ - Do not infer or add any unstated elements.+ - Words such as "best," "highest," "cheapest," "latest," "most recent," "lowest," "closest," "highest-rated," "largest," and "newest" must go through the sort function(e.g., the key point should be "Filter by highest").++**Respond with**:+- **Key Points**: A numbered list of the explicit key points for completing this task, one per line, without explanations or additional details."""++ prompt = """Task: {task}"""+ text = prompt.format(task=task)++ input_images_msg = []++ if input_image_paths != None:+ for input_image_path in input_image_paths:+ input_images_jpg_base64_str = encode_image(Image.open(input_image_path))+ input_images_msg.append(+ {+ 'type': 'image_url',+ 'image_url': {"url": f"data:image/png;base64,{input_images_jpg_base64_str}", "detail": "high"}+ }+ )++ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text}+ ]+ input_images_msg,+ }+ ]+ responses = await model.agenerate(messages)+ return responses[0]++async def judge_image(task, input_image_paths, image_path, key_points, model):+ system_msg = """You are an expert evaluator tasked with determining whether an image contains information about the necessary steps to complete a task.++**Objective**: Analyze the provided image and decide if it shows essential steps or evidence required for completing the task. Use your reasoning to explain your decision before assigning a score.++**Instructions**:+1. Provide a detailed description of the image, including its contents, visible elements, text (if any), and any notable features.++2. Carefully examine the image and evaluate whether it contains necessary steps or evidence crucial to task completion:+- Identify key points that could be relevant to task completion, such as actions, progress indicators, tool usage, applied filters, or step-by-step instructions.+- Does the image show actions, progress indicators, or critical information directly related to completing the task?+- Is this information indispensable for understanding or ensuring task success?+- If the image contains partial but relevant information, consider its usefulness rather than dismissing it outright.++3. Provide your response in the following format:+- **Reasoning**: Explain your thought process and observations. Mention specific elements in the image that indicate necessary steps, evidence, or lack thereof.+- **Score**: Assign a score based on the reasoning, using the following scale:+ - **1**: The image does not contain any necessary steps or relevant information.+ - **2**: The image contains minimal or ambiguous information, unlikely to be essential.+ - **3**: The image includes some relevant steps or hints but lacks clarity or completeness.+ - **4**: The image contains important steps or evidence that are highly relevant but not fully comprehensive.+ - **5**: The image clearly displays necessary steps or evidence crucial for completing the task.++Respond with:+### Reasoning**: [Your explanation]+### Score**: [1-5]"""+++ prompt = """**Task**: {task}++**Key Points for Task Completion**: {key_points}++The snapshot of the web page is shown in the image."""+ text = prompt.format(task=task,key_points=key_points)++ input_images_msg = []+ if input_image_paths != None:+ for input_image_path in input_image_paths:+ input_images_jpg_base64_str = encode_image(Image.open(input_image_path))+ input_images_msg.append(+ {+ 'type': 'image_url',+ 'image_url': {"url": f"data:image/png;base64,{input_images_jpg_base64_str}", "detail": "high"}+ }+ )+ messages = [{"role": "system", "content": system_msg}]++ if input_images_msg:+ messages.append({+ "role": "user",+ "content": [{"type": "text", "text": "The input images are:"}] + input_images_msg+ })++ jpg_base64_str = encode_image(Image.open(image_path))+ messages.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{jpg_base64_str}", "detail": "high"},+ },+ ]+ }+ )++ responses = await model.agenerate(messages)+ return responses[0]+++async def WebJudge_general_eval(task, input_image_paths, action_thoughts, last_actions, images_path, model, score_threshold):+ system_msg = """You are an expert in evaluating the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. Given the user's task, the agent's action history, key points for task completion, some potentially important web pages in the agent's trajectory and their reasons, your goal is to determine whether the agent has completed the task and achieved all requirements.++Your response must strictly follow the following evaluation criteria!+*Important Evaluation Criteria*:+1: The filtered results must be displayed correctly. If filters were not properly applied (i.e., missing selection, missing confirmation, or no visible effect in results), it should be considered a failure.+2: You must carefully check whether these snapshots and action history meet these key points. Ensure that specific filter conditions, such as "best," "highest," "cheapest," "latest," "most recent," "lowest," "closest," "highest-rated," "largest," and "newest" are correctly applied using the filter function(e.g., sort function).+3: Certain key points or requirements should be applied by the filter. Otherwise, a search with all requirements as input will be deemed a failure since it cannot guarantee that all results meet the requirements!+4: If the task requires filtering by a specific range of money, years, or the number of beds and bathrooms, the applied filter must exactly match the given requirement. Any deviation results in failure. To ensure the task is successful, the applied filter must precisely match the specified range without being too broad or too narrow.+5: Some tasks require a submission action or a display of results to be considered successful. Repeat actions or actions that do not lead to a visible result should be considered a failure.+6: If the agent loops through a sequence of actions that do not make progress toward the goal (including failing to click "Save" or "Submit," etc.), it should be considered a failure.++Format your response into two lines as shown below:+Thoughts: <your thoughts and reasoning process should base on double-checking each key points and the evaluation criteria>+Status: "success" or "failure"+"""+ prompt = """User Task: {task}++Key Points: {key_points}++Action History:+{last_actions}++The potentially important snapshots of the webpage in the agent's trajectory and their reasons:+{thoughts}"""+++ key_points = await identify_key_points(task, input_image_paths, model)+ key_points = key_points.replace("\n\n", "\n")++ try:+ key_points = key_points.split("**Key Points**:")[1]+ key_points = "\n".join(line.lstrip() for line in key_points.splitlines())+ except:+ key_points = key_points.split("Key Points:")[-1]+ key_points = "\n".join(line.lstrip() for line in key_points.splitlines())++ tasks = [judge_image(task, input_image_paths, image_path, key_points, model) for image_path in images_path]+ image_responses = await asyncio.gather(*tasks)++ input_images_msg = []+ whole_content_img = []+ whole_thoughts = []+ record = []+ pattern = r"[1-5]"+ for response, image_path in zip(image_responses, images_path):+ try:+ score_text = response.split("### Score")[1]+ thought = response.split("### Reasoning:")[-1].strip().lstrip("\n").split("### Score")[0].replace('\n',' ')+ score = re.findall(pattern, score_text)[0]+ record.append({"Response": response, "Score": int(score)})+ except Exception as e:+ print(f"Error processing response: {e}")+ score = 0+ record.append({"Response": response, "Score": 0})++ if int(score) >= score_threshold:+ jpg_base64_str = encode_image(Image.open(image_path))+ whole_content_img.append(+ {+ 'type': 'image_url',+ 'image_url': {"url": f"data:image/png;base64,{jpg_base64_str}", "detail": "high"}+ }+ )+ if thought != "":+ whole_thoughts.append(thought)++ whole_content_img = whole_content_img[:MAX_IMAGE]+ whole_thoughts = whole_thoughts[:MAX_IMAGE]+ if len(whole_content_img) == 0:+ prompt = """User Task: {task}++Key Points: {key_points}++Action History:+{last_actions}"""++ if action_thoughts != None:+ text = prompt.format(task=task, last_actions="\n".join(f"{i+1}. {action}. Reasoning: {action_thought}" for i, (action, action_thought) in enumerate(zip(last_actions,action_thoughts))), key_points=key_points, thoughts = "\n".join(f"{i+1}. {thought}" for i, thought in enumerate(whole_thoughts)))++ else:+ text = prompt.format(task=task, last_actions="\n".join(f"{i+1}. {action}" for i, action in enumerate(last_actions)), key_points=key_points, thoughts = "\n".join(f"{i+1}. {thought}" for i, thought in enumerate(whole_thoughts)))++ input_images_msg = []+ if input_image_paths is not None:+ for path in input_image_paths:+ input_images_jpg_base64_str = encode_image(Image.open(path))+ input_images_msg.append({+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{input_images_jpg_base64_str}", "detail": "high"}+ })++ messages = [{"role": "system", "content": system_msg}]++ if input_images_msg:+ messages.append({+ "role": "user",+ "content": [{"type": "text", "text": "The input images are:"}] + input_images_msg+ })++ messages.append({+ "role": "user",+ "content": [{"type": "text", "text": text}] + whole_content_img+ })++ return messages, text, system_msg, record, key_points\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_online_mind2web.py b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_online_mind2web.pynew file mode 100644index 0000000..a40c8ae--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webjudge_online_mind2web.py@@ -0,0 +1,184 @@+from webeval.benchmarks.om2w.impl.src.utils import encode_image+from PIL import Image+import re+import asyncio+MAX_IMAGE =50++async def identify_key_points(task, model):+ system_msg = """You are an expert tasked with analyzing a given task to identify the key points explicitly stated in the task description.++**Objective**: Carefully analyze the task description and extract the critical elements explicitly mentioned in the task for achieving its goal.++**Instructions**:+1. Read the task description carefully.+2. Identify and extract **key points** directly stated in the task description.+ - A **key point** is a critical element, condition, or step explicitly mentioned in the task description.+ - Do not infer or add any unstated elements.+ - Words such as "best," "highest," "cheapest," "latest," "most recent," "lowest," "closest," "highest-rated," "largest," and "newest" must go through the sort function(e.g., the key point should be "Filter by highest").++**Respond with**:+- **Key Points**: A numbered list of the explicit key points for completing this task, one per line, without explanations or additional details."""+ prompt = """Task: {task}"""+ text = prompt.format(task=task)+ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text}+ ],+ }+ ]+ responses = await model.agenerate(messages)+ return responses[0]++async def judge_image(task, image_path, key_points, model):+ system_msg = """You are an expert evaluator tasked with determining whether an image contains information about the necessary steps to complete a task.++**Objective**: Analyze the provided image and decide if it shows essential steps or evidence required for completing the task. Use your reasoning to explain your decision before assigning a score.++**Instructions**:+1. Provide a detailed description of the image, including its contents, visible elements, text (if any), and any notable features.++2. Carefully examine the image and evaluate whether it contains necessary steps or evidence crucial to task completion:+- Identify key points that could be relevant to task completion, such as actions, progress indicators, tool usage, applied filters, or step-by-step instructions.+- Does the image show actions, progress indicators, or critical information directly related to completing the task?+- Is this information indispensable for understanding or ensuring task success?+- If the image contains partial but relevant information, consider its usefulness rather than dismissing it outright.++3. Provide your response in the following format:+- **Reasoning**: Explain your thought process and observations. Mention specific elements in the image that indicate necessary steps, evidence, or lack thereof.+- **Score**: Assign a score based on the reasoning, using the following scale:+ - **1**: The image does not contain any necessary steps or relevant information.+ - **2**: The image contains minimal or ambiguous information, unlikely to be essential.+ - **3**: The image includes some relevant steps or hints but lacks clarity or completeness.+ - **4**: The image contains important steps or evidence that are highly relevant but not fully comprehensive.+ - **5**: The image clearly displays necessary steps or evidence crucial for completing the task.++Respond with:+1. **Reasoning**: [Your explanation]+2. **Score**: [1-5]"""++ jpg_base64_str = encode_image(Image.open(image_path))++ prompt = """**Task**: {task}++**Key Points for Task Completion**: {key_points}++The snapshot of the web page is shown in the image."""+ text = prompt.format(task=task,key_points=key_points)++ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{jpg_base64_str}", "detail": "high"},+ },+ ],+ }+ ]++ responses = await model.agenerate(messages)+ return responses[0]++async def WebJudge_Online_Mind2Web_eval(task, last_actions, images_path, model, score_threshold):+ system_msg = """You are an expert in evaluating the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. Given the user's task, the agent's action history, key points for task completion, some potentially important web pages in the agent's trajectory and their reasons, your goal is to determine whether the agent has completed the task and achieved all requirements.++Your response must strictly follow the following evaluation criteria!+*Important Evaluation Criteria*:+1: The filtered results must be displayed correctly. If filters were not properly applied (i.e., missing selection, missing confirmation, or no visible effect in results), the task is not considered successful.+2: You must carefully check whether these snapshots and action history meet these key points. Ensure that specific filter conditions, such as "best," "highest," "cheapest," "latest," "most recent," "lowest," "closest," "highest-rated," "largest," and "newest" are correctly applied using the filter function(e.g., sort function).+3: Certain key points or requirements should be applied by the filter. Otherwise, a search with all requirements as input will be deemed a failure since it cannot guarantee that all results meet the requirements!+4: If the task requires filtering by a specific range of money, years, or the number of beds and bathrooms, the applied filter must exactly match the given requirement. Any deviation results in failure. To ensure the task is successful, the applied filter must precisely match the specified range without being too broad or too narrow.+Examples of Failure Cases:+- If the requirement is less than $50, but the applied filter is less than $25, it is a failure.+- If the requirement is $1500-$2500, but the applied filter is $2000-$2500, it is a failure.+- If the requirement is $25-$200, but the applied filter is $0-$200, it is a failure.+- If the required years are 2004-2012, but the filter applied is 2001-2012, it is a failure.+- If the required years are before 2015, but the applied filter is 2000-2014, it is a failure.+- If the task requires exactly 2 beds, but the filter applied is 2+ beds, it is a failure.+5: Some tasks require a submission action or a display of results to be considered successful.+6: If the retrieved information is invalid or empty(e.g., No match was found), but the agent has correctly performed the required action, it should still be considered successful.+7: If the current page already displays all available items, then applying a filter is not necessary. As long as the agent selects items that meet the requirements (e.g., the cheapest or lowest price), the task is still considered successful.++*IMPORTANT*+Format your response into two lines as shown below:++Thoughts: <your thoughts and reasoning process based on double-checking each key points and the evaluation criteria>+Status: "success" or "failure"+"""+ prompt = """User Task: {task}++Key Points: {key_points}++Action History:+{last_actions}++The potentially important snapshots of the webpage in the agent's trajectory and their reasons:+{thoughts}"""+++ key_points = await identify_key_points(task, model)+ key_points = key_points.replace("\n\n", "\n")++ try:+ key_points = key_points.split("**Key Points**:")[1]+ key_points = "\n".join(line.lstrip() for line in key_points.splitlines())+ except:+ key_points = key_points.split("Key Points:")[-1]+ key_points = "\n".join(line.lstrip() for line in key_points.splitlines())++ tasks = [judge_image(task, image_path, key_points, model) for image_path in images_path]+ image_responses = await asyncio.gather(*tasks)++ whole_content_img = []+ whole_thoughts = []+ record = []+ pattern = r"[1-5]"+ for response, image_path in zip(image_responses, images_path):+ try:+ score_text = response.split("Score")[1]+ thought = response.split("**Reasoning**:")[-1].strip().lstrip("\n").split("\n\n")[0].replace('\n',' ')+ score = re.findall(pattern, score_text)[0]+ record.append({"Response": response, "Score": int(score)})+ except Exception as e:+ print(f"Error processing response: {e}")+ score = 0+ record.append({"Response": response, "Score": 0})++ if int(score) >= score_threshold:+ jpg_base64_str = encode_image(Image.open(image_path))+ whole_content_img.append(+ {+ 'type': 'image_url',+ 'image_url': {"url": f"data:image/png;base64,{jpg_base64_str}", "detail": "high"}+ }+ )+ if thought != "":+ whole_thoughts.append(thought)++ whole_content_img = whole_content_img[:MAX_IMAGE]+ whole_thoughts = whole_thoughts[:MAX_IMAGE]+ if len(whole_content_img) == 0:+ prompt = """User Task: {task}++Key Points: {key_points}++Action History:+{last_actions}"""+ text = prompt.format(task=task, last_actions="\n".join(f"{i+1}. {action}" for i, action in enumerate(last_actions)), key_points=key_points, thoughts = "\n".join(f"{i+1}. {thought}" for i, thought in enumerate(whole_thoughts)))++ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [+ {"type": "text", "text": text}]+ + whole_content_img+ }+ ]+ return messages, text, system_msg, record, key_points\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webvoyager_eval.py b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webvoyager_eval.pynew file mode 100644index 0000000..11080ef--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/methods/webvoyager_eval.py@@ -0,0 +1,49 @@+from webeval.benchmarks.om2w.impl.src.utils import encode_image+from PIL import Image+MAX_IMAGE =50++def WebVoyager_eval(task, images_path, response, k=0):+ system_msg = """As an evaluator, you will be presented with three primary components to assist you in your role:++1. Web Task Instruction: This is a clear and specific directive provided in natural language, detailing the online activity to be carried out. These requirements may include conducting searches, verifying information, comparing prices, checking availability, or any other action relevant to the specified web service (such as Amazon, Apple, ArXiv, BBC News, Booking etc).++2. Result Screenshots: This is a visual representation of the screen showing the result or intermediate state of performing a web task. It serves as visual proof of the actions taken in response to the instruction.++3. Result Response: This is a textual response obtained after the execution of the web task. It serves as textual result in response to the instruction.++-- You DO NOT NEED to interact with web pages or perform actions such as booking flights or conducting searches on websites.+-- You SHOULD NOT make assumptions based on information not presented in the screenshot when comparing it to the instructions.+-- Your primary responsibility is to conduct a thorough assessment of the web task instruction against the outcome depicted in the screenshot and in the response, evaluating whether the actions taken align with the given instructions.+-- NOTE that the instruction may involve more than one task, for example, locating the garage and summarizing the review. Failing to complete either task, such as not providing a summary, should be considered unsuccessful.+-- NOTE that the screenshot is authentic, but the response provided by LLM is generated at the end of web browsing, and there may be discrepancies between the text and the screenshots.+-- Note the difference: 1) Result response may contradict the screenshot, then the content of the screenshot prevails, 2) The content in the Result response is not mentioned on the screenshot, choose to believe the content.++You should elaborate on how you arrived at your final evaluation and then provide a definitive verdict on whether the task has been successfully accomplished, either as 'SUCCESS' or 'FAILURE'."""+ prompt = """TASK: {task}++Result Response: {response}++{num} screenshots at the end: """++ whole_content_img = []+ images_path = images_path[:MAX_IMAGE]+ text = prompt.format(task=task, response=response, num = len(images_path) if k == 0 else k)++ for image in images_path[-k:]:+ jpg_base64_str = encode_image(Image.open(image))+ whole_content_img.append(+ {+ 'type': 'image_url',+ 'image_url': {"url": f"data:image/png;base64,{jpg_base64_str}", "detail": "high"}+ }+ )+ messages = [+ {"role": "system", "content": system_msg},+ {+ "role": "user",+ "content": [{"type": "text", "text": text}]+ + whole_content_img+ + [{'type': 'text', 'text': "Your verdict:\n"}]+ }+ ]+ return messages, text, system_msg\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/run.py b/webeval/src/webeval/benchmarks/om2w/impl/src/run.pynew file mode 100644index 0000000..33d567e--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/run.py@@ -0,0 +1,162 @@+import argparse+import os+from methods.agenttrek_eval import *+from methods.automomous_eval import *+from methods.webjudge_general_eval import *+from methods.webjudge_online_mind2web import *+from methods.webvoyager_eval import *+from utils import OpenaiEngine, extract_predication+import json+import copy+import asyncio+import multiprocessing++def auto_eval(args, task_subset, final_predicted_labels, lock, model):++ ################## get the already done task id ###############+ output_json_path = os.path.join(args.output_path, f"{args.mode}_{args.model}_score_threshold_{args.score_threshold}_auto_eval_results.json")+ already_ids = []+ if os.path.exists(output_json_path):+ with open(output_json_path,"r") as f:+ already_data = f.read()+ already_tasks = already_data.splitlines()+ for item in already_tasks:+ item = json.loads(item)+ already_ids.append(item["task_id"])++ print(f"The number of already done tasks: {len(already_ids)}")++ for task_id in task_subset:+ #Skip already done task+ if task_id in already_ids:+ continue++ trajectory_images_path = os.path.join(args.trajectories_dir, task_id, "trajectory")+ screenshot_paths = []+ thoughts = None+ action_history = None+ final_result_response = None+ input_image_paths = None+ task_description = None+ # Load results+ with open(os.path.join(args.trajectories_dir, task_id, "result.json")) as f:+ result = json.load(f)+ output_results = copy.deepcopy(result)+ task_description = result["task"]+ if "action_history" in result:+ action_history = result["action_history"]+ if "thoughts" in result:+ thoughts = result["thoughts"]+ if "final_result_response" in result:+ final_result_response = result["final_result_response"]+ if "input_image_paths" in result:+ input_image_paths = result["input_image_paths"]++ print(f"Start evaluation for {task_description}")+ # Do the auto-eval+ if args.mode == "Autonomous_eval":+ for image in sorted(os.listdir(trajectory_images_path), key=lambda x: int(re.findall(r'\d+', x)[0])):+ screenshot_paths.append(os.path.join(trajectory_images_path, image))+ messages, text, system_msg = Autonomous_eval(task_description, action_history, screenshot_paths[-1])++ elif args.mode == "AgentTrek_eval":+ for image in sorted(os.listdir(trajectory_images_path), key=lambda x: int(re.findall(r'\d+', x)[0])):+ screenshot_paths.append(os.path.join(trajectory_images_path, image))+ messages, text, system_msg = AgentTrek_eval(task_description, action_history, thoughts, screenshot_paths[-1])++ elif args.mode == "WebVoyager_eval":+ for image in sorted(os.listdir(trajectory_images_path), key=lambda x: int(re.findall(r'\d+', x)[0])):+ screenshot_paths.append(os.path.join(trajectory_images_path, image))+ messages, text, system_msg = WebVoyager_eval(task_description, screenshot_paths, final_result_response)++ elif args.mode == "WebJudge_Online_Mind2Web_eval":+ for image in sorted(os.listdir(trajectory_images_path), key=lambda x: int(re.findall(r'\d+', x)[0])):+ screenshot_paths.append(os.path.join(trajectory_images_path, image))+ messages, text, system_msg, record, key_points = asyncio.run(WebJudge_Online_Mind2Web_eval(task_description, action_history, screenshot_paths, model, args.score_threshold))+ output_results["image_judge_record"] = record+ output_results["key_points"] = key_points++ elif args.mode == "WebJudge_general_eval":+ for image in sorted(os.listdir(trajectory_images_path), key=lambda x: int(re.findall(r'\d+', x)[0])):+ screenshot_paths.append(os.path.join(trajectory_images_path, image))+ messages, text, system_msg, record, key_points = asyncio.run(WebJudge_general_eval(task_description, input_image_paths, thoughts, action_history, screenshot_paths, model, args.score_threshold))+ output_results["image_judge_record"] = record+ output_results["key_points"] = key_points++ else:+ raise ValueError(f"Unknown mode: {args.mode}")++ response = model.generate(messages)[0]+ predicted_label = extract_predication(response, args.mode)++ #Store evaluation details+ evaluation_results = {"response": response, "predicted_label": predicted_label}+ output_results["task_id"] = task_id+ output_results["input_text"] = text+ output_results["system_msg"] = system_msg+ output_results["evaluation_details"] = evaluation_results+ output_results["predicted_label"] = predicted_label++ with lock:+ final_predicted_labels.append(predicted_label)++ print(f"Finish evaluation for {task_description}")+ print("="*20)+ os.makedirs(args.output_path, exist_ok=True)+ with lock:+ with open(os.path.join(args.output_path, f"{args.mode}_{args.model}_score_threshold_{args.score_threshold}_auto_eval_results.json"), "a+") as f_out:+ f_out.write(json.dumps(output_results) + "\n")+++def process_subset(task_subset, args, final_predicted_labels, lock, model):++ auto_eval(args, task_subset, final_predicted_labels, lock, model)++def parallel_eval(args, num_workers=60):++ #Evaluate in parallel based on num of works+ task_dirs = [+ d for d in sorted(os.listdir(args.trajectories_dir))+ if os.path.isdir(os.path.join(args.trajectories_dir, d))+ ]+ print(f"Evaluating {len(task_dirs)} tasks in total.")+ chunk_size = len(task_dirs) // num_workers+ task_subsets = [task_dirs[i:i + chunk_size] for i in range(0, len(task_dirs), chunk_size)]++ #Load model+ model = OpenaiEngine(+ model=args.model,+ api_key=args.api_key+ )++ lock = multiprocessing.Lock()+ with multiprocessing.Manager() as manager:+ final_predicted_labels = manager.list()+ processes = []+ for subset in task_subsets:+ p = multiprocessing.Process(target=process_subset, args=(subset, args, final_predicted_labels, lock, model))+ p.start()+ processes.append(p)++ for p in processes:+ p.join()++ success_num = sum(final_predicted_labels)++ print("Evaluation complete.")+ print(f"The success rate is {(success_num / len(task_dirs)) * 100}.")+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Auto evaluation of web navigation tasks.")+ parser.add_argument('--mode', type=str, default='Online_Mind2Web_eval', help='the mode of evaluation')+ parser.add_argument('--model', type=str, default='gpt-4o')+ parser.add_argument("--trajectories_dir", type=str, required=True, help="Path to trajectories directory")+ parser.add_argument("--api_key", type=str, required=True, help="The api key")+ parser.add_argument("--output_path", type=str, required=True, help="The output path")+ parser.add_argument('--score_threshold', type=int, default=3)+ parser.add_argument('--num_worker', type=int, default=60)+ args = parser.parse_args()++ parallel_eval(args, args.num_worker)+diff --git a/webeval/src/webeval/benchmarks/om2w/impl/src/utils.py b/webeval/src/webeval/benchmarks/om2w/impl/src/utils.pynew file mode 100644index 0000000..f3db59d--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/impl/src/utils.py@@ -0,0 +1,125 @@+import base64+import io+from openai import (+ APIConnectionError,+ APIError,+ RateLimitError,+ AzureOpenAI,+ OpenAI+)+import os+import backoff++def encode_image(image):+ """Convert a PIL image to base64 string."""+ if image.mode == "RGBA":+ image = image.convert("RGB")+ buffered = io.BytesIO()+ image.save(buffered, format="JPEG")+ return base64.b64encode(buffered.getvalue()).decode('utf-8')++def extract_predication(response, mode):+ """Extract the prediction from the response."""+ if mode == "Autonomous_eval":+ try:+ if "success" in response.lower().split('status:')[1]:+ return 1+ else:+ return 0+ except:+ return 0+ elif mode == "AgentTrek_eval":+ try:+ if "success" in response.lower().split('status:')[1]:+ return 1+ else:+ return 0+ except:+ return 0+ elif mode == "WebVoyager_eval":+ if "FAILURE" in response:+ return 0+ else:+ return 1+ elif mode == "WebJudge_Online_Mind2Web_eval":+ try:+ if "success" in response.lower().split('status:')[1]:+ return 1+ else:+ return 0+ except:+ return 0+ elif mode == "WebJudge_general_eval":+ try:+ if "success" in response.lower().split('status:')[1]:+ return 1+ else:+ return 0+ except:+ return 0+ else:+ raise ValueError(f"Unknown mode: {mode}")+++class OpenaiEngine():+ def __init__(+ self,+ api_key=None,+ stop=[],+ rate_limit=-1,+ model=None,+ tokenizer=None,+ temperature=0,+ port=-1,+ endpoint_target_uri = "",+ **kwargs,+ ) -> None:+ """Init an OpenAI GPT/Codex engine++ Args:+ api_key (_type_, optional): Auth key from OpenAI. Defaults to None.+ stop (list, optional): Tokens indicate stop of sequence. Defaults to ["\n"].+ rate_limit (int, optional): Max number of requests per minute. Defaults to -1.+ model (_type_, optional): Model family. Defaults to None.+ """+ assert (+ os.getenv("OPENAI_API_KEY", api_key) is not None+ ), "must pass on the api_key or set OPENAI_API_KEY in the environment"+ if api_key is None:+ api_key = os.getenv("OPENAI_API_KEY", api_key)+ if isinstance(api_key, str):+ self.api_keys = [api_key]+ elif isinstance(api_key, list):+ self.api_keys = api_key+ else:+ raise ValueError("api_key must be a string or list")+ self.stop = stop+ self.temperature = temperature+ self.model = model+ # convert rate limit to minmum request interval+ self.request_interval = 0 if rate_limit == -1 else 60.0 / rate_limit+ self.next_avil_time = [0] * len(self.api_keys)+ self.client = OpenAI(+ api_key=api_key,+ )++ def log_error(details):+ print(f"Retrying in {details['wait']:0.1f} seconds due to {details['exception']}")++ @backoff.on_exception(+ backoff.expo,+ (APIError, RateLimitError, APIConnectionError),+ max_tries=3,+ on_backoff=log_error+ )+ def generate(self, messages, max_new_tokens=512, temperature=0, model=None, **kwargs):+ model = model if model else self.model+ response = self.client.chat.completions.create(+ model=model if model else self.model,+ messages=messages,+ max_tokens=max_new_tokens,+ temperature=temperature,+ **kwargs,+ )+ return [choice.message.content for choice in response.choices]+\ No newline at end of filediff --git a/webeval/src/webeval/benchmarks/om2w/om2w.py b/webeval/src/webeval/benchmarks/om2w/om2w.pynew file mode 100644index 0000000..3e1f4b5--- /dev/null+++ b/webeval/src/webeval/benchmarks/om2w/om2w.py@@ -0,0 +1,190 @@++from ...benchmark import Benchmark+from typing import Optional, Dict, Any, List+from pathlib import Path+import os+from autogen_core.components.models import UserMessage, SystemMessage, LLMMessage, ChatCompletionClient+from autogen_core.components import Image as AGImage+from autogen_core.components.models import ChatCompletionClient+from webeval.benchmarks.om2w.impl.src.methods import agenttrek_eval, automomous_eval, webjudge_general_eval, webjudge_online_mind2web, webvoyager_eval+from webeval.benchmarks.om2w.impl.src.utils import extract_predication+from webeval.trajectory import Trajectory, FinalAnswer+from ...evaluators import compute_aggregate_metrics_gpt_evaluator, safe_mean, compute_refusal_metrics+import asyncio+import json++def _content_2_agento(content):+ result = []+ if isinstance(content, str):+ return content+ elif isinstance(content, dict):+ if content['type'] == 'text':+ return content['text']+ elif content['type'] == 'image_url':+ return AGImage.from_uri(content['image_url']['url'])+ else:+ raise ValueError(f"Unsupported content type: {content['type']}")+ elif isinstance(content, list):+ return [_content_2_agento(c) for c in content]+ else:+ raise ValueError(f"Unsupported content type: {type(content)}")++def _messages_2_agento(messages):+ result = []+ for m in messages:+ if m['role'] == 'user':+ result.append(UserMessage(content=_content_2_agento(m['content']), source='user'))+ elif m['role'] == 'system':+ result.append(SystemMessage(content=_content_2_agento(m['content'])))+ return result++class _ModelWrapper:+ def __init__(self, model_client: ChatCompletionClient):+ self.model_client = model_client++ def generate(self, messages, **kwargs) -> LLMMessage:+ response = asyncio.run(self.model_client.create(_messages_2_agento(messages), **kwargs))+ return [response.content]++ async def agenerate(self, messages, **kwargs) -> LLMMessage:+ response = await self.model_client.create(_messages_2_agento(messages), **kwargs)+ return [response.content]++class OnlineM2WBenchmark(Benchmark):+ def __init__(self,+ data_dir: Path,+ eval_method = 'AgentTrek_eval',+ model_client: Optional[ChatCompletionClient] = None,+ data_az_url: Optional[str] = None,+ score_threshold: float = 3):+ self.eval_method = eval_method+ self.original_model_client = model_client+ self.model_client = _ModelWrapper(model_client)+ data_dir = Path(data_dir).expanduser()+ data_dir.mkdir(parents=True, exist_ok=True)+ self.data_file = data_dir / 'Online_Mind2Web_06042025.json'+ super().__init__(+ name='OnlineM2W',+ data_dir=data_dir)+ self.data_az_url = "Online_Mind2Web_06042025.json" if data_az_url is None else data_az_url+ self.score_threshold = score_threshold++ def download_dataset(self) -> None:+ print("Updated datasets for om2w are stored locally in webeval/data/om2w")+ pass++ def load_dataset(self) -> None:+ with open(self.data_file, 'r', encoding='utf-8') as f:+ examples = json.load(f)+ self.examples = [+ {+ 'id': ex['task_id'],+ 'level': ex['level'],+ 'question': ex['confirmed_task'],+ 'web': ex['website'],+ 'reference_length': ex['reference_length']+ } for ex in examples+ ]++ def evaluator(self, task_data: Dict[str, Any], candidate: Trajectory) -> Any:+ if self.eval_method == 'AgentTrek_eval':+ messages, _, __ = agenttrek_eval.AgentTrek_eval(+ task=task_data['question'],+ last_actions = candidate.actions,+ thoughts = candidate.thoughts,+ images_path = candidate.latest_screenshot+ )+ elif self.eval_method == 'Autonomous_eval':+ messages, _, __ = automomous_eval.Autonomous_eval(+ task=task_data['question'],+ last_actions = candidate.actions,+ images_path = candidate.latest_screenshot+ )+ elif self.eval_method == "WebJudge_general_eval":+ messages, _, __, ___, ____ = asyncio.run(+ webjudge_general_eval.WebJudge_general_eval(+ task_data['question'],+ None, # TODO: input image path+ candidate.thoughts,+ candidate.actions,+ candidate.screenshots,+ self.model_client,+ self.score_threshold))+ elif self.eval_method == "WebJudge_Online_Mind2Web_eval":+ messages, _, __, ___, ____ = asyncio.run(+ webjudge_online_mind2web.WebJudge_Online_Mind2Web_eval(+ task_data['question'],+ candidate.actions,+ candidate.screenshots,+ self.model_client,+ self.score_threshold))+ else:+ raise NotImplementedError(f"Evaluation method {self.eval_method} is not implemented.")+ verdict, gpt_response_text = asyncio.run(+ self.gpt_evaluator_async(messages)+ )+ return verdict, gpt_response_text++ async def gpt_evaluator_async(self, messages) -> float:+ response = await self.model_client.agenerate(messages)+ verdict = extract_predication(response[0], self.eval_method)+ return verdict, response[0]+++ def get_split_examples(self, split: str) -> List[Dict[str, Any]]:+ exs = None+ if split == 'easy':+ return [ex for ex in self.examples if ex["level"] == "easy"]+ elif split == 'medium':+ return [ex for ex in self.examples if ex["level"] == "medium"]+ elif split == 'hard':+ return [ex for ex in self.examples if ex["level"] == "hard"]+ elif split == '*':+ return self.examples++ raise ValueError(f"Unsupported split: {split}")++ def compute_aggregate_metrics(self, results: List[Any]) -> Dict[str, float]:+ # Build a mapping from id to level for all examples+ id_to_level = {+ ex["id"]: ex["level"]+ for ex in self.examples+ }++ # Group scores by level (using all results, not filtered)+ level_to_scores = {}+ for score in results:+ level = id_to_level[score.qid]+ level_to_scores.setdefault(level, []).append(score.score)++ # Compute mean score (accuracy) for each level+ accuracy_by_level = {+ k: (safe_mean(v), len(v)) for k, v in level_to_scores.items()+ }++ # Add individual accuracy metrics for each level+ accuracy_for_key = {}+ samples_by_key = {}+ for level, level_scores in level_to_scores.items():+ accuracy_for_key[f"accuracy_{level}"] = safe_mean(level_scores)+ samples_by_key[f"samples_{level}"] = len(level_scores)++ # Compute global metrics+ metrics = compute_aggregate_metrics_gpt_evaluator(results)++ if self.original_model_client is not None:+ refusal_metrics = compute_refusal_metrics(results, self.original_model_client)+ metrics.update(refusal_metrics)++ metrics["accuracy_by_level"] = accuracy_by_level+ metrics.update(accuracy_for_key)+ metrics.update(samples_by_key)+ return metrics+++ def exec_hash(self) -> str:+ return f"{super().exec_hash()}_{(self.data_az_url or '').split('/')[-1]}"++ def eval_hash(self) -> str:+ return f"{self.eval_method}-{self.score_threshold}"+