Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

15.6 KB
Raw Blame
{
  "source": "The full end-to-end run, run 1b (Chrome, narrated), rows 8003 to 8017 from cpki.trajectory_events: the person talks while they work, each phrase 4 to 9 s long",
  "rows": [
    {
      "name": "transcript",
      "type": "CUSTOM",
      "value": {
        "text": "I am opening order 10:43 because the customer wants a refund",
        "endedAt": "2026-10-21T02:03:47.061Z",
        "language": "en",
        "startedAt": "2026-10-21T02:03:43.129Z"
      },
      "timestamp": 1792548223129
    },
    {
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 238,
          "width": 139,
          "height": 42
        },
        "url": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "click",
        "route": "/",
        "target": {
          "tag": "button",
          "name": "Load my profile",
          "role": "button",
          "text": "Load my profile",
          "input": "pointer",
          "label": "Load my profile",
          "action": null,
          "attributes": {
            "id": "load-profile",
            "type": "button",
            "class": "ghost"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "id",
            "selector": "#load-profile"
          },
          {
            "by": "role",
            "selector": "role=button[name=\"Load my profile\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Load my profile\""
          },
          {
            "by": "css",
            "selector": "html > body > main > button"
          }
        ]
      },
      "timestamp": 1792548227264
    },
    {
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 387,
          "width": 720,
          "height": 40
        },
        "url": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "click",
        "route": "/",
        "target": {
          "tag": "input",
          "name": "Search orders",
          "role": "textbox",
          "text": "",
          "input": "pointer",
          "label": "Search orders",
          "value": "",
          "action": null,
          "inputType": "text",
          "attributes": {
            "name": "q",
            "style": "",
            "aria-label": "Search orders",
            "placeholder": "Order number or customer"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "name",
            "selector": "input[name=\"q\"]"
          },
          {
            "by": "role",
            "selector": "role=textbox[name=\"Search orders\"]"
          },
          {
            "by": "css",
            "selector": "#search > input"
          }
        ]
      },
      "timestamp": 1792548229218
    },
    {
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 387,
          "width": 720,
          "height": 40
        },
        "url": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "input",
        "route": "/",
        "target": {
          "tag": "input",
          "name": "Search orders",
          "role": "textbox",
          "text": "",
          "value": "1043",
          "action": null,
          "inputType": "text",
          "attributes": {
            "name": "q",
            "style": "",
            "aria-label": "Search orders",
            "placeholder": "Order number or customer"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "name",
            "selector": "input[name=\"q\"]"
          },
          {
            "by": "role",
            "selector": "role=textbox[name=\"Search orders\"]"
          },
          {
            "by": "css",
            "selector": "#search > input"
          }
        ]
      },
      "timestamp": 1792548229807
    },
    {
      "name": "key",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 387,
          "width": 720,
          "height": 40
        },
        "key": "Enter",
        "url": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "enter",
        "route": "/",
        "target": {
          "tag": "input",
          "name": "Search orders",
          "role": "textbox",
          "inputType": "text",
          "attributes": {
            "name": "q",
            "aria-label": "Search orders",
            "placeholder": "Order number or customer"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "name",
            "selector": "input[name=\"q\"]"
          },
          {
            "by": "role",
            "selector": "role=textbox[name=\"Search orders\"]"
          },
          {
            "by": "css",
            "selector": "#search > input"
          }
        ]
      },
      "timestamp": 1792548231104
    },
    {
      "name": "transcript",
      "type": "CUSTOM",
      "value": {
        "text": "the parcel arrived damaged so I will send the money back to her card",
        "endedAt": "2026-10-21T02:03:53.590Z",
        "language": "en",
        "startedAt": "2026-10-21T02:03:47.061Z"
      },
      "timestamp": 1792548227061
    },
    {
      "name": "submit",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 387,
          "width": 720,
          "height": 40
        },
        "url": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "submit",
        "route": "/",
        "target": {
          "tag": "form",
          "name": "",
          "role": "search",
          "attributes": {
            "id": "search",
            "role": "search",
            "method": "get"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "id",
            "selector": "#search"
          },
          {
            "by": "css",
            "selector": "html > body > main > form"
          }
        ]
      },
      "timestamp": 1792548231105
    },
    {
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/search?q=1043",
        "from": "http://127.0.0.1:7420/",
        "page": {
          "url": "http://127.0.0.1:7420/search?q=1043",
          "title": "Search - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "push"
      },
      "timestamp": 1792548231126.219
    },
    {
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 403,
          "y": 186,
          "width": 78,
          "height": 18
        },
        "url": "http://127.0.0.1:7420/search?q=1043",
        "page": {
          "url": "http://127.0.0.1:7420/search?q=1043",
          "title": "Search - Corner Shop admin"
        },
        "step": "click",
        "route": "/search",
        "target": {
          "tag": "a",
          "name": "Order 1043",
          "role": "link",
          "text": "Order 1043",
          "input": "pointer",
          "label": "Order 1043",
          "action": null,
          "attributes": {
            "href": "/orders/1043"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=link[name=\"Order 1043\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Order 1043\""
          },
          {
            "by": "css",
            "selector": "html > body > main > p > a"
          }
        ]
      },
      "timestamp": 1792548234093
    },
    {
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/orders/1043",
        "from": "http://127.0.0.1:7420/search?q=1043",
        "page": {
          "url": "http://127.0.0.1:7420/orders/1043",
          "title": "Order 1043 - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "push"
      },
      "timestamp": 1792548234105.815
    },
    {
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 248,
          "width": 720,
          "height": 38
        },
        "url": "http://127.0.0.1:7420/orders/1043",
        "page": {
          "url": "http://127.0.0.1:7420/orders/1043",
          "title": "Order 1043 - Corner Shop admin"
        },
        "step": "input",
        "route": "/orders/1043",
        "target": {
          "tag": "select",
          "name": "Status",
          "role": "combobox",
          "text": "ProcessingShippedDeliveredReturned",
          "value": "returned",
          "action": null,
          "inputType": "select-one",
          "attributes": {
            "id": "status",
            "name": "status"
          },
          "selectedValues": ["returned"]
        },
        "request": {
          "url": "http://127.0.0.1:7420/api/orders/1043/status",
          "body": {
            "text": "{\"status\":\"returned\"}",
            "status": "complete",
            "encoding": "utf-8"
          },
          "method": "PUT",
          "status": 200,
          "response": {
            "text": "{\"id\":1043,\"status\":\"returned\"}",
            "status": "complete",
            "encoding": "utf-8"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "id",
            "selector": "#status"
          },
          {
            "by": "name",
            "selector": "select[name=\"status\"]"
          },
          {
            "by": "role",
            "selector": "role=combobox[name=\"Status\"]"
          },
          {
            "by": "css",
            "selector": "html > body > main > select"
          }
        ]
      },
      "timestamp": 1792548236629
    },
    {
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 284,
          "y": 304,
          "width": 13,
          "height": 13
        },
        "url": "http://127.0.0.1:7420/orders/1043",
        "page": {
          "url": "http://127.0.0.1:7420/orders/1043",
          "title": "Order 1043 - Corner Shop admin"
        },
        "step": "click",
        "route": "/orders/1043",
        "target": {
          "tag": "input",
          "name": "Notify the customer by email",
          "role": "checkbox",
          "text": "",
          "input": "pointer",
          "value": "on",
          "action": null,
          "checked": true,
          "inputType": "checkbox",
          "attributes": {
            "id": "notify",
            "type": "checkbox"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "id",
            "selector": "#notify"
          },
          {
            "by": "role",
            "selector": "role=checkbox[name=\"Notify the customer by email\"]"
          },
          {
            "by": "css",
            "selector": "html > body > main > label:nth-of-type(2) > input"
          }
        ]
      },
      "timestamp": 1792548238671
    },
    {
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 284,
          "y": 304,
          "width": 13,
          "height": 13
        },
        "url": "http://127.0.0.1:7420/orders/1043",
        "page": {
          "url": "http://127.0.0.1:7420/orders/1043",
          "title": "Order 1043 - Corner Shop admin"
        },
        "step": "input",
        "route": "/orders/1043",
        "target": {
          "tag": "input",
          "name": "Notify the customer by email",
          "role": "checkbox",
          "text": "",
          "value": "on",
          "action": null,
          "checked": true,
          "inputType": "checkbox",
          "attributes": {
            "id": "notify",
            "type": "checkbox"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "id",
            "selector": "#notify"
          },
          {
            "by": "role",
            "selector": "role=checkbox[name=\"Notify the customer by email\"]"
          },
          {
            "by": "css",
            "selector": "html > body > main > label:nth-of-type(2) > input"
          }
        ]
      },
      "timestamp": 1792548238671
    },
    {
      "name": "transcript",
      "type": "CUSTOM",
      "value": {
        "text": "the card number is [redacted]",
        "endedAt": "2026-10-21T02:04:02.847Z",
        "language": "en",
        "startedAt": "2026-10-21T02:03:53.590Z"
      },
      "timestamp": 1792548233590
    },
    {
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 300,
          "y": 360,
          "width": 9,
          "height": 8
        },
        "url": "http://127.0.0.1:7420/orders/1043",
        "page": {
          "url": "http://127.0.0.1:7420/orders/1043",
          "title": "Order 1043 - Corner Shop admin"
        },
        "step": "click",
        "route": "/orders/1043",
        "target": {
          "tag": "button",
          "name": "Archive order",
          "role": "button",
          "text": "Archive order",
          "input": "pointer",
          "label": "Archive order",
          "action": null,
          "attributes": {
            "id": "archive",
            "type": "button",
            "class": "ghost",
            "data-testid": "archive-order"
          }
        },
        "request": {
          "url": "http://127.0.0.1:7420/api/orders/1043",
          "body": {
            "text": "{\"archived\":true}",
            "status": "complete",
            "encoding": "utf-8"
          },
          "method": "PATCH",
          "status": 200,
          "response": {
            "text": "{\"id\":1043,\"archived\":true,\"archivedAt\":\"2026-10-21T02:04:00.694Z\"}",
            "status": "complete",
            "encoding": "utf-8"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "testid",
            "selector": "[data-testid=\"archive-order\"]"
          },
          {
            "by": "id",
            "selector": "#archive"
          },
          {
            "by": "role",
            "selector": "role=button[name=\"Archive order\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Archive order\""
          },
          {
            "by": "css",
            "selector": "html > body > main > button"
          }
        ]
      },
      "timestamp": 1792548240692
    }
  ]
}