Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

13.9 KB
Raw Blame
{
  "source": "The final end-to-end run's run 1 (trajectory 764dddf8), rows ...68097 to ...68112 from cpki.trajectory_events: on /support the person opened the account in a new tab (row ...68104), saved the memo there, then went Home in the first tab. seq and each save's request (the service worker attaches it) removed; navigation rows are what Chrome's webNavigation reported, as the service worker recorded them.",
  "rows": [
    {
      "row": "68097",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 674,
          "y": 15,
          "width": 55,
          "height": 18
        },
        "url": "http://127.0.0.1:7420/address",
        "page": {
          "url": "http://127.0.0.1:7420/address",
          "title": "Delivery address - Corner Shop admin"
        },
        "step": "click",
        "route": "/address",
        "target": {
          "tag": "a",
          "name": "Support",
          "role": null,
          "text": "Support",
          "input": "pointer",
          "label": "Support",
          "action": null,
          "ariaRole": "link",
          "attributes": {
            "href": "/support"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=link[name=\"Support\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Support\""
          },
          {
            "by": "href",
            "selector": "a[href=\"/support\"]"
          },
          {
            "by": "css",
            "selector": "html > body > header > nav > a:nth-of-type(7)"
          }
        ]
      },
      "timestamp": 1792555639943
    },
    {
      "row": "68098",
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/support",
        "from": "http://127.0.0.1:7420/address",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "push",
        "transitionType": "link",
        "transitionQualifiers": []
      },
      "timestamp": 1792555639955.078
    },
    {
      "row": "68100",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 285,
          "width": 720,
          "height": 62
        },
        "url": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "click",
        "route": "/support",
        "target": {
          "tag": "textarea",
          "name": "Reply to Grace",
          "role": null,
          "text": "",
          "input": "pointer",
          "value": "",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "textarea",
          "attributes": {
            "id": "reply"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Reply to Grace\"]"
          },
          {
            "by": "id",
            "selector": "#reply"
          },
          {
            "by": "css",
            "selector": "html > body > main > textarea"
          }
        ]
      },
      "timestamp": 1792555643003
    },
    {
      "row": "68102",
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 285,
          "width": 720,
          "height": 62
        },
        "url": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "input",
        "route": "/support",
        "target": {
          "tag": "textarea",
          "name": "Reply to Grace",
          "role": null,
          "text": "",
          "value": "Sorry about the mug, a refund is on its way",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "textarea",
          "attributes": {
            "id": "reply"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Reply to Grace\"]"
          },
          {
            "by": "id",
            "selector": "#reply"
          },
          {
            "by": "css",
            "selector": "html > body > main > textarea"
          }
        ]
      },
      "timestamp": 1792555645345
    },
    {
      "row": "68103",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 367,
          "width": 107,
          "height": 42
        },
        "url": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "click",
        "route": "/support",
        "target": {
          "tag": "button",
          "name": "Send reply",
          "role": null,
          "text": "Send reply",
          "input": "pointer",
          "label": "Send reply",
          "action": null,
          "ariaRole": "button",
          "attributes": {
            "id": "send",
            "type": "button"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=button[name=\"Send reply\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Send reply\""
          },
          {
            "by": "id",
            "selector": "#send"
          },
          {
            "by": "css",
            "selector": "html > body > main > button"
          }
        ]
      },
      "timestamp": 1792555646910
    },
    {
      "row": "68104",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 224,
          "width": 209,
          "height": 18
        },
        "url": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "click",
        "route": "/support",
        "target": {
          "tag": "a",
          "name": "Open the account in a new tab",
          "role": null,
          "text": "Open the account in a new tab",
          "input": "pointer",
          "label": "Open the account in a new tab",
          "action": null,
          "ariaRole": "link",
          "attributes": {
            "id": "newtab",
            "href": "/account",
            "target": "_blank"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=link[name=\"Open the account in a new tab\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Open the account in a new tab\""
          },
          {
            "by": "id",
            "selector": "#newtab"
          },
          {
            "by": "href",
            "selector": "a[href=\"/account\"]"
          },
          {
            "by": "css",
            "selector": "html > body > main > p > a"
          }
        ]
      },
      "timestamp": 1792555649900
    },
    {
      "row": "68105",
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/account",
        "from": "http://127.0.0.1:7420/account",
        "page": {
          "url": "http://127.0.0.1:7420/account",
          "title": "Account - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "push",
        "transitionType": "link",
        "transitionQualifiers": []
      },
      "timestamp": 1792555649936.539
    },
    {
      "row": "68107",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 450,
          "width": 720,
          "height": 40
        },
        "url": "http://127.0.0.1:7420/account",
        "page": {
          "url": "http://127.0.0.1:7420/account",
          "title": "Account - Corner Shop admin"
        },
        "step": "click",
        "route": "/account",
        "target": {
          "tag": "input",
          "name": "Internal memo",
          "role": null,
          "text": "",
          "input": "pointer",
          "value": "",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "memo",
            "name": "memo"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Internal memo\"]"
          },
          {
            "by": "id",
            "selector": "#memo"
          },
          {
            "by": "name",
            "selector": "input[name=\"memo\"]"
          },
          {
            "by": "css",
            "selector": "#account > input:nth-of-type(4)"
          }
        ]
      },
      "timestamp": 1792555652991
    },
    {
      "row": "68108",
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 450,
          "width": 720,
          "height": 40
        },
        "url": "http://127.0.0.1:7420/account",
        "page": {
          "url": "http://127.0.0.1:7420/account",
          "title": "Account - Corner Shop admin"
        },
        "step": "input",
        "route": "/account",
        "target": {
          "tag": "input",
          "name": "Internal memo",
          "role": null,
          "text": "",
          "value": "memo [redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "memo",
            "name": "memo"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Internal memo\"]"
          },
          {
            "by": "id",
            "selector": "#memo"
          },
          {
            "by": "name",
            "selector": "input[name=\"memo\"]"
          },
          {
            "by": "css",
            "selector": "#account > input:nth-of-type(4)"
          }
        ]
      },
      "timestamp": 1792555654420
    },
    {
      "row": "68109",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 280,
          "y": 505,
          "width": 126,
          "height": 42
        },
        "url": "http://127.0.0.1:7420/account",
        "page": {
          "url": "http://127.0.0.1:7420/account",
          "title": "Account - Corner Shop admin"
        },
        "step": "click",
        "route": "/account",
        "target": {
          "tag": "button",
          "name": "Save account",
          "role": null,
          "text": "Save account",
          "input": "keyboard",
          "label": "Save account",
          "action": null,
          "ariaRole": "button",
          "attributes": {
            "type": "submit",
            "class": "ghost"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=button[name=\"Save account\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Save account\""
          },
          {
            "by": "css",
            "selector": "#account > button"
          }
        ]
      },
      "timestamp": 1792555655974
    },
    {
      "row": "68111",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 213,
          "y": 15,
          "width": 41,
          "height": 18
        },
        "url": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/support",
          "title": "Support - Corner Shop admin"
        },
        "step": "click",
        "route": "/support",
        "target": {
          "tag": "a",
          "name": "Home",
          "role": null,
          "text": "Home",
          "input": "pointer",
          "label": "Home",
          "action": null,
          "ariaRole": "link",
          "attributes": {
            "href": "/"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=link[name=\"Home\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Home\""
          },
          {
            "by": "href",
            "selector": "a[href=\"/\"]"
          },
          {
            "by": "css",
            "selector": "html > body > header > nav > a:nth-of-type(1)"
          }
        ]
      },
      "timestamp": 1792555660484
    },
    {
      "row": "68112",
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/",
        "from": "http://127.0.0.1:7420/support",
        "page": {
          "url": "http://127.0.0.1:7420/",
          "title": "Today - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "push",
        "transitionType": "link",
        "transitionQualifiers": []
      },
      "timestamp": 1792555660499.094
    }
  ]
}