Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

16.4 KB
Raw Blame
{
  "source": "The final end-to-end run's verify recording (trajectory 7983e67d), rows ...34002 and ...34004 to ...34012 from cpki.trajectory_events (the narration, row ...34003, left out): on the shop's checkout the person typed only the admin password (shown, a 9 added at its end, hidden again) and the card, then clicked Send refund (row ...34012), which carried no request. seq removed. `requests` are what the page sent at that click, from the shop's own log (logs/shop-requests.log, `at` in milliseconds after row ...34012) with the headers and answers shop.ts gave them: the page's telemetry, and the refund POST 4 ms after the click.",
  "rows": [
    {
      "row": "34002",
      "name": "navigation",
      "type": "CUSTOM",
      "value": {
        "to": "http://127.0.0.1:7420/checkout?order=1043",
        "tab": 1,
        "from": "",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "navigate",
        "frameId": 0,
        "parentFrameId": -1,
        "navigationType": "start"
      },
      "timestamp": 1792561192912
    },
    {
      "row": "34004",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 397,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Your admin password",
          "role": null,
          "text": "[redacted]",
          "input": "pointer",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "password",
          "attributes": {
            "id": "admin-password",
            "name": "password",
            "type": "password",
            "autocomplete": "current-password"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Your admin password\"]"
          },
          {
            "by": "id",
            "selector": "#admin-password"
          },
          {
            "by": "name",
            "selector": "input[name=\"password\"]"
          },
          {
            "by": "css",
            "selector": "#refund-form > input:nth-of-type(2)"
          }
        ]
      },
      "timestamp": 1792561201390
    },
    {
      "row": "34005",
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 397,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "input",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Your admin password",
          "role": null,
          "text": "[redacted]",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "password",
          "attributes": {
            "id": "admin-password",
            "name": "password",
            "type": "password",
            "autocomplete": "current-password"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Your admin password\"]"
          },
          {
            "by": "id",
            "selector": "#admin-password"
          },
          {
            "by": "name",
            "selector": "input[name=\"password\"]"
          },
          {
            "by": "css",
            "selector": "#refund-form > input:nth-of-type(2)"
          }
        ]
      },
      "timestamp": 1792561202735
    },
    {
      "row": "34006",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 443,
          "width": 141,
          "height": 42
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "button",
          "name": "Show password",
          "role": null,
          "text": "Show password",
          "input": "pointer",
          "label": "Show password",
          "action": null,
          "ariaRole": "button",
          "attributes": {
            "id": "show-pw",
            "type": "button",
            "class": "ghost",
            "style": "margin-top:6px"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=button[name=\"Show password\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Show password\""
          },
          {
            "by": "id",
            "selector": "#show-pw"
          },
          {
            "by": "css",
            "selector": "#refund-form > button:nth-of-type(1)"
          }
        ]
      },
      "timestamp": 1792561204323
    },
    {
      "row": "34007",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 397,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Your admin password",
          "role": null,
          "text": "[redacted]",
          "input": "pointer",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "admin-password",
            "name": "password",
            "type": "text",
            "autocomplete": "current-password"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Your admin password\"]"
          },
          {
            "by": "id",
            "selector": "#admin-password"
          },
          {
            "by": "name",
            "selector": "input[name=\"password\"]"
          },
          {
            "by": "css",
            "selector": "#refund-form > input:nth-of-type(2)"
          }
        ]
      },
      "timestamp": 1792561205856
    },
    {
      "row": "34008",
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 356,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "input",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Your admin password",
          "role": null,
          "text": "[redacted]",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "admin-password",
            "name": "password",
            "type": "text",
            "autocomplete": "current-password"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Your admin password\"]"
          },
          {
            "by": "id",
            "selector": "#admin-password"
          },
          {
            "by": "name",
            "selector": "input[name=\"password\"]"
          },
          {
            "by": "css",
            "selector": "#refund-form > input:nth-of-type(2)"
          }
        ]
      },
      "timestamp": 1792561206163
    },
    {
      "row": "34009",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 402,
          "width": 141,
          "height": 42
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "button",
          "name": "Show password",
          "role": null,
          "text": "Show password",
          "input": "pointer",
          "label": "Show password",
          "action": null,
          "ariaRole": "button",
          "attributes": {
            "id": "show-pw",
            "type": "button",
            "class": "ghost",
            "style": "margin-top:6px"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=button[name=\"Show password\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Show password\""
          },
          {
            "by": "id",
            "selector": "#show-pw"
          },
          {
            "by": "css",
            "selector": "#refund-form > button:nth-of-type(1)"
          }
        ]
      },
      "timestamp": 1792561208194
    },
    {
      "row": "34010",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 483,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Card number",
          "role": null,
          "text": "[redacted]",
          "input": "pointer",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "card-number",
            "name": "cardnumber",
            "style": "",
            "value": "[redacted]",
            "data-last": "",
            "inputmode": "numeric"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Card number\"]"
          },
          {
            "by": "id",
            "selector": "#card-number"
          },
          {
            "by": "name",
            "selector": "input[name=\"cardnumber\"]"
          },
          {
            "by": "css",
            "selector": "#card-root > input"
          }
        ]
      },
      "timestamp": 1792561209723
    },
    {
      "row": "34011",
      "name": "input",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 483,
          "width": 720,
          "height": 40
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "input",
        "route": "/checkout",
        "target": {
          "tag": "input",
          "name": "Card number",
          "role": null,
          "text": "[redacted]",
          "value": "[redacted]",
          "action": null,
          "ariaRole": "textbox",
          "inputType": "text",
          "attributes": {
            "id": "card-number",
            "name": "cardnumber",
            "style": "",
            "value": "[redacted]",
            "data-last": "[redacted]",
            "inputmode": "numeric"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "eventType": "input",
        "selectors": [
          {
            "by": "role",
            "selector": "role=textbox[name=\"Card number\"]"
          },
          {
            "by": "id",
            "selector": "#card-number"
          },
          {
            "by": "name",
            "selector": "input[name=\"cardnumber\"]"
          },
          {
            "by": "css",
            "selector": "#card-root > input"
          }
        ]
      },
      "timestamp": 1792561211158
    },
    {
      "row": "34012",
      "name": "click",
      "type": "CUSTOM",
      "value": {
        "box": {
          "x": 273,
          "y": 725,
          "width": 118,
          "height": 42
        },
        "tab": 1,
        "url": "http://127.0.0.1:7420/checkout?order=1043",
        "page": {
          "url": "http://127.0.0.1:7420/checkout?order=1043",
          "title": "Refund to card - Corner Shop admin"
        },
        "step": "click",
        "route": "/checkout",
        "target": {
          "tag": "button",
          "name": "Send refund",
          "role": null,
          "text": "Send refund",
          "input": "pointer",
          "label": "Send refund",
          "action": null,
          "ariaRole": "button",
          "attributes": {
            "id": "pay",
            "type": "submit"
          }
        },
        "viewport": {
          "width": 1280,
          "height": 860
        },
        "selectors": [
          {
            "by": "role",
            "selector": "role=button[name=\"Send refund\"]"
          },
          {
            "by": "text",
            "selector": "text=\"Send refund\""
          },
          {
            "by": "id",
            "selector": "#pay"
          },
          {
            "by": "css",
            "selector": "#refund-form > button:nth-of-type(2)"
          }
        ]
      },
      "timestamp": 1792561212746
    }
  ],
  "requests": [
    {
      "at": 1,
      "type": "Ping",
      "method": "POST",
      "url": "http://127.0.0.1:7420/v1/rum/track",
      "headers": {
        "content-type": "text/plain;charset=UTF-8"
      },
      "postData": "{\"event\":\"ui_click\",\"sid\":\"e3bvgvb5rxa\",\"el\":\"BUTTON\",\"at\":1792561212746}",
      "status": 204,
      "responseHeaders": {},
      "body": ""
    },
    {
      "at": 1,
      "type": "Fetch",
      "method": "POST",
      "url": "http://127.0.0.1:7420/api/telemetry/events",
      "headers": {
        "content-type": "application/json"
      },
      "postData": "{\"type\":\"click\",\"sid\":\"e3bvgvb5rxa\",\"path\":\"/checkout\"}",
      "status": 204,
      "responseHeaders": {},
      "body": ""
    },
    {
      "at": 4,
      "type": "Fetch",
      "method": "POST",
      "url": "http://127.0.0.1:7420/api/orders/1043/refunds",
      "headers": {
        "content-type": "application/json",
        "authorization": "Bearer nocpkBearerTokAAAA1111BBBB2222CCCC3333",
        "x-api-key": "nocpkApiKey-7f3e9a2b4c5d6e7f8a9b0c1d"
      },
      "postData": "{\"order\":1043,\"fullName\":\"\",\"reason\":\"\",\"password\":\"Kestrel-Pw-58213!9\",\"cardNumber\":\"4929 7311 2846 5036\",\"notes\":\"\",\"reference\":\"\",\"amount\":58.2}",
      "status": 200,
      "responseHeaders": {
        "content-type": "application/json",
        "set-cookie": "refund_auth=nocpkSetCookieVal44556677; Path=/; HttpOnly"
      },
      "body": "{\"refundId\":\"rf_77\",\"status\":\"queued\",\"echo\":{\"order\":1043,\"fullName\":\"\",\"reason\":\"\",\"password\":\"Kestrel-Pw-58213!9\",\"cardNumber\":\"4929 7311 2846 5036\",\"notes\":\"\",\"reference\":\"\",\"amount\":58.2},\"sessionToken\":\"sess_nocpkSessTokQQQQRRRRSSSS112233\"}"
    }
  ]
}