The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
/**
* The transcript oracle.
*
* Invariant, checked after every transition: each persisted user/assistant
* message is rendered EXACTLY ONCE, in persisted order; nothing scenario-made
* is rendered that is not persisted; the persisted assistant rows are exactly
* what the provider streamed; and on the wire, each turn's concatenated
* message.delta text equals what the provider streamed for that turn (and its
* message.complete text equals the final completion).
*
* Every scenario text carries a unique marker (`U<n>-<nonce>` for the user,
* `A<n>-<nonce>` / `A<n>i-<nonce>` for replies, `R<n>-<nonce>` for reasoning),
* so "exactly once" is a count over rendered text, independent of how the
* renderer groups bubbles (live vs hydrated grouping legitimately differs).
*
* Final-state checks converge with a deadline (hydration is asynchronous);
* transient faults are caught separately by an in-page MutationObserver
* sampler that is never allowed to see a marker twice, a reply above its own
* prompt, or two messages swap places between frames.
*/
import { expect, type Page } from '@playwright/test'
import type { GatewayEventFrame, PersistedMessage, WsRecorder } from './harness'
import { persistedTranscript } from './harness'
import type { RecordedCompletion, ScriptedProvider } from './provider'
export const ANY_MARKER_RE = /\b[UAR]\d+i?-[a-z0-9]{3,}\b/g
const FIRST_MARKER_RE = /\b[UAR]\d+i?-[a-z0-9]{3,}\b/
const norm = (text: string) => text.replace(/\s+/g, ' ').trim()
function countOccurrences(haystack: string, needle: string): number {
if (!needle) {
return 0
}
let count = 0
let at = haystack.indexOf(needle)
while (at !== -1) {
count++
at = haystack.indexOf(needle, at + needle.length)
}
return count
}
function countMarker(haystack: string, marker: string): number {
return (haystack.match(new RegExp(`\\b${marker}\\b`, 'g')) ?? []).length
}
/** The user marker a reply/reasoning marker answers: A3-x / A3i-x / R3-x → U3-x. */
export function userMarkerFor(marker: string): string {
return marker.replace(/^[AR](\d+)i?-/, 'U$1-')
}
// ─── Transient duplicate / order sampler ────────────────────────────────
/** Idempotent; re-run after every reload (the observer dies with the document). */
export async function installDuplicateSampler(page: Page): Promise<void> {
await page.evaluate(source => {
const w = window as any
if (w.__coreSampler) {
return
}
const re = new RegExp(source, 'g')
w.__coreSampler = {
samples: 0,
violations: [] as { marker: string; count: number; text: string }[],
order: [] as { detail: string; text: string }[]
}
// "x|y": x has been seen above y. A later frame with y above x is a flip.
const above = new Set<string>()
let scheduled = false
// The last few bubble layouts (newest bubbles: id = markers), so a red
// says which rows drew the copy and what the view showed just before.
const timeline: { at: number; route: string; layout: string[] }[] = []
let lastLayout = ''
const seen = (text: string) => {
const flat = text.replace(/\s+/g, ' ')
return flat.length > 600 ? `${flat.slice(0, 150)} … ${flat.slice(-450)}` : flat
}
const recordLayout = (viewport: HTMLElement) => {
const layout = (
[
...viewport.querySelectorAll('[data-slot="aui_user-message-root"], [data-slot="aui_assistant-message-root"]')
] as HTMLElement[]
)
.slice(-6)
.map(el => `${el.getAttribute('data-message-id') ?? '?'}=${[...new Set(el.innerText.match(re) ?? [])].join('+')}`)
const key = `${location.hash}|${layout.join('|')}`
if (key !== lastLayout) {
lastLayout = key
timeline.push({ at: Math.round(performance.now()), route: location.hash, layout })
timeline.splice(0, Math.max(0, timeline.length - 8))
}
}
const orderViolation = (detail: string, text: string) => {
if (w.__coreSampler.order.length < 20) {
w.__coreSampler.order.push({
detail,
at: Math.round(performance.now()),
route: location.hash,
text: seen(text),
...(w.__coreSampler.order.length === 0 ? { timeline: [...timeline] } : {})
})
}
}
const sample = () => {
scheduled = false
const viewports = [...document.querySelectorAll('[data-slot="aui_thread-viewport"]')] as HTMLElement[]
for (const viewport of viewports) {
if (viewport.getClientRects().length === 0) {
continue
}
const text = viewport.innerText
w.__coreSampler.samples++
recordLayout(viewport)
const counts = new Map<string, number>()
for (const match of text.match(re) ?? []) {
counts.set(match, (counts.get(match) ?? 0) + 1)
}
// Message order, top to bottom by first appearance. Reasoning is left
// out: live and history place the thinking block differently.
const sequence = [...counts.keys()].filter(marker => !marker.startsWith('R'))
sequence.forEach((marker, index) => {
const prompt = marker.replace(/^A(\d+)i?-/, 'U$1-')
if (prompt !== marker && sequence.indexOf(prompt) > index) {
orderViolation(`reply ${marker} rendered above its prompt ${prompt}`, text)
}
for (const later of sequence.slice(index + 1)) {
if (above.has(`${later}|${marker}`)) {
orderViolation(`${marker} moved above ${later}`, text)
}
above.add(`${marker}|${later}`)
}
})
for (const [marker, count] of counts) {
if (count > 1 && w.__coreSampler.violations.length < 20) {
// Where the copies live: one bubble with doubled text vs two bubbles.
const bubbles = (
[
...viewport.querySelectorAll(
'[data-slot="aui_user-message-root"], [data-slot="aui_assistant-message-root"]'
)
] as HTMLElement[]
)
.filter(el => el.innerText.includes(marker))
.map(el => `${el.getAttribute('data-slot')}#${el.getAttribute('data-message-id') ?? el.id ?? ''}`)
w.__coreSampler.violations.push({
marker,
count,
at: Math.round(performance.now()),
route: location.hash,
bubbles,
text: seen(text),
...(w.__coreSampler.violations.length === 0 ? { timeline: [...timeline] } : {})
})
}
}
}
}
new MutationObserver(() => {
if (!scheduled) {
scheduled = true
requestAnimationFrame(sample)
}
}).observe(document.body, { childList: true, subtree: true, characterData: true })
}, ANY_MARKER_RE.source)
}
async function samplerViolations(page: Page): Promise<{ order: any[]; samples: number; violations: any[] }> {
return page.evaluate(() => (window as any).__coreSampler ?? { order: [], samples: 0, violations: [] })
}
// ─── Rendered view ──────────────────────────────────────────────────────
interface RenderedView {
text: string
userBubbles: string[]
}
async function renderedView(page: Page): Promise<RenderedView> {
return page.evaluate(() => {
const viewport = ([...document.querySelectorAll('[data-slot="aui_thread-viewport"]')] as HTMLElement[]).find(
el => el.getClientRects().length > 0
)
if (!viewport) {
return { text: '', userBubbles: [] }
}
return {
text: viewport.innerText.replace(/\s+/g, ' '),
userBubbles: ([...viewport.querySelectorAll('[data-slot="aui_user-message-root"]')] as HTMLElement[]).map(el =>
el.innerText.replace(/\s+/g, ' ').trim()
)
}
})
}
// ─── Wire check ─────────────────────────────────────────────────────────
interface WireTurn {
sessionId: string
deltas: string
reasoning: string
complete: null | string
}
/**
* Backend-truth turns: events deduplicated by (runtime session, seq) across
* every socket (the backend stamps seq once before fan-out), split at
* message.start, in seq order.
*/
export function wireTurns(events: GatewayEventFrame[]): WireTurn[] {
const unique = new Map<string, GatewayEventFrame>()
for (const event of events) {
if (event.seq === null || !event.sessionId) {
continue
}
const key = `${event.sessionId}#${event.seq}`
if (!unique.has(key)) {
unique.set(key, event)
}
}
const bySession = new Map<string, GatewayEventFrame[]>()
for (const event of unique.values()) {
const list = bySession.get(event.sessionId) ?? []
list.push(event)
bySession.set(event.sessionId, list)
}
const turns: WireTurn[] = []
for (const [sessionId, list] of bySession) {
list.sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
let turn: null | WireTurn = null
for (const event of list) {
if (event.type === 'message.start') {
turn = { sessionId, deltas: '', reasoning: '', complete: null }
turns.push(turn)
} else if (!turn) {
continue
} else if (event.type === 'message.delta') {
turn.deltas += String(event.payload?.text ?? '')
} else if (event.type === 'reasoning.delta') {
turn.reasoning += String(event.payload?.text ?? '')
} else if (event.type === 'message.complete') {
turn.complete = String(event.payload?.text ?? '')
}
}
}
return turns
}
/**
* Split a turn's streamed text at scenario markers: every scripted completion
* starts with its own marker, so each segment is one completion's output.
*/
function segments(text: string, re: RegExp): { marker: string; text: string }[] {
const out: { marker: string; text: string }[] = []
const flat = norm(text)
const hits = [...flat.matchAll(new RegExp(re.source, 'g'))]
hits.forEach((hit, i) => {
const end = i + 1 < hits.length ? hits[i + 1]!.index : flat.length
out.push({ marker: hit[0], text: flat.slice(hit.index, end).trim() })
})
return out
}
/**
* Backend stream integrity per turn: the concatenated message.delta (and
* reasoning.delta) text is exactly the provider's output, segment by segment —
* each completion once, whole (or a prefix when the backend itself hung up on
* it, e.g. a steer), in order — and message.complete equals the final
* completion. Only whitespace between tool-iteration segments may differ.
*/
function isSubsequence(words: string[], of: string[]): boolean {
let i = 0
for (const word of of) {
if (i < words.length && words[i] === word) {
i++
}
}
return i === words.length
}
function wireViolations(
ws: WsRecorder,
provider: ScriptedProvider,
userMarkers: Set<string>,
lossy: Set<string>
): string[] {
const problems: string[] = []
const byOpening = new Map<string, RecordedCompletion>()
for (const completion of provider.completions) {
const opening = FIRST_MARKER_RE.exec(completion.sentText)?.[0]
const reasoningOpening = FIRST_MARKER_RE.exec(completion.sentReasoning)?.[0]
if (opening) {
byOpening.set(opening, completion)
}
if (reasoningOpening) {
byOpening.set(reasoningOpening, completion)
}
}
for (const turn of wireTurns(ws.events)) {
const textSegments = segments(turn.deltas, /\bA\d+i?-[a-z0-9]{3,}\b/)
if (textSegments.length === 0 || !userMarkers.has(userMarkerFor(textSegments.at(-1)!.marker))) {
continue
}
// A turn that straddled an injected socket drop may have frames that went
// to the dead socket (the renderer recovers them from the persisted
// transcript, which the render oracle checks). Its frames must still be an
// in-order subsequence of what was streamed — never doubled or foreign.
const gapped = lossy.has(userMarkerFor(textSegments.at(-1)!.marker))
if (turn.complete === null && !gapped) {
problems.push(`wire ${textSegments.at(-1)!.marker}: turn never completed`)
continue
}
const seen = new Set<string>()
const check = (kind: string, segs: { marker: string; text: string }[], pick: (c: RecordedCompletion) => string) => {
segs.forEach((seg, i) => {
const completion = byOpening.get(seg.marker)
if (!completion) {
problems.push(`wire: ${kind} segment ${seg.marker} was never streamed by the provider`)
return
}
if (seen.has(`${kind}:${seg.marker}`)) {
problems.push(`wire: ${kind} segment ${seg.marker} delivered twice in one turn`)
}
seen.add(`${kind}:${seg.marker}`)
const sent = norm(pick(completion))
const partialAllowed = completion.aborted && i < segs.length - 1
if (gapped) {
if (!isSubsequence(seg.text.split(' '), sent.split(' '))) {
problems.push(
`wire: ${kind} ${JSON.stringify(seg.text)} is not an in-order subsequence of ${JSON.stringify(sent)}`
)
}
} else if (partialAllowed ? !sent.startsWith(seg.text) : seg.text !== sent) {
problems.push(
`wire: ${kind} ${JSON.stringify(seg.text)} != provider ${JSON.stringify(sent)}${completion.aborted ? ' (aborted)' : ''}`
)
}
})
}
check('message.delta', textSegments, c => c.sentText)
check('reasoning.delta', segments(turn.reasoning, /\bR\d+-[a-z0-9]{3,}\b/), c => c.sentReasoning)
const final = byOpening.get(textSegments.at(-1)!.marker)
if (final && turn.complete !== null && norm(turn.complete) !== norm(final.sentText)) {
problems.push(
`wire: message.complete ${JSON.stringify(turn.complete)} != final completion ${JSON.stringify(final.sentText)}`
)
}
}
return problems
}
// ─── The oracle ─────────────────────────────────────────────────────────
// A steer that lands at a tool boundary is stored as its own user row, wrapped
// for the model in the out-of-band marker (agent/prompt_builder.py); the user
// sees only their words.
const STEER_RE = /\[OUT-OF-BAND USER MESSAGE[^\]]*\]\s*([\s\S]*?)\s*\[\/OUT-OF-BAND USER MESSAGE\]/
function unwrapSteer(content: string): string {
return STEER_RE.exec(content)?.[1] ?? content
}
// Rows the backend writes itself (hydration.ts NOTICE_DISPLAY_KINDS, plus
// hidden scaffolding): drawn as a notice or not at all, never as a message.
const NOTICE_KINDS = new Set([
'model_switch',
'async_delegation_complete',
'process_complete',
'auto_continue',
'personality_switch',
'failed_turn',
'hidden'
])
const isNotice = (m: PersistedMessage) => m.displayKind !== undefined && NOTICE_KINDS.has(m.displayKind)
export interface OracleTarget {
/** Stored session id (the route id). */
sessionId: string
profile?: string
/** User markers this session must contain (guards against an empty/wrong session passing vacuously). */
expectUserMarkers: string[]
/** User markers whose turn straddled an injected socket drop (wire frames may be gapped, never doubled). */
lossyWire?: string[]
}
function transcriptViolations(persisted: PersistedMessage[], view: RenderedView, target: OracleTarget): string[] {
const problems: string[] = []
const rows = persisted
.filter(m => (m.role === 'user' || m.role === 'assistant') && !isNotice(m) && norm(m.content))
.map(m => (m.role === 'user' ? { ...m, content: unwrapSteer(m.content) } : m))
const persistedMarkers = new Set<string>()
for (const marker of target.expectUserMarkers) {
if (!rows.some(row => row.role === 'user' && countMarker(row.content, marker) === 1)) {
problems.push(`persisted: user message ${marker} missing from the session`)
}
}
for (const marker of target.expectUserMarkers) {
if (rows.filter(row => row.role === 'user' && row.content.includes(marker)).length > 1) {
problems.push(`persisted: user message ${marker} stored more than once`)
}
}
let cursor = -1
for (const row of rows) {
const content = norm(row.content)
for (const marker of content.match(ANY_MARKER_RE) ?? []) {
persistedMarkers.add(marker)
}
const occurrences = countOccurrences(view.text, content)
if (occurrences !== 1) {
problems.push(`rendered ${occurrences}x (want 1): ${row.role} ${JSON.stringify(content.slice(0, 80))}`)
continue
}
const at = view.text.indexOf(content)
if (at < cursor) {
problems.push(`order: ${row.role} ${JSON.stringify(content.slice(0, 40))} rendered before an earlier message`)
}
cursor = at
}
const renderedMarkers = view.text.match(ANY_MARKER_RE) ?? []
const counts = new Map<string, number>()
for (const marker of renderedMarkers) {
counts.set(marker, (counts.get(marker) ?? 0) + 1)
}
for (const [marker, count] of counts) {
if (count > 1) {
problems.push(`marker ${marker} rendered ${count}x`)
}
// Reasoning is not part of the display content; every other marker must be persisted.
if (!marker.startsWith('R') && !persistedMarkers.has(marker)) {
problems.push(`rendered but not persisted: ${marker}`)
}
}
const userRows = rows.filter(row => row.role === 'user').map(row => norm(row.content))
if (view.userBubbles.length !== userRows.length) {
problems.push(`user bubbles ${view.userBubbles.length} != persisted user rows ${userRows.length}`)
}
return problems
}
/**
* Assert the invariant for the session currently on screen. Converges on the
* final state (deadline), then requires zero transient duplicates since the
* sampler was installed and a clean wire.
*/
export async function assertTranscriptOracle(
page: Page,
ws: WsRecorder,
provider: ScriptedProvider,
target: OracleTarget,
label: string
): Promise<void> {
let last: { problems: string[]; persisted: PersistedMessage[]; view: RenderedView } = {
problems: ['not evaluated'],
persisted: [],
view: { text: '', userBubbles: [] }
}
// Wire first: wait until every expected turn's message.complete is on the
// wire (it can trail the persisted row under load), so the DOM check below
// also covers whatever the renderer does on completion. Duplicate/garbled
// frames never heal, so polling cannot mask them.
await expect
.poll(() => wireViolations(ws, provider, new Set(target.expectUserMarkers), new Set(target.lossyWire ?? [])), {
timeout: 60_000,
intervals: [250, 500, 1000],
message: `wire integrity [${label}]`
})
.toEqual([])
await expect
.poll(
async () => {
const persisted = await persistedTranscript(page, target.sessionId, target.profile)
const view = await renderedView(page)
last = { problems: transcriptViolations(persisted, view, target), persisted, view }
return last.problems
},
{ timeout: 60_000, intervals: [250, 500, 1000], message: `transcript oracle [${label}]` }
)
.toEqual([])
.catch(error => {
throw new Error(
`transcript oracle [${label}] failed:\n ${last.problems.join('\n ')}\n` +
`persisted: ${JSON.stringify(last.persisted.map(m => [m.role, m.content.slice(0, 60)]))}\n` +
`rendered: ${JSON.stringify(last.view.text.slice(0, 1500))}\n(${(error as Error).message.split('\n')[0]})`
)
})
const transient = await samplerViolations(page)
expect(transient.violations, `transient duplicate render during [${label}] (${transient.samples} samples)`).toEqual(
[]
)
expect(transient.order, `transient misordered render during [${label}] (${transient.samples} samples)`).toEqual([])
}