The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
// PURE: ChatTurn -> normalized OTLP log records for the beacon collector.
// No I/O, no globals, no chrome.* — this is the primary unit-tested seam.
//
// The collector's beaconjson exporter fills vendor/product/schema_version/
// endpoint/user/timestamp itself, so we MUST NOT emit those. We emit the
// beacon.* + gen_ai.* attributes that converter.go maps into the Event schema.
import type { ChatTurn, Msg, Retention, ToolCall } from './types.js';
import {
type KeyValue,
type LogRecord,
buildLogsEnvelope,
bool,
int,
msToUnixNano,
str,
strArray,
} from './otlp.js';
import type { BrowserIdentity } from './browser-identity.js';
import {
type EventAction,
MAX_FIELD_BYTES,
ORIGIN,
SERVICE_NAME,
categoryFor,
providerFor,
severityNumber,
} from './vocab.js';
import { recordId, turnId } from './ids.js';
export interface NormalizedTurn {
resourceAttributes: KeyValue[];
logRecords: LogRecord[];
}
const encoder = new TextEncoder();
const utf8Bytes = (text: string) => encoder.encode(text).length;
const jsonBytes = (value: unknown) => encoder.encode(JSON.stringify(value)).length;
/** Truncate a string to a byte budget, as UTF-8 or as measured by `size`,
* never splitting a surrogate pair; returns [text, wasTruncated]. */
function capBytes(text: string, max = MAX_FIELD_BYTES, size = utf8Bytes): [string, boolean] {
if (size(text) <= max) return [text, false];
const prefix = (end: number) =>
text.slice(0, /[\uD800-\uDBFF]/.test(text[end - 1] ?? '') ? end - 1 : end);
// Binary-search the largest prefix that fits (chars ≠ bytes for multibyte).
let lo = 0;
let hi = text.length;
while (lo < hi) {
const mid = (lo + hi + 1) >> 1;
if (size(prefix(mid)) <= max) lo = mid;
else hi = mid - 1;
}
return [prefix(lo), true];
}
/** Light client-side scrubber for retention=redacted (defense in depth; the
* collector also redacts). Masks emails and long token-like strings. */
function scrub(text: string): string {
return text
// The lookbehind starts a match only where a run of these characters begins; without it, a
// long run with no '@' (a base64 blob, say) is rescanned from every position, quadratically.
.replace(/(?<![\w.+-])[\w.+-]+@[\w-]+\.[\w.-]+/g, '[email]')
.replace(/\b(sk|pk|ghp|xox[baprs]|AKIA)[A-Za-z0-9_-]{12,}\b/g, '[token]');
}
interface Shaped {
text: string;
truncated: boolean;
}
/** Apply retention + size cap to a text field. Returns null when the field
* must be dropped entirely (metadata retention). */
function shapeText(text: string, retention: Retention): Shaped | null {
if (retention === 'metadata') return null;
const shown = retention === 'redacted' ? scrub(text) : text;
const [capped, truncated] = capBytes(shown);
return { text: capped, truncated };
}
/** Retention and cap for gen_ai.*.messages, as semconv JSON. The budget is the
* serialized value, so a capped field still parses: messages are kept in order
* and the first that does not fit is cut to the longest text that does. */
function shapeMessages(messages: Msg[], retention: Retention): Shaped | null {
if (retention === 'metadata') return null;
const out: { role: Msg['role']; parts: { type: 'text'; content: string }[] }[] = [];
let bytes = 2; // array brackets
for (const { role, text } of messages) {
const content = retention === 'redacted' ? scrub(text) : text;
const message = { role, parts: [{ type: 'text' as const, content }] };
const size = jsonBytes(message) + (out.length ? 1 : 0);
if (bytes + size <= MAX_FIELD_BYTES) {
out.push(message);
bytes += size;
continue;
}
const room = jsonBytes(content) - (bytes + size - MAX_FIELD_BYTES);
if (room >= 2) out.push({ role, parts: [{ type: 'text', content: capBytes(content, room, jsonBytes)[0] }] });
return { text: JSON.stringify(out), truncated: true };
}
return { text: JSON.stringify(out), truncated: false };
}
/**
* Normalize a chat turn into 1..N OTLP log records:
* - one response record (agent.response.completed, or agent.response if partial)
* - one tool.invoked record per captured tool call
*
* `identity` is the browser identity sw.ts resolved at startup (see browser-identity.ts). It is
* a resource attribute, so every record of the turn carries it, and it is
* metadata rather than content, so it survives every retention mode.
*/
export function normalizeTurn(
turn: ChatTurn,
retention: Retention,
identity?: BrowserIdentity,
): NormalizedTurn {
const resourceAttributes: KeyValue[] = [
str('beacon.origin', ORIGIN),
str('beacon.harness.name', turn.site),
str('service.name', SERVICE_NAME),
str('gen_ai.provider.name', providerFor(turn.site)),
...browserAttributes(identity),
];
const tid = turn.turnId || turnId(turn.sessionId, 0);
const startNano = msToUnixNano(turn.startedAt);
const endNano = msToUnixNano(turn.completedAt ?? turn.startedAt);
const logRecords: LogRecord[] = [];
// Correlation attrs shared by every record in the turn.
const base = (action: EventAction, index = 0): KeyValue[] => [
str('beacon.event.action', action),
str('beacon.event.category', categoryFor(action)),
str('beacon.session.id', turn.sessionId),
str('gen_ai.conversation.id', turn.sessionId),
str('beacon.content.retention', retention),
str('beacon.record.id', recordId(tid, action, index)),
str('beacon.capture.mode', turn.captureMode),
...promptIdAttr(turn),
];
// ---- prompt.submitted ----
// Emitted as its own event with category "prompt" so the collector's
// converter lifts the text into the top-level Event.prompt.text field (it
// only does so for prompt-category events). Without this, the prompt stays
// buried in raw.attributes and never reaches the queryable prompt.text column.
// A turn with no prompt (a regenerated answer) gets no prompt record.
const hasPrompt = turn.promptText !== '' || turn.inputMessages.length > 0;
const prompt = shapeText(turn.promptText, retention);
const inputMsgs = shapeMessages(turn.inputMessages, retention);
if (hasPrompt && (prompt || inputMsgs)) {
const attrs = base('prompt.submitted');
if (turn.requestModel) attrs.push(str('gen_ai.request.model', turn.requestModel));
let truncated = false;
if (prompt) {
attrs.push(str('beacon.prompt.text', prompt.text));
truncated ||= prompt.truncated;
}
if (inputMsgs) {
attrs.push(str('gen_ai.input.messages', inputMsgs.text));
truncated ||= inputMsgs.truncated;
}
if (truncated) attrs.push(bool('beacon.field_truncated', true));
logRecords.push({
timeUnixNano: startNano,
severityNumber: severityNumber('info'),
severityText: 'INFO',
body: { stringValue: `${turn.site} prompt.submitted` },
attributes: attrs,
});
}
// ---- agent.response(.completed) ----
const completed = turn.completedAt != null;
const respAction: EventAction = completed ? 'agent.response.completed' : 'agent.response';
const respAttrs = base(respAction);
if (turn.requestModel) respAttrs.push(str('gen_ai.request.model', turn.requestModel));
if (turn.responseModel) respAttrs.push(str('gen_ai.response.model', turn.responseModel));
// The turn's inputTokens is uncached input (Anthropic's usage.input_tokens),
// but the OTel GenAI semconv input_tokens "SHOULD include all types of input
// tokens, including cached tokens", and the collector reads it that way:
// GenAIUsageFromAttrs subtracts the cache counts below to store uncached input.
// So the semconv count is emitted here, cache reads and writes folded in.
if (turn.usage?.inputTokens != null)
respAttrs.push(
int(
'gen_ai.usage.input_tokens',
turn.usage.inputTokens +
(turn.usage.cacheReadInputTokens ?? 0) +
(turn.usage.cacheCreationInputTokens ?? 0),
),
);
if (turn.usage?.outputTokens != null)
respAttrs.push(int('gen_ai.usage.output_tokens', turn.usage.outputTokens));
// Cache counts use the dotted spelling the collector reads first
// (GenAIUsageFromAttrs). They are subsets of input_tokens above, and the
// collector stores them disjoint from the uncached remainder.
if (turn.usage?.cacheCreationInputTokens != null) {
const writes = turn.usage.cacheCreationInputTokens;
respAttrs.push(int('gen_ai.usage.cache_creation.input_tokens', writes));
// The one-hour subset is a breakdown of the writes, not a fourth count, and
// can never exceed them.
const oneHour = turn.usage.cacheCreation1hInputTokens;
if (oneHour != null)
respAttrs.push(
int(
'gen_ai.usage.cache_creation.ephemeral_1h_input_tokens',
Math.min(Math.max(oneHour, 0), Math.max(writes, 0)),
),
);
}
if (turn.usage?.cacheReadInputTokens != null)
respAttrs.push(int('gen_ai.usage.cache_read.input_tokens', turn.usage.cacheReadInputTokens));
let respTrunc = false;
const outputMsgs = shapeMessages(turn.outputMessages, retention);
if (outputMsgs) {
respAttrs.push(str('gen_ai.output.messages', outputMsgs.text));
respTrunc ||= outputMsgs.truncated;
}
// Always-available metadata (survives metadata retention).
respAttrs.push(int('beacon.response.chars', turn.responseText.length));
respAttrs.push(int('beacon.tool_calls.count', turn.toolCalls.length));
if (respTrunc) respAttrs.push(bool('beacon.field_truncated', true));
logRecords.push({
timeUnixNano: endNano,
severityNumber: severityNumber('info'),
severityText: 'INFO',
body: { stringValue: `${turn.site} ${respAction}` },
attributes: respAttrs,
});
// ---- tool.invoked ----
turn.toolCalls.forEach((tool, i) => {
logRecords.push(toolRecord(turn, tool, i, retention, endNano));
});
return { resourceAttributes, logRecords };
}
/**
* The turn's id as beacon.prompt.id, which the collector promotes to the
* event's prompt.id: one chat request is one prompt and everything the reply
* did for it. Only an id the adapter derived from the request is used; the
* session-level fallback in tid would put every unidentified turn under one id.
*/
function promptIdAttr(turn: ChatTurn): KeyValue[] {
return turn.turnId ? [str('beacon.prompt.id', turn.turnId)] : [];
}
function toolRecord(
turn: ChatTurn,
tool: ToolCall,
index: number,
retention: Retention,
when: string,
): LogRecord {
const tid = turn.turnId || turnId(turn.sessionId, 0);
const action: EventAction = 'tool.invoked';
const attrs: KeyValue[] = [
str('beacon.event.action', action),
str('beacon.event.category', categoryFor(action)),
str('beacon.session.id', turn.sessionId),
str('gen_ai.conversation.id', turn.sessionId),
str('beacon.content.retention', retention),
str('beacon.record.id', recordId(tid, action, index)),
...promptIdAttr(turn),
str('tool.name', tool.name),
str('gen_ai.tool.call.id', tool.id),
str('gen_ai.tool.call.name', tool.name),
];
// A call whose input the site does not expose has no arguments, not "null".
const args = tool.arguments === undefined ? null : shapeText(stringify(tool.arguments), retention);
let truncated = args?.truncated ?? false;
if (args) attrs.push(str('gen_ai.tool.call.arguments', args.text));
if (tool.result !== undefined) {
const res = shapeText(stringify(tool.result), retention);
if (res) {
attrs.push(str('gen_ai.tool.call.result', res.text));
truncated ||= res.truncated;
}
}
if (truncated) attrs.push(bool('beacon.field_truncated', true));
return {
timeUnixNano: when,
severityNumber: severityNumber('info'),
severityText: 'INFO',
body: { stringValue: `${turn.site} ${action} ${tool.name}` },
attributes: attrs,
};
}
/** OTel semconv browser/user-agent attributes. The collector promotes
* user_agent.name/version into the event's typed `user_agent` field. */
function browserAttributes(identity: BrowserIdentity | undefined): KeyValue[] {
if (!identity) return [];
const attrs: KeyValue[] = [str('user_agent.name', identity.name)];
if (identity.version) attrs.push(str('user_agent.version', identity.version));
if (identity.original) attrs.push(str('user_agent.original', identity.original));
if (identity.brands.length > 0) attrs.push(strArray('browser.brands', identity.brands));
return attrs;
}
function stringify(v: unknown): string {
return typeof v === 'string' ? v : JSON.stringify(v ?? null);
}
/** Convenience: normalize straight into a single OTLP logs envelope. */
export function turnToEnvelope(turn: ChatTurn, retention: Retention, identity?: BrowserIdentity) {
const { resourceAttributes, logRecords } = normalizeTurn(turn, retention, identity);
return buildLogsEnvelope(resourceAttributes, logRecords, SERVICE_NAME);
}