Skip to content
Python

Add optional SDK observability and evaluation exports

Rohit P88c353516659
diff --git a/.github/workflows/harness-packages.yml b/.github/workflows/harness-packages.yml
index 16f20b2e..1c00d8fd 100644
--- a/.github/workflows/harness-packages.yml
+++ b/.github/workflows/harness-packages.yml
@@ -54,7 +54,7 @@ jobs:
run: cargo test --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --features test-support --test desktop_api --test tool_free -- --test-threads=1
# Build after qualification so test-support cannot affect the shipped host.
- name: Build native host
- run: cargo build --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export
+ run: cargo build --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export,recording-postgres,evaluation-export
- name: Prepare immutable candidate artifacts
run: node harness/release/prepare.mjs "$GITHUB_WORKSPACE/harness/target/release/${{ matrix.exe }}" '${{ matrix.target }}' .sdk-release
- name: Install tarballs and run the exact TypeScript guide
diff --git a/.github/workflows/rust-sdk.yml b/.github/workflows/rust-sdk.yml
index 88e74531..c3bbea90 100644
--- a/.github/workflows/rust-sdk.yml
+++ b/.github/workflows/rust-sdk.yml
@@ -21,7 +21,7 @@ jobs:
with:
node-version: 24
- name: Public layout and retained transport tests
- run: node --test harness/tests/release-layout.test.mjs harness/tests/stdio-client.test.mjs harness/desktop-starters/test.mjs
+ run: node --test harness/tests/release-layout.test.mjs harness/tests/stdio-client.test.mjs harness/desktop-starters/test.mjs harness/service-ui/test/observability.test.mjs harness/service-ui/test/server.test.mjs
- name: Build SDK and flagship app without launching Electron
run: |
npm --prefix harness/typescript-sdk ci
@@ -71,6 +71,7 @@ jobs:
run: |
npm --prefix typescript-sdk ci
npm --prefix typescript-sdk test
+ npm --prefix typescript-sdk run test:observability
if [ "$RUNNER_OS" = Windows ]; then
echo "OW_TS_TEST_HOST=$(pwd -W)/target/debug/sdk-test-host.exe" >> "$GITHUB_ENV"
else
@@ -219,12 +220,14 @@ jobs:
CARGO_PROFILE_RELEASE_LTO: "false"
CARGO_PROFILE_RELEASE_OPT_LEVEL: "1"
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
- run: cargo build --locked --release -p openworker-sdk --features test-support --bin sdk-test-host
+ run: cargo build --locked --release -p openworker-sdk --features test-support,recording,evaluation-export --bin sdk-test-host
- name: TypeScript SDK transport and real-host storage parity
run: |
export OW_TS_TEST_HOST="$PWD/target/release/sdk-test-host"
npm --prefix typescript-sdk ci
npm --prefix typescript-sdk test
+ node --test typescript-sdk/test/observability-live.test.mjs typescript-sdk/test/evaluation-delivery-live.test.mjs
+ node ../harness/trace-viewer/sync.mjs --check
npm --prefix typescript-sdk run test:integration
npm --prefix typescript-sdk run test:dynamic-agents
node --test --test-concurrency=2 --test-reporter=tap typescript-sdk/test/chat-usage.test.mjs typescript-sdk/test/openrouter-streaming.test.mjs typescript-sdk/test/anthropic-caching.test.mjs typescript-sdk/test/gemini-compat.test.mjs typescript-sdk/test/session-options.test.mjs typescript-sdk/test/stream-events.test.mjs
@@ -284,6 +287,8 @@ jobs:
run: |
cargo test --locked -p openworker-runtime -- --test-threads=1
cargo test --locked -p openworker-telemetry --features otlp
+ OW_OBSERVABILITY_TEST_URL="$OW_PG_TEST_URL" cargo test --locked -p openworker-observability --features postgres,evaluations -- --include-ignored --test-threads=1
+ cargo test --locked -p openworker-store-postgres --features test-support --test environments terminal_team_save_ -- --ignored --test-threads=1
cargo test --locked -p openworker-store-postgres --test store imported_history_and_turn_context_ -- --ignored --test-threads=1
- name: Shared PostgreSQL team contract
run: cargo test --locked -p openworker-store-postgres --features test-support --test team_contract --test agent_team_contract --test agent_team_worker --test standalone_subagents -- --ignored --test-threads=1
diff --git a/deploy/reference/check-recorded-langfuse.mjs b/deploy/reference/check-recorded-langfuse.mjs
new file mode 100644
index 00000000..89d6b36a
--- /dev/null
+++ b/deploy/reference/check-recorded-langfuse.mjs
@@ -0,0 +1,30 @@
+// Compare an exported OpenShell service qualification with local Langfuse.
+import assert from 'node:assert/strict';
+import {readFile,writeFile} from 'node:fs/promises';
+import {setTimeout as delay} from 'node:timers/promises';
+const [file,url,credentialFile,output]=process.argv.slice(2);
+const endpoint=new URL(url);assert(['127.0.0.1','localhost'].includes(endpoint.hostname));
+const fixture=JSON.parse(await readFile(file,'utf8'));
+const credentials=JSON.parse(await readFile(credentialFile,'utf8'));
+const auth='Basic '+Buffer.from(credentials.publicKey+':'+credentials.secretKey).toString('base64');
+const records=[fixture,...fixture.childTraces].flatMap(f=>f.timeline.page.steps.map(r=>r.step));
+assert.equal(new Set(records.map(r=>r.span_id)).size,records.length,'Duplicate local span');
+const ids=[...new Set(records.map(r=>r.trace_id))];
+const start=new Date(Math.min(...records.map(r=>r.started_ms))-60000).toISOString();
+let observations=[];const deadline=Date.now()+90000;
+do {
+ observations=[];
+ for(const traceId of ids){const qs=new URLSearchParams({traceId,limit:'1000',fields:'core,basic,usage,io,model,metadata',fromStartTime:start,toStartTime:new Date().toISOString()});
+ const response=await fetch(`${endpoint.origin}/api/public/v2/observations?${qs}`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});assert.equal(response.status,200);observations.push(...(await response.json()).data);}
+ if(records.every(r=>observations.some(o=>o.id===r.span_id)))break;await delay(1000);
+}while(Date.now()<deadline);
+const observed=new Map(observations.map(o=>[o.id,o]));
+for(const r of records){const o=observed.get(r.span_id);assert(o,`Missing span ${r.span_id}`);assert.equal(o.traceId,r.trace_id);assert.equal(o.parentObservationId??null,r.parent_span_id??null);assert(!o.input&&!o.output,'Content leaked');
+ if(r.operation==='model.attempt'){
+ assert.equal(observations.filter(o=>o.id===r.span_id).length,1,'Duplicate model span');
+ if(r.usage?.input!=null)assert.equal((o.usageDetails.input??0)+(o.usageDetails.input_cached_tokens??0),r.usage.input);
+ if(r.usage?.output!=null)assert.equal(o.usageDetails.output,r.usage.output);
+ }
+}
+const result={ok:true,qualifiedAt:new Date().toISOString(),matchedSpans:records.length,modelAttempts:records.filter(r=>r.operation==='model.attempt').length,toolCalls:records.filter(r=>r.operation==='tool.execute').length,visibleTurns:fixture.subtree.selection.runs.length,traceIds:ids,contentExported:false};
+await writeFile(output,JSON.stringify(result,null,2)+'\n',{mode:0o600});console.log(JSON.stringify(result));
diff --git a/deploy/reference/qualify-sdk-observability.mjs b/deploy/reference/qualify-sdk-observability.mjs
new file mode 100644
index 00000000..dab752c0
--- /dev/null
+++ b/deploy/reference/qualify-sdk-observability.mjs
@@ -0,0 +1,89 @@
+// Actual managed SDK, deterministic model, persistent recording and local Langfuse.
+// Credentials are supplied explicitly and never written into evidence output.
+import assert from 'node:assert/strict';
+import {createHash} from 'node:crypto';
+import {createServer} from 'node:http';
+import {mkdtemp,readFile,writeFile,rm} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {setTimeout as delay} from 'node:timers/promises';
+import {Harness,langfuseTelemetry} from '../../harness/typescript-sdk/dist/index.js';
+const [executable,url,credentialFile,outputFile]=process.argv.slice(2);
+assert(executable&&url&&credentialFile&&outputFile,'Usage: qualify-sdk-observability.mjs HOST LOCAL_LANGFUSE_URL CREDENTIAL_JSON OUTPUT_JSON');
+const endpoint=new URL(url);assert(['127.0.0.1','localhost'].includes(endpoint.hostname)&&endpoint.protocol==='http:','Qualification requires local loopback Langfuse');
+const credentials=JSON.parse(await readFile(credentialFile,'utf8'));
+const auth='Basic '+Buffer.from(credentials.publicKey+':'+credentials.secretKey).toString('base64');
+const previous=process.env.OW_SDK_OBSERVABILITY_AUTH;
+process.env.OW_SDK_OBSERVABILITY_AUTH=auth;
+const root=await mkdtemp(join(tmpdir(),'ow-sdk-observability-'));
+let calls=0;
+const model=createServer(async(req,res)=>{
+ for await(const _chunk of req){};calls++;
+ res.writeHead(200,{'content-type':'text/event-stream'});
+ for(const frame of [
+ {choices:[{index:0,delta:{role:'assistant',content:'Fixture answer.'},finish_reason:null}]},
+ {choices:[{index:0,delta:{},finish_reason:'stop'}]},
+ {choices:[],usage:{prompt_tokens:42,completion_tokens:7,total_tokens:49,prompt_tokens_details:{cached_tokens:12}}},
+ ])res.write(`data: ${JSON.stringify(frame)}\n\n`);
+ res.end('data: [DONE]\n\n');
+});
+await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));
+let harness;
+try {
+ harness=await Harness.start({executable,stateDir:join(root,'state'),sandbox:'off',recording:{database:{type:'sqlite',path:join(root,'traces.sqlite')},evaluations:true,evaluationSinks:[{name:'local-langfuse',url:endpoint.origin,authorizationEnv:'OW_SDK_OBSERVABILITY_AUTH'}]},telemetry:langfuseTelemetry(`${endpoint.origin}/api/public/otel`,'OW_SDK_OBSERVABILITY_AUTH')});
+ await harness.setModelProvider({name:'ObservabilityFixture',api:'chat-completions',baseUrl:`http://127.0.0.1:${model.address().port}/v1`});
+ const session=await harness.createAgent({name:'recording-demo',instructions:'Reply briefly.',tools:[],planning:false,memory:false,scheduling:false,defaultModel:{provider:'ObservabilityFixture',name:'fixture'}}).newSession();
+ const result=await session.run('Private synthetic prompt for recording qualification.');assert.equal(result.status,'completed');assert.equal(calls,1);
+ const turns=await harness.observability.listTurns({sessionId:session.id});
+ const query={sessionId:session.id,turnId:turns.turns[0].turn_id};
+ let timeline,summary;
+ const deadline=Date.now()+15000;
+ do {timeline=await harness.observability.timeline(query);summary=await harness.observability.summary({...query,through:timeline.page.through});if(summary.summary.usage.input.total===42&&timeline.page.steps.some(s=>s.step.operation==='agent.execute'))break;await delay(100);}while(Date.now()<deadline);
+ assert.equal(summary.summary.usage.input.total,42);assert.equal(summary.summary.usage.output.total,7);assert.equal(summary.summary.usage.cache_read.total,12);
+ const modelSpan=timeline.page.steps.find(r=>r.step.operation==='model.attempt').step;
+ const scoreInput={id:`qualification-${query.turnId}`,name:'fixture-correctness',evaluator:{name:'fixture',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},createdMs:Date.now()};
+ const score=await harness.observability.recordEvaluation(query,scoreInput,modelSpan.span_id);
+ assert.equal(score.duplicate,false);
+ assert.equal((await harness.observability.recordEvaluation(query,scoreInput,modelSpan.span_id)).duplicate,true);
+ const scoreDeadline=Date.now()+15000;
+ while((await harness.observability.health()).evaluation_export.sent<1&&Date.now()<scoreDeadline)await delay(100);
+ assert.equal((await harness.observability.health()).evaluation_export.sent,1);
+ assert.equal(calls,1,'Evaluation export must not rerun the model');
+ await harness.shutdown();harness=undefined;
+ const scoreId=createHash('sha256').update(`sdk\0${scoreInput.id}`).digest('hex');
+ const source=timeline.page.steps.map(s=>s.step);const traceIds=[...new Set(source.map(s=>s.trace_id))];
+ const until=Date.now()+90000;let observations=[];
+ do {
+ observations=[];
+ for(const traceId of traceIds){
+ const qs=new URLSearchParams({traceId,limit:'1000',fields:'core,basic,usage,io,model,metadata',fromStartTime:new Date(Date.now()-120000).toISOString(),toStartTime:new Date().toISOString()});
+ const response=await fetch(`${endpoint.origin}/api/public/v2/observations?${qs}`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});
+ assert.equal(response.status,200,'Langfuse observation query failed');observations.push(...(await response.json()).data);
+ }
+ if(source.every(s=>observations.some(o=>o.id===s.span_id)))break;
+ await delay(1000);
+ }while(Date.now()<until);
+ const byId=new Map(observations.map(o=>[o.id,o]));
+ for(const s of source){const o=byId.get(s.span_id);assert(o,`Missing exported span ${s.span_id}`);assert.equal(o.traceId,s.trace_id);assert.equal(o.parentObservationId??null,s.parent_span_id??null);assert(!o.input&&!o.output,'Metadata-only export leaked content');}
+ const generations=source.filter(s=>s.operation==='model.attempt');assert.equal(generations.length,1);
+ const usage=byId.get(generations[0].span_id).usageDetails;
+ assert.equal((usage.input??0)+(usage.input_cached_tokens??0),42);assert.equal(usage.output,7);assert.equal(usage.total,49);
+ assert.equal(observations.filter(o=>o.id===generations[0].span_id).length,1,'Duplicate model observation');
+ let vendorScore;
+ const scoreUntil=Date.now()+90000;
+ do {
+ const response=await fetch(`${endpoint.origin}/api/public/v3/scores?id=${scoreId}&fields=subject,details&limit=100`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});
+ assert.equal(response.status,200,'Langfuse scores query failed');
+ vendorScore=(await response.json()).data.find(s=>s.id===scoreId);
+ if(vendorScore)break;
+ await delay(1000);
+ }while(Date.now()<scoreUntil);
+ assert(vendorScore,'Exported evaluation missing from Langfuse');
+ assert.equal(vendorScore.subject.traceId,modelSpan.trace_id);assert.equal(vendorScore.subject.id,modelSpan.span_id);assert.equal(vendorScore.value,true);
+ assert.equal(vendorScore.name,scoreInput.name);
+ await writeFile(outputFile,JSON.stringify({version:1,qualifiedAt:new Date().toISOString(),calls,matchedSpans:source.length,evaluationRecord:score.result,evaluation:{id:scoreId,traceId:modelSpan.trace_id,spanId:modelSpan.span_id,name:scoreInput.name,value:1},usage:{input:42,output:7,cacheRead:12},timeline,summary,langfuse:{url:endpoint.origin,traceIds}},null,2)+'\n',{mode:0o600});
+ console.log(JSON.stringify({ok:true,matchedSpans:source.length,input:42,output:7,cacheRead:12}));
+} finally {
+ await harness?.shutdown();model.closeAllConnections();await new Promise(resolve=>model.close(resolve));await rm(root,{recursive:true,force:true});
+ if(previous===undefined)delete process.env.OW_SDK_OBSERVABILITY_AUTH;else process.env.OW_SDK_OBSERVABILITY_AUTH=previous;
+}
diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.md
index 5aa282cf..fb193e2b 100644
--- a/docs/harness/guides/observability.md
+++ b/docs/harness/guides/observability.md
@@ -1,14 +1,167 @@
# Observe agent work
-Use the management UI to search for a run, inspect its timeline and compare grouped
-model/tool activity. Export OpenTelemetry to your collector or supported analysis
-service when you need organization-wide dashboards.
-
-Observability and compliance have different jobs. Traces and metrics help explain
-latency, usage and failures. The compliance journal records committed policy and
-execution metadata with scoped queries and durable export. Do not turn on prompt
-or tool-content capture just to obtain usage statistics.
-
-See the [observability setup](../../../harness/observability/README.md) for
-storage and export configuration, and [compliance](compliance.md) for audit
-retention and SIEM delivery. A missing exporter must not trigger tool replay.
+Inspect each turn's timeline, model usage, tools and spawned tasks. Keep recordings
+locally, forward traces to a collector or Langfuse, or use both. These are separate
+options; neither is needed to run an agent.
+
+The high-level recording and evaluation APIs below are an **unpublished source
+candidate**. They are not in the current `0.1.0-beta.2` npm release. Use matching
+local SDK/runtime builds until the next release is reviewed.
+
+## Record locally
+
+Use a recording database separate from execution storage. Content capture is off;
+timing, outcome, usage and identifiers are enough for most product dashboards.
+
+```typescript
+import {Harness} from '@openworkerai/harness-sdk';
+
+const harness = await Harness.start({
+ stateDir: '/absolute/private/app/state',
+ recording: {
+ database: {type: 'sqlite', path: '/absolute/private/app/observations.sqlite'},
+ retentionDays: 30,
+ queueCapacity: 1024,
+ },
+});
+// Create/run your agent as usual; then query its accepted turn.
+const turns = await harness.observability.listTurns({sessionId});
+const query = {sessionId, turnId: turns.turns[0].turn_id};
+await harness.observability.flush(5000);
+const timeline = await harness.observability.timeline(query);
+const summary = await harness.observability.summary({...query, through: timeline.page.through});
+```
+
+Rust uses `Harness::builder().recording(RecordingConfig::sqlite(path))` and
+`harness.observability().timeline(TraceQuery::turn(session_id, turn_id))`.
+Enable Cargo feature `recording`; add `recording-postgres` for PostgreSQL,
+`evaluations` for durable results, and `evaluation-export` for background score
+delivery. `otlp` is a separate default feature; `default-features = false` removes
+it alongside default guardrail/compliance implementations. No UI dependency is
+added to the runtime. Standard packaged binaries include optional capabilities,
+but recording, evidence and evaluation delivery remain disabled until configured.
+
+For PostgreSQL use a second `connectionEnv` and schema under
+`recording.database: {type: 'postgres', connectionEnv: 'OBSERVABILITY_DATABASE_URL',
+schema: 'ow_observability'}`. The execution and recording connections may point to
+the same server or different servers. Run the explicit Rust
+`RecordingConfig::migrate_postgres().await` setup operation before startup.
+Production connections verify TLS; `loopbackPlaintext` is only for local fixtures.
+
+Turn-list cursors refer to execution events; timeline cursors refer to recording
+rows. Follow `next_cursor` even if a bounded turn-list page contains no turns.
+Timeline `through` aligns reads to a watermark, not a cross-request transaction.
+A flush is a bounded barrier, not a guarantee that previous dropped records were
+recovered. Check `health()` counters for drops, write failures and export failures.
+
+## Export traces
+
+```typescript
+import {langfuseTelemetry} from '@openworkerai/harness-sdk';
+
+const telemetry = langfuseTelemetry(
+ 'https://your-langfuse.example/api/public/otel',
+ 'LANGFUSE_AUTHORIZATION',
+);
+// Pass telemetry to Harness.start. The environment value is
+// "Basic " + base64(publicKey + ":" + secretKey), stored only in the backend.
+```
+
+The helper selects Langfuse v4 trace acknowledgements and metadata-only capture.
+For a standard collector, set `telemetry.endpoint` to its OTLP HTTP base URL and
+use the default `otlp` acknowledgement mode. Do not install a second exporter over
+the same event stream. Rust has the matching `TelemetryConfig::langfuse` helper.
+See [storage and collector setup](../../../harness/observability/README.md).
+
+Use [Langfuse's public API](https://langfuse.com/docs/api-and-data-platform/features/public-api)
+for downstream analytics. Traces are best effort, not a billing ledger. Cached
+input and reasoning tokens are subsets; never add them again to input/output.
+Missing usage is unknown. Tool duration and result bytes do not imply exact token
+cost per tool. Export failure must never trigger replay of a tool or agent turn.
+
+## Inspect a turn and its children
+
+Local SDK queries and `service.observability` share timeline/summary shapes. The
+service checks access against execution storage before and after recording reads.
+It authorizes every child independently; parent visibility grants no child access.
+
+`children(query)` returns durable spawn links, including `child_turn_id` when the
+first task was accepted. `subtreeSummary(query)` counts the root plus visible
+spawned initial turns, bounded to 25 scopes. Cycles are deduplicated. Later inbox
+turns remain separate. The response names the selected runs and any truncation;
+concurrent tasks have no single wall elapsed duration.
+
+The optional [trace viewer](../../../harness/trace-viewer/README.md) provides
+Usage/Timeline views, model/tool groups, child navigation and basic score display.
+Your app supplies an authorized browser adapter. Do not put SDK, database or
+exporter credentials in the browser.
+
+## Record evaluations
+
+Enable `recording.evaluations: true`, then submit results from your app or evaluator:
+
+```typescript
+const recorded = await harness.observability.recordEvaluation(query, {
+ id: 'quality-review-1', name: 'correctness',
+ evaluator: {name: 'answer-check', version: '1', source: 'code'},
+ status: 'completed', score: {type: 'boolean', value: true},
+ createdMs: Date.now(), datasetCase: 'case-42', experiment: 'candidate-a',
+}, modelSpanId);
+```
+
+Use a stable ID and original timestamp when retrying the same result. Identical
+writes return the existing record; conflicts fail. Corrections append a new ID
+with `supersedes` referencing the old result. `skipped` and `error` have no score.
+Model evaluators may attach their own model/usage; it is not added to agent usage.
+This feature records results; it does not run judges or incur model charges.
+
+To deliver results automatically, configure `recording.evaluationSinks`:
+
+```typescript
+const evaluationSinks = [{
+ name: 'langfuse', url: 'https://your-langfuse.example',
+ authorizationEnv: 'LANGFUSE_AUTHORIZATION',
+ includeExplanation: false,
+}];
+```
+
+The separate durable queue retries scores with the same ID/name/timestamp, never
+traces or original actions. Delivery depends on destination idempotency semantics;
+unknown acknowledgements are not proof of failure. Lease/backoff survives restart.
+A sink name binds to its configuration; use a new name for a different destination.
+`maxPendingEvaluations` defaults to 10,000; delivered retained results do not use
+pending capacity. `evaluationRetentionDays` defaults to 30 and pruning protects
+undelivered rows. Keep the recording database to resume pending delivery. Shutdown
+is bounded and may leave pending scores for the next host start. Scores without a
+recorded trace target and skipped/error results remain local and are explicitly
+counted as not exportable. Explanations are excluded remotely unless opted in.
+
+For caller-controlled one-shot export, import `exportLangfuseScore` from
+`@openworkerai/harness-sdk/evaluation-export`. It does not retry uncertain responses.
+Choose automatic or caller-controlled delivery for each destination.
+
+## Evidence and audits
+
+Set `recording.evidence` only when content is needed: `userInput`, `assistantOutput`,
+`toolArguments`, `toolResults`, and `maxBytes` (256–65,536). `evidence(query)` projects
+selected committed events, applies best-effort secret redaction, and reports fields
+omitted for size. It excludes failed-attempt deltas, hidden reasoning and blocked
+output. Current-turn input is available; historical input is not reconstructed.
+Execution event IDs are separate from recording cursors.
+
+`evidenceJsonl` in the optional export subpath combines authorized timeline,
+summary, evaluations and explicitly supplied content with a versioned manifest,
+selection, watermark and checksum. It covers only supplied pages, not all history.
+It includes no content unless the caller supplies it. Keep expected answers and
+rubrics in your dataset keyed by `datasetCase`; never inject them into the tested
+agent. Artifact revisions can be included through selected tool results or your
+application's evidence records. Checksums do not establish completeness or
+administrator-resistant integrity.
+
+Use the included session/turn/event references and configuration digests to join
+existing [compliance records and SIEM delivery](compliance.md). Compliance records
+remain the authority for approvals and effects; traces and evaluations are not
+proof of compliance. A metadata reader is not a content auditor. In server apps,
+authorize evidence access separately before returning it, and keep database/host
+APIs in the trusted backend. The service's existing compliance content permission
+continues to control audit reads. Cluster experiment/review tooling comes later.
diff --git a/harness/Cargo.lock b/harness/Cargo.lock
index 5a21cb04..355f3eee 100644
--- a/harness/Cargo.lock
+++ b/harness/Cargo.lock
@@ -1882,10 +1882,11 @@ dependencies = [
[[package]]
name = "openworker-observability"
-version = "0.1.0"
+version = "0.1.0-beta.2"
dependencies = [
"anyhow",
"native-tls",
+ "openworker-compliance",
"openworker-telemetry",
"postgres",
"postgres-native-tls",
@@ -1940,8 +1941,10 @@ version = "0.1.0-beta.2"
dependencies = [
"anyhow",
"async-stream",
+ "chrono",
"fs2",
"futures-util",
+ "openworker-observability",
"openworker-runtime",
"openworker-security-adapters",
"openworker-store-postgres",
diff --git a/harness/harness-sdk/Cargo.toml b/harness/harness-sdk/Cargo.toml
index 252d857a..1c27fd64 100644
--- a/harness/harness-sdk/Cargo.toml
+++ b/harness/harness-sdk/Cargo.toml
@@ -10,7 +10,12 @@ license.workspace = true
publish = ["crates-io"]
[features]
-default = ["guardrails", "compliance"]
+default = ["guardrails", "compliance", "otlp"]
+otlp = ["openworker-telemetry/otlp"]
+recording = ["dep:openworker-observability"]
+evaluations = ["recording", "openworker-observability/evaluations"]
+evaluation-export = ["evaluations", "reqwest/blocking", "dep:chrono"]
+recording-postgres = ["recording", "openworker-observability/postgres"]
guardrails = ["openworker-runtime/guardrails", "openworker-store-postgres/guardrails"]
compliance = ["openworker-runtime/compliance", "openworker-store-postgres/compliance"]
test-support = []
@@ -22,6 +27,8 @@ name = "openworker-harness"
path = "src/main.rs"
[dependencies]
+chrono = { version = "0.4", default-features = false, features = ["std"], optional = true }
+openworker-observability = { path = "../observability", version = "=0.1.0-beta.2", optional = true }
openworker-security-adapters = { version = "=0.1.0-beta.2", path = "../security-adapters", optional = true }
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
anyhow = "1"
@@ -29,7 +36,7 @@ fs2 = "0.4"
rusqlite = { version = "0.32", features = ["bundled"] }
openworker-store-postgres = { version = "=0.1.0-beta.2", path = "../store-postgres" }
openworker-runtime = { version = "=0.1.0-beta.2", path = "../runtime", default-features = false }
-openworker-telemetry = { version = "=0.1.0-beta.2", path = "../telemetry", features = ["otlp"] }
+openworker-telemetry = { version = "=0.1.0-beta.2", path = "../telemetry" }
openworker-tools = { version = "=0.1.0-beta.2", path = "../tools-sdk" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
diff --git a/harness/harness-sdk/README.md b/harness/harness-sdk/README.md
index b4b73eaa..1e946cd9 100644
--- a/harness/harness-sdk/README.md
+++ b/harness/harness-sdk/README.md
@@ -343,3 +343,14 @@ coordination engine without application-managed team setup. This requires a
matching feature runtime; it is not in published runtime 0.1.0-beta.2.
See [standalone sub-agents](../../docs/harness/guides/sessions.md#standalone-sub-agents)
for limits, approvals and shared-sandbox behavior.
+
+
+## Optional turn traces and evaluation evidence
+
+The source candidate includes opt-in recording, per-turn timeline/grouped queries,
+child drill-down and durable evaluation results. Recording and evaluation delivery
+are disabled until configured; browser UI is a separate optional component.
+See the [observability guide](../../docs/harness/guides/observability.md) for local
+SQLite/PostgreSQL, Langfuse export, content selection and audit references. These
+new APIs require matching local candidate builds and are not in the published
+0.1.0-beta.2 packages yet.
diff --git a/harness/harness-sdk/src/agents/harness.rs b/harness/harness-sdk/src/agents/harness.rs
index 6a9e1b7f..3ddf2b43 100644
--- a/harness/harness-sdk/src/agents/harness.rs
+++ b/harness/harness-sdk/src/agents/harness.rs
@@ -20,6 +20,7 @@ pub struct HarnessBuilder {
sandbox: Sandbox,
scripted: bool,
telemetry: Option<TelemetryConfig>,
+ recording: Option<crate::RecordingConfig>,
options: ClientOptions,
}
@@ -129,7 +130,8 @@ impl HarnessBuilder {
/// TelemetryHeader::literal("x-langfuse-ingestion-version", "4"),
/// ],
/// signals: vec![TelemetrySignal::Traces],
- /// content: ContentPolicy { capture: Capture::Full, ..ContentPolicy::default() },
+ /// acknowledgement: TelemetryAcknowledgement::LangfuseV4,
+ /// content: ContentPolicy { capture: Capture::Metadata, ..ContentPolicy::default() },
/// ..TelemetryConfig::default()
/// });
/// # }
@@ -138,6 +140,11 @@ impl HarnessBuilder {
self.telemetry = Some(config);
self
}
+ /// Record diagnostics independently of execution storage and external export.
+ pub fn recording(mut self, config: crate::RecordingConfig) -> Self {
+ self.recording = Some(config);
+ self
+ }
pub fn options(mut self, options: ClientOptions) -> Self {
self.options = options;
self
@@ -175,6 +182,7 @@ impl HarnessBuilder {
sandbox: self.sandbox,
scripted: self.scripted,
telemetry: self.telemetry.clone(),
+ recording: self.recording.clone(),
agents: self.agents.clone(),
})
}
@@ -310,6 +318,7 @@ impl Harness {
sandbox: Sandbox::default(),
scripted: false,
telemetry: None,
+ recording: None,
options: ClientOptions::default(),
}
}
@@ -338,6 +347,9 @@ impl Harness {
})
}
/// The underlying protocol client, for anything this layer does not wrap.
+ pub fn observability(&self) -> crate::recording::Observability {
+ crate::recording::Observability::new(self.client.clone())
+ }
pub fn client(&self) -> &HarnessClient {
&self.client
}
diff --git a/harness/harness-sdk/src/agents/mod.rs b/harness/harness-sdk/src/agents/mod.rs
index 37709d7a..9557f942 100644
--- a/harness/harness-sdk/src/agents/mod.rs
+++ b/harness/harness-sdk/src/agents/mod.rs
@@ -51,8 +51,8 @@ mod team;
pub use crate::{
AccessToken, ApprovalDecision, Capture, ContentPolicy, Credentials, Model, ModelChoice,
- ModelProvider, ProviderKind, Sandbox, Storage, TelemetryConfig, TelemetryHeader,
- TelemetrySignal, TokenRefreshReason, ToolRegistry,
+ ModelProvider, ProviderKind, Sandbox, Storage, TelemetryAcknowledgement, TelemetryConfig,
+ TelemetryHeader, TelemetrySignal, TokenRefreshReason, ToolRegistry,
};
pub use agent_team::{
AgentTeam, AgentTeamRef, ChildContext, ChildOptions, ChildWorkspace, Communication,
diff --git a/harness/harness-sdk/src/evaluation_export.rs b/harness/harness-sdk/src/evaluation_export.rs
new file mode 100644
index 00000000..17596327
--- /dev/null
+++ b/harness/harness-sdk/src/evaluation_export.rs
@@ -0,0 +1,197 @@
+//! Optional durable score delivery using the compliance checkpoint contract.
+//! An export retry never submits a turn or executes a tool.
+use crate::recording::{EvaluationSink, RecordingConfig};
+use anyhow::{ensure, Context, Result};
+use openworker_observability::evaluations::{Evaluation, Score, Status};
+use serde_json::{json, Value};
+use std::{
+ io::Read,
+ sync::{
+ atomic::{AtomicBool, AtomicU64, Ordering},
+ mpsc, Arc, Mutex,
+ },
+ time::Duration,
+};
+#[derive(Default)]
+struct Counters {
+ sent: AtomicU64,
+ skipped: AtomicU64,
+ failed: AtomicU64,
+}
+pub(crate) struct Controller {
+ stop: Arc<AtomicBool>,
+ wake: mpsc::SyncSender<()>,
+ done: Mutex<mpsc::Receiver<()>>,
+ counters: Arc<Counters>,
+}
+impl Controller {
+ pub fn notify(&self) {
+ let _ = self.wake.try_send(());
+ }
+ pub fn health(&self) -> Value {
+ json!({"scope":"sdk_host","sent":self.counters.sent.load(Ordering::Relaxed),"skipped_not_exportable":self.counters.skipped.load(Ordering::Relaxed),"failed_or_uncertain":self.counters.failed.load(Ordering::Relaxed)})
+ }
+ pub fn shutdown(&self) {
+ self.stop.store(true, Ordering::Release);
+ self.notify();
+ let _ = self
+ .done
+ .lock()
+ .unwrap()
+ .recv_timeout(Duration::from_secs(5));
+ }
+}
+impl Drop for Controller {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Release);
+ self.notify();
+ }
+}
+fn now() -> u64 {
+ openworker_runtime::store::now_ms()
+}
+pub(crate) fn start(options: RecordingConfig) -> Result<Option<Arc<Controller>>> {
+ if options.evaluation_sinks.is_empty() {
+ return Ok(None);
+ }
+ for sink in &options.evaluation_sinks {
+ sink.validate()?;
+ ensure!(
+ std::env::var(&sink.authorization_env).is_ok(),
+ "Evaluation export credential unavailable"
+ );
+ }
+ let mut store = options.open()?;
+ for sink in &options.evaluation_sinks {
+ let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());
+ store.register_evaluation_sink(&options.deployment, &sink.name, &binding)?;
+ }
+ drop(store);
+ let (wake, rx) = mpsc::sync_channel(1);
+ let (done, wait) = mpsc::sync_channel(1);
+ let stop = Arc::new(AtomicBool::new(false));
+ let stopped = stop.clone();
+ let counters = Arc::new(Counters::default());
+ let stats = counters.clone();
+ std::thread::Builder::new()
+ .name("ow-evaluation-export".into())
+ .spawn(move || {
+ let client = reqwest::blocking::Client::builder()
+ .no_proxy()
+ .redirect(reqwest::redirect::Policy::none())
+ .timeout(Duration::from_secs(3))
+ .build();
+ if let Ok(client) = client {
+ while !stopped.load(Ordering::Acquire) {
+ for sink in &options.evaluation_sinks {
+ if stopped.load(Ordering::Acquire) {
+ break;
+ }
+ if deliver_next(&options, sink, &client, &stats).is_err() {
+ stats.failed.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ let _ = rx.recv_timeout(Duration::from_secs(1));
+ }
+ } else {
+ stats.failed.fetch_add(1, Ordering::Relaxed);
+ }
+ let _ = done.send(());
+ })?;
+ Ok(Some(Arc::new(Controller {
+ stop,
+ wake,
+ done: Mutex::new(wait),
+ counters,
+ })))
+}
+fn deliver_next(
+ options: &RecordingConfig,
+ sink: &EvaluationSink,
+ client: &reqwest::blocking::Client,
+ stats: &Counters,
+) -> Result<()> {
+ let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());
+ let token = openworker_runtime::id("eval");
+ let mut store = options.open()?;
+ let Some(delivery) =
+ store.claim_evaluation_export(&options.deployment, &sink.name, &binding, &token, now())?
+ else {
+ return Ok(());
+ };
+ let value = &delivery.row.evaluation;
+ // Non-quality outcomes stay available locally without fabricating a score.
+ let result = if value.result.status != Status::Completed || value.target.trace_id.is_none() {
+ stats.skipped.fetch_add(1, Ordering::Relaxed);
+ Ok(())
+ } else {
+ send(client, sink, value).map(|_| {
+ stats.sent.fetch_add(1, Ordering::Relaxed);
+ })
+ };
+ // Stable score ID/name/timestamp survive lease expiry, restart and lost ACK.
+ // This checkpoint advances only for an acknowledged score or explicit skip.
+ store.settle_evaluation_export(
+ &options.deployment,
+ &sink.name,
+ &token,
+ now(),
+ result.is_ok(),
+ )?;
+ result
+}
+pub(crate) fn payload(e: &Evaluation, explanation: bool) -> Result<Value> {
+ e.validate()?;
+ ensure!(e.result.status == Status::Completed, "Not a quality score");
+ let trace = e.target.trace_id.as_ref().context("Score trace missing")?;
+ let (kind, value) = match e.result.score.as_ref().context("Score missing")? {
+ Score::Numeric(v) => ("NUMERIC", json!(v)),
+ Score::Boolean(v) => ("BOOLEAN", json!(u8::from(*v))),
+ Score::Categorical(v) => ("CATEGORICAL", json!(v)),
+ Score::Text(v) => ("TEXT", json!(v)),
+ };
+ let id =
+ openworker_runtime::bundle::digest(format!("{}\0{}", e.deployment, e.result.id).as_bytes());
+ let timestamp = chrono::DateTime::from_timestamp_millis(e.result.created_ms as i64)
+ .context("Invalid score time")?
+ .to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
+ let mut body = json!({"id":id,"name":e.result.name,"traceId":trace,"dataType":kind,"value":value,"timestamp":timestamp,"metadata":{"openworker.evaluation.id":e.result.id,"evaluator":e.result.evaluator.name,"evaluatorVersion":e.result.evaluator.version,"evaluatorSource":e.result.evaluator.source,"evaluatorModel":e.result.evaluator.model,"evaluatorUsage":e.result.evaluator.usage,"supersedes":e.result.supersedes}});
+ if let Some(span) = &e.target.span_id {
+ body["observationId"] = json!(span);
+ }
+ if explanation {
+ if let Some(comment) = &e.result.explanation {
+ body["comment"] = json!(comment);
+ }
+ }
+ Ok(body)
+}
+fn send(client: &reqwest::blocking::Client, sink: &EvaluationSink, e: &Evaluation) -> Result<()> {
+ let body = payload(e, sink.include_explanation)?;
+ let authorization = zeroize::Zeroizing::new(
+ std::env::var(&sink.authorization_env).context("Evaluation credential unavailable")?,
+ );
+ let response = client
+ .post(format!(
+ "{}/api/public/scores",
+ sink.url.trim_end_matches('/')
+ ))
+ .header("Authorization", authorization.as_str())
+ .json(&body)
+ .send()
+ .map_err(|_| anyhow::anyhow!("Evaluation delivery outcome unknown"))?;
+ ensure!(
+ response.status().as_u16() == 200,
+ "Evaluation delivery not acknowledged"
+ );
+ let mut bytes = Vec::new();
+ response.take(65537).read_to_end(&mut bytes)?;
+ ensure!(bytes.len() <= 65536, "Evaluation acknowledgement too large");
+ let ack: Value = serde_json::from_slice(&bytes)
+ .map_err(|_| anyhow::anyhow!("Invalid evaluation acknowledgement"))?;
+ ensure!(
+ ack["id"] == body["id"],
+ "Evaluation acknowledgement identity mismatch"
+ );
+ Ok(())
+}
diff --git a/harness/harness-sdk/src/host.rs b/harness/harness-sdk/src/host.rs
index 3a7a1f37..e2f5283e 100644
--- a/harness/harness-sdk/src/host.rs
+++ b/harness/harness-sdk/src/host.rs
@@ -20,10 +20,18 @@ use tokio::sync::broadcast;
use tokio_util::sync::CancellationToken;
#[path = "host_catalog.rs"]
mod catalog;
+#[cfg(feature = "evaluations")]
+#[path = "host_evaluations.rs"]
+mod evaluations;
+#[cfg(feature = "recording")]
+#[path = "host_evidence.rs"]
+mod evidence;
#[path = "host_named_lifecycle.rs"]
mod named_lifecycle;
#[path = "host_named_types.rs"]
mod named_types;
+#[path = "host_observability.rs"]
+mod observability;
#[path = "host_runs.rs"]
mod runs;
@@ -106,6 +114,8 @@ pub struct CoworkerConfig {
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HostConfig {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub recording: Option<crate::RecordingConfig>,
#[serde(default)]
pub desktop: crate::DesktopOptions,
pub state_dir: PathBuf,
@@ -119,7 +129,7 @@ pub struct HostConfig {
/// Opt-in OTLP/HTTP export (content-free spans, metrics and logs). Point
/// it at an OpenTelemetry Collector that forwards to your backend.
#[serde(default, skip_serializing_if = "Option::is_none")]
- pub telemetry: Option<openworker_telemetry::otlp::Config>,
+ pub telemetry: Option<openworker_telemetry::config::Config>,
/// Agent definitions compiled by the host into coworker bundles under
/// `state_dir/agents` (see `agents::Agent` for the JSON shape). May be
/// used instead of, or alongside, hand-written `coworkers`.
@@ -133,6 +143,12 @@ pub struct HostProcessConfig {
pub restart: RestartPolicy,
}
pub(crate) struct Host {
+ #[cfg(feature = "evaluation-export")]
+ evaluation_exporter: Option<Arc<crate::evaluation_export::Controller>>,
+ #[cfg(feature = "recording")]
+ recording: Option<Arc<crate::instrumentation::Recording>>,
+ #[cfg(feature = "recording")]
+ recording_readers: Arc<tokio::sync::Semaphore>,
pub events: broadcast::Sender<openworker_runtime::store::Event>,
pub gaps: broadcast::Sender<()>,
pub failed: CancellationToken,
@@ -164,6 +180,50 @@ impl HostConfig {
!self.scripted || cfg!(feature = "test-support"),
"Scripted models require a development SDK build with test-support"
);
+ ensure!(
+ self.telemetry.is_none() || cfg!(feature = "otlp"),
+ "Telemetry exporter not included; enable the otlp feature"
+ );
+ if let Some(recording) = &self.recording {
+ ensure!(
+ cfg!(feature = "recording"),
+ "Recording not included; enable the recording feature"
+ );
+ recording.validate()?;
+ ensure!(
+ recording.evaluation_sinks.is_empty()
+ || (recording.evaluations && cfg!(feature = "evaluation-export")),
+ "Evaluation export requires evaluations and the evaluation-export feature"
+ );
+ ensure!(
+ !recording.evaluations || cfg!(feature = "evaluations"),
+ "Evaluation support not included"
+ );
+ if let crate::RecordingDatabase::Postgres { schema, .. } = &recording.database {
+ ensure!(
+ cfg!(feature = "recording-postgres"),
+ "PostgreSQL recording not included"
+ );
+ if let Storage::Postgres {
+ schema: execution, ..
+ } = &self.storage
+ {
+ ensure!(
+ schema != execution,
+ "Recording schema must differ from execution schema"
+ );
+ }
+ }
+ if let crate::RecordingDatabase::Sqlite { path } = &recording.database {
+ let execution = self
+ .desktop
+ .session_dir
+ .as_ref()
+ .unwrap_or(&self.state_dir)
+ .join("runtime.sqlite3");
+ ensure!(path != &execution, "Recording must use a separate database");
+ }
+ }
self.desktop.compaction.validate()?;
self.desktop.safety.validate()?;
ensure!(
@@ -410,16 +470,13 @@ impl Host {
let store = Arc::new(Mutex::new(storage));
// The exporter builds a blocking HTTP client, which must not happen on
// a Tokio worker; a short-lived plain thread works on any runtime flavor.
- let (telemetry, controller) = match config.telemetry.clone() {
- Some(c) => {
- let (t, ctl) = std::thread::spawn(move || c.start("sdk-host"))
- .join()
- .map_err(|_| anyhow::anyhow!("Telemetry startup panicked"))?
- .context("Telemetry configuration failed; check the explicit endpoint")?;
- (t, Some(ctl))
- }
- None => (openworker_telemetry::Telemetry::default(), None),
- };
+ let instrumentation_config = config.clone();
+ let started =
+ std::thread::spawn(move || crate::instrumentation::start(&instrumentation_config))
+ .join()
+ .map_err(|_| anyhow::anyhow!("Instrumentation startup panicked"))??;
+ let telemetry = started.telemetry;
+ let controller = started.controller;
let mut profiles = BTreeMap::new();
for (c, registry, bundle) in prepared {
let mut rt = Runtime::open_instrumented(
@@ -461,6 +518,12 @@ impl Host {
let (events, _) = broadcast::channel(256);
let (gaps, _) = broadcast::channel(16);
let host = Arc::new(Self {
+ #[cfg(feature = "evaluation-export")]
+ evaluation_exporter: started.evaluation_exporter,
+ #[cfg(feature = "recording")]
+ recording: started.recording,
+ #[cfg(feature = "recording")]
+ recording_readers: Arc::new(tokio::sync::Semaphore::new(2)),
events,
gaps,
failed: CancellationToken::new(),
@@ -1028,6 +1091,9 @@ impl Host {
}
return Ok(json!({"registered":true}));
}
+ if method.starts_with("observability/") {
+ return self.observation_dispatch(method, p).await;
+ }
if method == "agent/register" {
return self.register_agent(p).await;
}
@@ -1052,6 +1118,17 @@ impl Host {
json!(cfg!(feature = "security-adapters")),
);
c.insert("compliance".into(), json!(cfg!(feature = "compliance")));
+ c.insert(
+ "evaluation_export".into(),
+ json!(cfg!(feature = "evaluation-export")),
+ );
+ c.insert("evaluations".into(), json!(cfg!(feature = "evaluations")));
+ c.insert("recording".into(), json!(cfg!(feature = "recording")));
+ c.insert(
+ "recording_postgres".into(),
+ json!(cfg!(feature = "recording-postgres")),
+ );
+ c.insert("telemetry_export".into(), json!(cfg!(feature = "otlp")));
c.insert("turn_limits".into(), json!(true));
c.insert("turn_context".into(), json!(true));
c.insert("initial_history".into(), json!(true));
@@ -1302,6 +1379,19 @@ impl Host {
);
}
}
+ if let Some(crate::RecordingConfig {
+ database: crate::RecordingDatabase::Sqlite { path },
+ ..
+ }) = &self.config.recording
+ {
+ let recording = path
+ .canonicalize()
+ .context("Recording database unavailable")?;
+ ensure!(
+ !recording.starts_with(&workspace),
+ "Workspace must not contain the private recording database"
+ );
+ }
if let Some(required) = &self.config.desktop.compliance {
// Application requirements cannot be disabled or weakened by
// a session option. Retain the application retention policy.
@@ -1353,6 +1443,10 @@ impl Host {
db.release(&s.id)?;
}
}
+ #[cfg(feature = "evaluation-export")]
+ if let Some(exporter) = self.evaluation_exporter.clone() {
+ let _ = tokio::task::spawn_blocking(move || exporter.shutdown()).await;
+ }
// Bounded flush (at most 5 s); export problems never fail shutdown.
let controller = self.telemetry.lock().unwrap().take();
if let Some(controller) = controller {
@@ -1360,6 +1454,16 @@ impl Host {
tokio::task::spawn_blocking(move || controller.shutdown(Duration::from_secs(5)))
.await;
}
+ #[cfg(feature = "recording")]
+ if let Some(recording) = &self.recording {
+ let controller = recording.controller.lock().unwrap().take();
+ if let Some(controller) = controller {
+ let _ = tokio::task::spawn_blocking(move || {
+ controller.shutdown(Duration::from_secs(5))
+ })
+ .await;
+ }
+ }
Ok(())
}
}
diff --git a/harness/harness-sdk/src/host_evaluations.rs b/harness/harness-sdk/src/host_evaluations.rs
new file mode 100644
index 00000000..947d2a19
--- /dev/null
+++ b/harness/harness-sdk/src/host_evaluations.rs
@@ -0,0 +1,103 @@
+//! Acknowledged evaluation writes. No action or model call is replayed on failure.
+use super::Host;
+use anyhow::{ensure, Context, Result};
+use openworker_observability::evaluations::{Evaluation, Input, Query, Target};
+use serde::Deserialize;
+use serde_json::{json, Value};
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Record {
+ session_id: String,
+ turn_id: String,
+ result: Input,
+ span_id: Option<String>,
+}
+impl Host {
+ pub(super) async fn evaluation_dispatch(&self, method: &str, p: Value) -> Result<Value> {
+ let recorder = self.recording.clone().context("Recording disabled")?;
+ ensure!(recorder.options.evaluations, "Evaluation storage disabled");
+ if method == "observability/evaluation-record" {
+ let q: Record = serde_json::from_value(p)?;
+ crate::TraceQuery::turn(&q.session_id, &q.turn_id).validate()?;
+ ensure!(
+ self.store
+ .lock()
+ .unwrap()
+ .has_turn(&q.session_id, &q.turn_id)?,
+ "Evaluation target turn unavailable"
+ );
+ let session_hash = openworker_runtime::bundle::digest(q.session_id.as_bytes());
+ let turn_hash = openworker_runtime::bundle::digest(q.turn_id.as_bytes());
+ let permit = self
+ .recording_readers
+ .clone()
+ .try_acquire_owned()
+ .context("Recording readers busy")?;
+ let value=tokio::task::spawn_blocking(move || -> Result<Value> {
+ let _permit=permit;
+ let mut store=recorder.options.open()?;
+ // Trace retention is independent from result retention. A valid
+ // duplicate must not need its old span to remain in raw storage.
+ if let Some(old)=store.evaluation(&recorder.options.deployment,&q.result.id)? {
+ ensure!(old.target.session_hash==session_hash&&old.target.turn_hash==turn_hash&&old.target.span_id==q.span_id,"Evaluation identity conflict");
+ let candidate=Evaluation{result:q.result.clone(),..old.clone()}.redacted();candidate.validate()?;
+ ensure!(candidate==old,"Evaluation identity conflict");
+ return Ok(json!({"result":old,"duplicate":true}));
+ }
+ let trace_id=if let Some(span)=&q.span_id {
+ let step=store.get(&recorder.options.deployment,span)?.context("Evaluation span unavailable")?;
+ ensure!(step.session_hash.as_deref()==Some(&session_hash)&&step.turn_hash.as_deref()==Some(&turn_hash),"Evaluation span scope mismatch");Some(step.trace_id)
+ } else {None};
+ let e=Evaluation{version:1,deployment:recorder.options.deployment.clone(),target:Target{session_hash,turn_hash,trace_id,span_id:q.span_id},result:q.result}.redacted();
+ e.validate()?;
+ let cutoff=openworker_runtime::store::now_ms().saturating_sub(u64::from(recorder.options.evaluation_retention_days)*86_400_000);
+ store.prune_evaluations(&recorder.options.deployment,cutoff,1000)?;
+ let result=store.record_evaluation_bounded(&e,recorder.options.max_pending_evaluations)?;
+ Ok(json!({"result":e,"duplicate":result==openworker_observability::Write::Duplicate}))
+ }).await??;
+ #[cfg(feature = "evaluation-export")]
+ if let Some(exporter) = &self.evaluation_exporter {
+ exporter.notify();
+ }
+ return Ok(value);
+ }
+ if method == "observability/evaluation-list" {
+ let q: crate::TraceQuery = serde_json::from_value(p)?;
+ q.validate()?;
+ ensure!(q.limit <= 50, "Evaluation page limit is 50");
+ ensure!(
+ self.store
+ .lock()
+ .unwrap()
+ .has_turn(&q.session_id, &q.turn_id)?,
+ "Evaluation target turn unavailable"
+ );
+ let query = Query {
+ deployment: recorder.options.deployment.clone(),
+ session_hash: openworker_runtime::bundle::digest(q.session_id.as_bytes()),
+ turn_hash: openworker_runtime::bundle::digest(q.turn_id.as_bytes()),
+ after: q.after,
+ limit: q.limit,
+ };
+ let permit = self
+ .recording_readers
+ .clone()
+ .try_acquire_owned()
+ .context("Recording readers busy")?;
+ let value = tokio::task::spawn_blocking(move || -> Result<Value> {
+ let _permit = permit;
+ Ok(json!(recorder.options.open()?.evaluations(&query)?))
+ })
+ .await??;
+ ensure!(
+ self.store
+ .lock()
+ .unwrap()
+ .has_turn(&q.session_id, &q.turn_id)?,
+ "Evaluation target turn unavailable"
+ );
+ return Ok(value);
+ }
+ anyhow::bail!("Unknown evaluation operation")
+ }
+}
diff --git a/harness/harness-sdk/src/host_evidence.rs b/harness/harness-sdk/src/host_evidence.rs
new file mode 100644
index 00000000..f108797a
--- /dev/null
+++ b/harness/harness-sdk/src/host_evidence.rs
@@ -0,0 +1,162 @@
+//! Opt-in, bounded projection of committed execution events. No attempt deltas,
+//! provider-private reasoning, rejected candidates or unselected tool payloads.
+use super::Host;
+use anyhow::{ensure, Context, Result};
+use serde::Deserialize;
+use serde_json::{json, Value};
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Query {
+ session_id: String,
+ turn_id: String,
+ #[serde(default)]
+ after_event_id: i64,
+}
+fn redact(v: Value) -> Value {
+ match v {
+ Value::String(s) => json!(openworker_telemetry::redact(&s)),
+ Value::Array(a) => Value::Array(a.into_iter().map(redact).collect()),
+ Value::Object(o) => Value::Object(o.into_iter().map(|(k, v)| (k, redact(v))).collect()),
+ v => v,
+ }
+}
+
+fn project(kind: &str, data: &Value, policy: &crate::recording::EvidencePolicy) -> Option<Value> {
+ match kind {
+ "assistant_message" if policy.assistant_output => Some(
+ json!({"messages":data["messages"].as_array().into_iter().flatten().map(|m|json!({"role":"assistant","content":m["content"].as_array().into_iter().flatten().filter(|p|p["type"]=="output_text").map(|p|json!({"type":"output_text","text":p["text"]})).collect::<Vec<_>>()})).collect::<Vec<_>>()}),
+ ),
+ "tool_started" if policy.tool_arguments => Some(
+ json!({"name":data["name"],"call_id":data["call_id"],"arguments":data["arguments"]}),
+ ),
+ "tool_finished"
+ if policy.tool_results
+ && data["outputBlocked"] != true
+ && data["result"]["outputBlocked"] != true =>
+ {
+ Some(
+ json!({"name":data["name"],"call_id":data["call_id"],"result":data["result"],"executed":data["executed"]}),
+ )
+ }
+ "turn_completed" | "turn_recovered" | "turn_failed" | "turn_cancelled"
+ | "turn_interrupted" => Some(
+ json!({"status":data["status"],"outcome_unknown_count":data["outcome_unknown_count"],"automatic_replay":data["automatic_replay"]}),
+ ),
+ _ => None,
+ }
+}
+impl Host {
+ pub(super) fn evidence_dispatch(&self, p: Value) -> Result<Value> {
+ let q: Query = serde_json::from_value(p)?;
+ crate::TraceQuery::turn(&q.session_id, &q.turn_id).validate()?;
+ ensure!(q.after_event_id >= 0, "Invalid evidence cursor");
+ let policy = self
+ .config
+ .recording
+ .as_ref()
+ .and_then(|r| r.evidence.as_ref())
+ .context("Content evidence is disabled")?;
+ let store = self.store.lock().unwrap();
+ ensure!(
+ store.has_turn(&q.session_id, &q.turn_id)?,
+ "Evidence target turn unavailable"
+ );
+ let session = store.metadata(&q.session_id)?;
+ let events = store.events(&q.session_id, q.after_event_id)?;
+ let through = store.cursor(&q.session_id)?;
+ let current = session.turn.as_ref().filter(|t| t.id == q.turn_id);
+ let mut bytes = 0;
+ let mut omitted = 0;
+ let mut result = Vec::new();
+ let input = if q.after_event_id == 0 && policy.user_input {
+ current.map(|t| openworker_telemetry::redact(&t.text))
+ } else {
+ None
+ };
+ let input = input.filter(|text| {
+ if text.len() <= policy.max_bytes {
+ bytes += text.len();
+ true
+ } else {
+ omitted += 1;
+ false
+ }
+ });
+ let mut after = q.after_event_id;
+ for e in events {
+ after = e.event_id;
+ if e.turn_id.as_deref() != Some(&q.turn_id) {
+ continue;
+ }
+ let payload = project(&e.kind, &e.data, policy);
+ if let Some(payload) = payload {
+ let projected = json!({"event_id":e.event_id,"kind":e.kind,"data":redact(payload)});
+ let size = serde_json::to_vec(&projected)?.len();
+ if bytes + size > policy.max_bytes {
+ omitted += 1;
+ continue;
+ }
+ bytes += size;
+ result.push(projected);
+ }
+ }
+ Ok(
+ json!({"version":1,"session_id":q.session_id,"turn_id":q.turn_id,"turn_status":current.map(|t|t.status.as_str()),"configuration":{"agent":session.coworker,"model":session.model,"bundle_digest":session.bundle_digest,"tool_schema_digest":session.schema_digest},"audit_reference":{"session_id":session.id,"turn_id":q.turn_id,"execution_event_through":through,"capture_enabled":session.compliance.is_some()},"input":input,"events":result,"next_event_id":(after<through).then_some(after),"through_event_id":through,"coverage":{"source":"committed_execution_events","input_available":current.is_some()&&policy.user_input,"input_scope":"current_turn_only","omitted_for_size":omitted,"redaction":"best_effort_secret_patterns","captured_fields":policy,"not_an_audit_archive":true}}),
+ )
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ fn policy() -> crate::recording::EvidencePolicy {
+ crate::recording::EvidencePolicy {
+ user_input: true,
+ assistant_output: true,
+ tool_arguments: true,
+ tool_results: true,
+ max_bytes: 1024,
+ }
+ }
+ #[test]
+ fn evidence_keeps_accepted_text_but_not_reasoning_or_attempt_deltas() {
+ let data = json!({"messages":[{"role":"assistant","content":[{"type":"output_text","text":"accepted"},{"type":"reasoning","text":"private"}]}],"provider_private":"hidden"});
+ let result = project("assistant_message", &data, &policy())
+ .unwrap()
+ .to_string();
+ assert!(result.contains("accepted"));
+ assert!(!result.contains("private"));
+ assert!(!result.contains("hidden"));
+ assert!(project("assistant_delta", &data, &policy()).is_none());
+ assert!(project("reasoning_delta", &data, &policy()).is_none());
+ }
+ #[test]
+ fn evidence_excludes_both_blocked_result_shapes_and_preserves_uncertainty() {
+ for data in [
+ json!({"outputBlocked":true,"result":{"text":"candidate"}}),
+ json!({"result":{"outputBlocked":true,"text":"candidate"}}),
+ ] {
+ assert!(project("tool_finished", &data, &policy()).is_none());
+ }
+ let value = project(
+ "tool_finished",
+ &json!({"call_id":"c","executed":true,"result":{"outcome":"unknown"}}),
+ &policy(),
+ )
+ .unwrap();
+ assert_eq!(value["result"]["outcome"], "unknown");
+ assert_eq!(value["executed"], true);
+ let recovery = project(
+ "turn_recovered",
+ &json!({"status":"interrupted","outcome_unknown_count":2,"automatic_replay":false}),
+ &policy(),
+ )
+ .unwrap();
+ assert_eq!(recovery["outcome_unknown_count"], 2);
+ assert_eq!(recovery["automatic_replay"], false);
+
+ let mut disabled = policy();
+ disabled.tool_results = false;
+ assert!(project("tool_finished", &value, &disabled).is_none());
+ }
+}
diff --git a/harness/harness-sdk/src/host_observability.rs b/harness/harness-sdk/src/host_observability.rs
new file mode 100644
index 00000000..ecb4ebf2
--- /dev/null
+++ b/harness/harness-sdk/src/host_observability.rs
@@ -0,0 +1,274 @@
+//! Trusted host query boundary. No model tool can call these SDK methods.
+use super::Host;
+use anyhow::{ensure, Result};
+use serde::Deserialize;
+use serde_json::{json, Value};
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Turns {
+ session_id: String,
+ #[serde(default)]
+ after: i64,
+ #[serde(default = "limit")]
+ limit: usize,
+}
+fn limit() -> usize {
+ 25
+}
+impl Host {
+ pub(super) async fn observation_dispatch(&self, method: &str, p: Value) -> Result<Value> {
+ ensure!(!self.stop.is_cancelled(), "Host stopped");
+ if method == "observability/health" {
+ ensure!(
+ p.as_object().is_some_and(|o| o.is_empty()),
+ "Health query accepts no parameters"
+ );
+ let recording: Option<Value> = None;
+ #[cfg(feature = "recording")]
+ let recording = self
+ .recording
+ .as_ref()
+ .map(|r| json!(r.statistics.snapshot()))
+ .or(recording);
+ let evaluation_export: Option<Value> = None;
+ #[cfg(feature = "evaluation-export")]
+ let evaluation_export = self
+ .evaluation_exporter
+ .as_ref()
+ .map(|e| e.health())
+ .or(evaluation_export);
+ return Ok(
+ json!({"evaluation_export":evaluation_export,"scope":"sdk_host","recording_enabled":self.config.recording.is_some(),"recording":recording,"telemetry":self.instrumentation.snapshot(),"exporter_enabled":self.config.telemetry.is_some()}),
+ );
+ }
+ if method == "observability/flush" {
+ #[derive(Deserialize)]
+ #[serde(deny_unknown_fields)]
+ struct Flush {
+ timeout_ms: u64,
+ }
+ let q: Flush = serde_json::from_value(p)?;
+ ensure!(
+ (1..=5000).contains(&q.timeout_ms),
+ "Flush budget must be 1..5000ms"
+ );
+ let telemetry = self.instrumentation.clone();
+ #[cfg(feature = "recording")]
+ let recording = self.recording.clone();
+ return tokio::task::spawn_blocking(move || {
+ let until=std::time::Instant::now()+std::time::Duration::from_millis(q.timeout_ms);
+ let exported=telemetry.flush(std::time::Duration::from_millis(q.timeout_ms));
+ let drained=true;
+ #[cfg(feature="recording")] let drained=if let Some(r)=recording {
+ while r.statistics.snapshot().pending>0 && std::time::Instant::now()<until { std::thread::sleep(std::time::Duration::from_millis(10)); }
+ r.statistics.snapshot().pending==0
+ } else {drained};
+ let _=until;
+ Ok(json!({"telemetry_barrier":exported,"recording_drained":drained,"guarantee":"bounded_best_effort","evaluation_delivery":"independent_durable_queue"}))
+ }).await?;
+ }
+ ensure!(self.config.recording.is_some(), "Recording is disabled");
+ if method == "observability/turns" {
+ let q: Turns = serde_json::from_value(p)?;
+ ensure!(
+ (1..=100).contains(&q.limit) && q.after >= 0,
+ "Invalid turn page"
+ );
+ let store = self.store.lock().unwrap();
+ let session = store.metadata(&q.session_id)?;
+ // Existing journal reads are bounded at 500 events / 512 KiB. An
+ // empty page can still have a next cursor; do not scan unbounded history.
+ let events = store.events(&q.session_id, q.after)?;
+ let end = store.cursor(&q.session_id)?;
+ let mut after = q.after;
+ let mut turns = Vec::new();
+ for e in events {
+ after = e.event_id;
+ if e.kind == "turn_accepted" {
+ if let Some(id) = e.turn_id {
+ let current = session.turn.as_ref().filter(|t| t.id == id);
+ turns.push(json!({"session_id":session.id,"turn_id":id,"status":current.map(|t| &t.status),"agent":session.coworker,"name":session.name}));
+ if turns.len() == q.limit {
+ break;
+ }
+ }
+ }
+ }
+ return Ok(
+ json!({"turns":turns,"next_cursor":(after < end).then_some(after),"through":end}),
+ );
+ }
+ #[cfg(feature = "evaluations")]
+ if method.starts_with("observability/evaluation-") {
+ return self.evaluation_dispatch(method, p).await;
+ }
+ #[cfg(feature = "recording")]
+ if method == "observability/evidence" {
+ return self.evidence_dispatch(p);
+ }
+ if method == "observability/children" {
+ let q: crate::TraceQuery = serde_json::from_value(p)?;
+ q.validate()?;
+ let children = self
+ .store
+ .lock()
+ .unwrap()
+ .observation_children(&q.session_id, &q.turn_id)?;
+ return Ok(json!({"children":children,"coverage":"captured_spawn_origins_only"}));
+ }
+ #[cfg(feature = "recording")]
+ if method == "observability/subtree-summary" {
+ use openworker_observability::{
+ query::{Query, Scope, MAX_SUMMARY_STEPS},
+ summary,
+ };
+ use openworker_runtime::observation_links::{subtree, Run};
+ let q: crate::TraceQuery = serde_json::from_value(p)?;
+ q.validate()?;
+ ensure!(
+ q.after == 0,
+ "Subtree summary cannot start at a page offset"
+ );
+ let selection = {
+ let store = self.store.lock().unwrap();
+ ensure!(
+ store.has_turn(&q.session_id, &q.turn_id)?,
+ "Trace target turn unavailable"
+ );
+ subtree(
+ Run {
+ session_id: q.session_id.clone(),
+ turn_id: q.turn_id.clone(),
+ },
+ |r| store.observation_children(&r.session_id, &r.turn_id),
+ )?
+ };
+ let recording = self
+ .recording
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("Recording unavailable"))?;
+ let read = Query {
+ deployment: recording.options.deployment.clone(),
+ scopes: selection
+ .runs
+ .iter()
+ .map(|r| Scope {
+ session_hash: openworker_runtime::bundle::digest(r.session_id.as_bytes()),
+ turn_hash: openworker_runtime::bundle::digest(r.turn_id.as_bytes()),
+ })
+ .collect(),
+ after: 0,
+ through: q.through,
+ limit: MAX_SUMMARY_STEPS,
+ };
+ let permit = self
+ .recording_readers
+ .clone()
+ .try_acquire_owned()
+ .map_err(|_| anyhow::anyhow!("Recording readers busy"))?;
+ let page = tokio::task::spawn_blocking(move || {
+ let _permit = permit;
+ recording.options.open()?.timeline(&read)
+ })
+ .await??;
+ for run in &selection.runs {
+ ensure!(
+ self.store
+ .lock()
+ .unwrap()
+ .has_turn(&run.session_id, &run.turn_id)?,
+ "Trace target unavailable"
+ );
+ }
+ return Ok(
+ json!({"selection":selection,"summary":summary::build(&page,summary::Canonical::default())}),
+ );
+ }
+ #[cfg(feature = "recording")]
+ if matches!(method, "observability/timeline" | "observability/summary") {
+ use openworker_observability::{
+ query::{Query, Scope, MAX_SUMMARY_STEPS},
+ summary::{self, Canonical},
+ };
+ let q: crate::TraceQuery = serde_json::from_value(p)?;
+ q.validate()?;
+ ensure!(
+ self.store
+ .lock()
+ .unwrap()
+ .has_turn(&q.session_id, &q.turn_id)?,
+ "Trace target turn unavailable"
+ );
+ let session = self.store.lock().unwrap().metadata(&q.session_id)?;
+ let current = session.turn.as_ref().filter(|t| t.id == q.turn_id);
+ let canonical = Canonical {
+ terminal: current.is_some_and(|t| openworker_runtime::store::terminal(&t.status)),
+ expected_segments: current
+ .and_then(|t| t.observation.as_ref())
+ .map(|o| o.segments),
+ first_started_ms: current
+ .and_then(|t| t.observation.as_ref())
+ .map(|o| o.first_started_at_ms),
+ last_span_id: current.and_then(|t| t.observation.as_ref()).map(|o| {
+ o.last_segment
+ .span
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect()
+ }),
+ pending_waits: current.map(|t| {
+ session
+ .interactions
+ .iter()
+ .filter(|i| i.turn_id == t.id && i.response.is_none())
+ .count() as u64
+ }),
+ };
+ let recording = self
+ .recording
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("Recording unavailable"))?;
+ let is_summary = method == "observability/summary";
+ ensure!(
+ !is_summary || q.after == 0,
+ "Summary cannot start at a page offset"
+ );
+ let read = Query {
+ deployment: recording.options.deployment.clone(),
+ scopes: vec![Scope {
+ session_hash: openworker_runtime::bundle::digest(q.session_id.as_bytes()),
+ turn_hash: openworker_runtime::bundle::digest(q.turn_id.as_bytes()),
+ }],
+ after: q.after,
+ through: q.through,
+ limit: if is_summary {
+ MAX_SUMMARY_STEPS
+ } else {
+ q.limit
+ },
+ };
+ let permit = self
+ .recording_readers
+ .clone()
+ .try_acquire_owned()
+ .map_err(|_| anyhow::anyhow!("Recording readers busy"))?;
+ let page = tokio::task::spawn_blocking(move || {
+ let _permit = permit;
+ recording.options.open()?.timeline(&read)
+ })
+ .await??;
+ // Recheck after a potentially slow independent database read.
+ self.store.lock().unwrap().metadata(&q.session_id)?;
+ let run = json!({"session_id":q.session_id,"turn_id":q.turn_id});
+ return if is_summary {
+ Ok(json!({"run":run,"summary":summary::build(&page,canonical)}))
+ } else {
+ Ok(
+ json!({"run":run,"page":page,"coverage":{"guarantee":"best_effort","order":"recording_cursor","snapshot_across_requests":false}}),
+ )
+ };
+ }
+ anyhow::bail!("Unsupported observability operation or capability")
+ }
+}
diff --git a/harness/harness-sdk/src/instrumentation.rs b/harness/harness-sdk/src/instrumentation.rs
new file mode 100644
index 00000000..a593149a
--- /dev/null
+++ b/harness/harness-sdk/src/instrumentation.rs
@@ -0,0 +1,153 @@
+//! SDK composition of the same recorder/exporter used by the service.
+use crate::HostConfig;
+use anyhow::Result;
+use openworker_telemetry::{Controller, Telemetry};
+
+pub(crate) struct Started {
+ #[cfg(feature = "evaluation-export")]
+ pub evaluation_exporter: Option<std::sync::Arc<crate::evaluation_export::Controller>>,
+ pub telemetry: Telemetry,
+ pub controller: Option<Controller>,
+ #[cfg(feature = "recording")]
+ pub recording: Option<std::sync::Arc<Recording>>,
+}
+pub(crate) fn start(config: &HostConfig) -> Result<Started> {
+ #[cfg(feature = "recording")]
+ if let Some(options) = &config.recording {
+ use openworker_observability::{Recorder, RecordingSink};
+ use openworker_telemetry::{Capture, ContentPolicy, Sink};
+ let exporter: Option<Box<dyn Sink>> = None;
+ #[cfg(feature = "otlp")]
+ let exporter = config
+ .telemetry
+ .as_ref()
+ .map(|c| {
+ c.build_sink("sdk-host")
+ .map(|s| Box::new(s) as Box<dyn Sink>)
+ })
+ .transpose()?
+ .or(exporter);
+ let export_content = config
+ .telemetry
+ .as_ref()
+ .map(|c| c.content.clone())
+ .unwrap_or_default();
+ let (recorder, controller) = Recorder::start(
+ options.open()?,
+ options.deployment.clone(),
+ options.queue_capacity,
+ options.retention_days,
+ )?;
+ let statistics = recorder.statistics();
+ let sink = RecordingSink::new(
+ recorder,
+ options.deployment.clone(),
+ "sdk-host".into(),
+ exporter,
+ export_content.capture,
+ )?;
+ let (telemetry, telemetry_controller) = Telemetry::start_with_content(
+ sink,
+ 2048,
+ ContentPolicy {
+ capture: export_content.capture.max(Capture::Metadata),
+ ..export_content
+ },
+ )?;
+ #[cfg(feature = "evaluation-export")]
+ let evaluation_exporter = crate::evaluation_export::start(options.clone())?;
+ return Ok(Started {
+ #[cfg(feature = "evaluation-export")]
+ evaluation_exporter,
+ telemetry,
+ controller: Some(telemetry_controller),
+ recording: Some(std::sync::Arc::new(Recording {
+ options: options.clone(),
+ statistics,
+ controller: std::sync::Mutex::new(Some(controller)),
+ })),
+ });
+ }
+ let (telemetry, controller) = match &config.telemetry {
+ #[cfg(feature = "otlp")]
+ Some(c) => {
+ let (t, c) = c.start("sdk-host")?;
+ (t, Some(c))
+ }
+ #[cfg(not(feature = "otlp"))]
+ Some(_) => anyhow::bail!("Telemetry exporter not included"),
+ None => (Telemetry::default(), None),
+ };
+ Ok(Started {
+ #[cfg(feature = "evaluation-export")]
+ evaluation_exporter: None,
+ telemetry,
+ controller,
+ #[cfg(feature = "recording")]
+ recording: None,
+ })
+}
+
+#[cfg(feature = "recording")]
+pub(crate) struct Recording {
+ pub options: crate::RecordingConfig,
+ pub statistics: openworker_observability::Statistics,
+ pub controller: std::sync::Mutex<Option<openworker_observability::Controller>>,
+}
+#[cfg(feature = "recording")]
+impl crate::RecordingConfig {
+ pub(crate) fn open(&self) -> Result<Box<dyn openworker_observability::Store>> {
+ self.validate()?;
+ match &self.database {
+ crate::RecordingDatabase::Sqlite { path } => Ok(Box::new(
+ openworker_observability::sqlite::Sqlite::open(path)?,
+ )),
+ #[cfg(feature = "recording-postgres")]
+ crate::RecordingDatabase::Postgres { .. } => Ok(Box::new(
+ openworker_observability::postgres::Postgres::open(&self.postgres()?)?,
+ )),
+ #[cfg(not(feature = "recording-postgres"))]
+ crate::RecordingDatabase::Postgres { .. } => {
+ anyhow::bail!("PostgreSQL recording not included")
+ }
+ }
+ }
+}
+
+#[cfg(feature = "recording-postgres")]
+impl crate::RecordingConfig {
+ fn postgres(&self) -> Result<openworker_observability::postgres::Config> {
+ use openworker_observability::postgres::{Config, Tls};
+ let crate::RecordingDatabase::Postgres {
+ connection_env,
+ schema,
+ ca_file,
+ loopback_plaintext,
+ } = &self.database
+ else {
+ anyhow::bail!("PostgreSQL recording configuration required")
+ };
+ self.validate()?;
+ let mut c = Config::new(
+ std::env::var(connection_env)
+ .map_err(|_| anyhow::anyhow!("Recording database credential unavailable"))?,
+ schema.clone(),
+ );
+ c.tls = if *loopback_plaintext {
+ Tls::LoopbackPlaintext
+ } else {
+ Tls::Verified {
+ ca_file: ca_file.clone(),
+ }
+ };
+ Ok(c)
+ }
+ /// Explicit trusted setup operation. Host startup never migrates PostgreSQL.
+ pub async fn migrate_postgres(&self) -> Result<()> {
+ let options = self.clone();
+ tokio::task::spawn_blocking(move || {
+ openworker_observability::postgres::Postgres::migrate(&options.postgres()?)
+ })
+ .await?
+ }
+}
diff --git a/harness/harness-sdk/src/lib.rs b/harness/harness-sdk/src/lib.rs
index 252cabce..5bdd7705 100644
--- a/harness/harness-sdk/src/lib.rs
+++ b/harness/harness-sdk/src/lib.rs
@@ -1,6 +1,11 @@
//! Development Rust API for the same engine embedded or over managed private stdio.
//! No request is automatically resubmitted after transport failure.
pub mod agents;
+#[cfg(feature = "evaluation-export")]
+mod evaluation_export;
+mod instrumentation;
+pub mod recording;
+pub use recording::{RecordingConfig, RecordingDatabase, TraceQuery};
mod application_tools;
mod callbacks;
mod provider_callbacks;
@@ -34,8 +39,9 @@ pub use openworker_runtime::team_service::{Command as TeamCommand, Create as Tea
pub use openworker_runtime::teams::{Member as TeamMember, Role as TeamRole};
/// OTLP/HTTP export settings: `endpoint` (collector origin, HTTPS or loopback
/// HTTP) and `deployment` (label, default `"default"`).
-pub use openworker_telemetry::otlp::{
- Config as TelemetryConfig, Header as TelemetryHeader, Signal as TelemetrySignal,
+pub use openworker_telemetry::config::{
+ Acknowledgement as TelemetryAcknowledgement, Config as TelemetryConfig,
+ Header as TelemetryHeader, Signal as TelemetrySignal,
};
/// Opt-in prompt/tool content in telemetry (off by default).
pub use openworker_telemetry::{Capture, ContentPolicy};
@@ -517,3 +523,6 @@ pub use openworker_runtime::compliance;
pub use openworker_security_adapters as security_adapters;
pub use openworker_runtime::compliance_export::Sink as ComplianceSink;
+
+#[cfg(feature = "evaluations")]
+pub use openworker_observability::evaluations;
diff --git a/harness/harness-sdk/src/recording.rs b/harness/harness-sdk/src/recording.rs
new file mode 100644
index 00000000..6a9ff83d
--- /dev/null
+++ b/harness/harness-sdk/src/recording.rs
@@ -0,0 +1,432 @@
+//! Optional local recording configuration. Does not initialize storage on import.
+use anyhow::{ensure, Result};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct RecordingConfig {
+ #[serde(default = "evaluation_capacity")]
+ pub max_pending_evaluations: u32,
+ #[serde(default = "retention")]
+ pub evaluation_retention_days: u16,
+ #[serde(default)]
+ pub evaluation_sinks: Vec<EvaluationSink>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub evidence: Option<EvidencePolicy>,
+ #[serde(default)]
+ pub evaluations: bool,
+ pub database: RecordingDatabase,
+ #[serde(default = "deployment")]
+ pub deployment: String,
+ #[serde(default = "retention")]
+ pub retention_days: u16,
+ #[serde(default = "capacity")]
+ pub queue_capacity: usize,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
+pub enum RecordingDatabase {
+ Sqlite {
+ path: PathBuf,
+ },
+ Postgres {
+ connection_env: String,
+ schema: String,
+ ca_file: Option<PathBuf>,
+ #[serde(default)]
+ loopback_plaintext: bool,
+ },
+}
+fn deployment() -> String {
+ "sdk".into()
+}
+fn evaluation_capacity() -> u32 {
+ 10_000
+}
+fn retention() -> u16 {
+ 30
+}
+fn capacity() -> usize {
+ 1024
+}
+impl RecordingConfig {
+ pub fn sqlite(path: impl Into<PathBuf>) -> Self {
+ Self {
+ database: RecordingDatabase::Sqlite { path: path.into() },
+ evaluations: false,
+ max_pending_evaluations: evaluation_capacity(),
+ evaluation_retention_days: retention(),
+ evaluation_sinks: vec![],
+ evidence: None,
+ deployment: deployment(),
+ retention_days: retention(),
+ queue_capacity: capacity(),
+ }
+ }
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ (1..=1_000_000).contains(&self.max_pending_evaluations),
+ "Invalid evaluation backlog limit"
+ );
+ ensure!(
+ !self.deployment.is_empty()
+ && self.deployment.len() <= 64
+ && self
+ .deployment
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),
+ "Invalid recording deployment"
+ );
+ ensure!(
+ (1..=365).contains(&self.retention_days) && (1..=2048).contains(&self.queue_capacity),
+ "Invalid recording retention or queue capacity"
+ );
+ ensure!(
+ (1..=3650).contains(&self.evaluation_retention_days),
+ "Invalid evaluation retention"
+ );
+ ensure!(
+ self.evaluation_sinks.len() <= 8,
+ "At most eight evaluation destinations"
+ );
+ let mut names = std::collections::BTreeSet::new();
+ for sink in &self.evaluation_sinks {
+ sink.validate()?;
+ ensure!(
+ names.insert(&sink.name),
+ "Duplicate evaluation destination name"
+ );
+ }
+ if let Some(p) = &self.evidence {
+ anyhow::ensure!(
+ (256..=65536).contains(&p.max_bytes),
+ "Evidence byte limit must be 256..65536"
+ );
+ }
+ match &self.database {
+ RecordingDatabase::Sqlite { path } => ensure!(
+ path.is_absolute() && path.file_name().is_some(),
+ "Recording path must be an absolute file path"
+ ),
+ RecordingDatabase::Postgres {
+ connection_env,
+ schema,
+ ca_file,
+ ..
+ } => {
+ ensure!(
+ !connection_env.is_empty()
+ && connection_env.len() <= 100
+ && connection_env
+ .bytes()
+ .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
+ "Invalid recording credential reference"
+ );
+ ensure!(
+ schema.starts_with("ow_")
+ && schema.len() <= 48
+ && schema
+ .bytes()
+ .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'),
+ "Invalid recording schema"
+ );
+ ensure!(
+ ca_file.as_ref().is_none_or(|p| p.is_absolute()),
+ "Recording CA path must be absolute"
+ );
+ }
+ }
+ Ok(())
+ }
+}
+
+/// A trusted SDK query. Service deployments apply their own session authorization.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TraceQuery {
+ pub session_id: String,
+ pub turn_id: String,
+ #[serde(default)]
+ pub after: u64,
+ pub through: Option<u64>,
+ #[serde(default = "page_size")]
+ pub limit: u16,
+}
+fn page_size() -> u16 {
+ 100
+}
+impl TraceQuery {
+ pub fn turn(session_id: impl Into<String>, turn_id: impl Into<String>) -> Self {
+ Self {
+ session_id: session_id.into(),
+ turn_id: turn_id.into(),
+ after: 0,
+ through: None,
+ limit: page_size(),
+ }
+ }
+ pub fn validate(&self) -> Result<()> {
+ for id in [&self.session_id, &self.turn_id] {
+ ensure!(
+ !id.is_empty()
+ && id.len() <= 80
+ && id
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),
+ "Invalid trace scope"
+ );
+ }
+ ensure!(
+ (1..=200).contains(&self.limit)
+ && self.after <= i64::MAX as u64
+ && self
+ .through
+ .is_none_or(|n| n >= self.after && n <= i64::MAX as u64),
+ "Invalid trace page"
+ );
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn configuration_is_explicit_bounded_and_secret_free() {
+ let path = std::env::temp_dir().join("ow-observability-contract.sqlite");
+ let mut c = RecordingConfig::sqlite(path);
+ assert!(c.validate().is_ok());
+ c.queue_capacity = 0;
+ assert!(c.validate().is_err());
+ assert!(serde_json::from_value::<RecordingConfig>(serde_json::json!({"database":{"kind":"postgres","connection":"secret","connection_env":"PG","schema":"ow_obs"}})).is_err());
+ }
+ #[test]
+ fn trace_query_rejects_raw_hash_scope_and_unbounded_pages() {
+ let mut q = TraceQuery::turn("session-1", "turn-2");
+ assert!(q.validate().is_ok());
+ q.limit = 201;
+ assert!(q.validate().is_err());
+ assert!(serde_json::from_value::<TraceQuery>(
+ serde_json::json!({"session_hash":"other","session_id":"s","turn_id":"t"})
+ )
+ .is_err());
+ }
+}
+
+/// Query adapter shared by embedded and managed-process SDK clients.
+#[derive(Clone)]
+pub struct Observability {
+ client: crate::HarnessClient,
+}
+impl Observability {
+ pub(crate) fn new(client: crate::HarnessClient) -> Self {
+ Self { client }
+ }
+ #[cfg(feature = "evaluations")]
+ pub async fn record_evaluation(
+ &self,
+ query: TraceQuery,
+ result: openworker_observability::evaluations::Input,
+ span_id: Option<String>,
+ ) -> crate::Result<serde_json::Value> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ self.client.request("observability/evaluation-record",serde_json::json!({"session_id":query.session_id,"turn_id":query.turn_id,"span_id":span_id,"result":result})).await
+ }
+ #[cfg(feature = "evaluations")]
+ pub async fn evaluations(
+ &self,
+ query: TraceQuery,
+ ) -> crate::Result<openworker_observability::evaluations::Page> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ let value = self
+ .client
+ .request(
+ "observability/evaluation-list",
+ serde_json::to_value(query)
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,
+ )
+ .await?;
+ serde_json::from_value(value).map_err(|e| crate::Error::Remote(e.to_string()))
+ }
+ pub async fn flush(&self, timeout_ms: u64) -> crate::Result<serde_json::Value> {
+ self.client
+ .request(
+ "observability/flush",
+ serde_json::json!({"timeout_ms":timeout_ms}),
+ )
+ .await
+ }
+ pub async fn health(&self) -> crate::Result<serde_json::Value> {
+ self.client
+ .request("observability/health", serde_json::json!({}))
+ .await
+ }
+ pub async fn turns(
+ &self,
+ session_id: &str,
+ after: i64,
+ limit: u16,
+ ) -> crate::Result<serde_json::Value> {
+ self.client
+ .request(
+ "observability/turns",
+ serde_json::json!({"session_id":session_id,"after":after,"limit":limit}),
+ )
+ .await
+ }
+ pub async fn subtree_summary(&self, query: TraceQuery) -> crate::Result<serde_json::Value> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ self.client
+ .request(
+ "observability/subtree-summary",
+ serde_json::to_value(query)
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,
+ )
+ .await
+ }
+ pub async fn children(&self, query: TraceQuery) -> crate::Result<serde_json::Value> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ self.client
+ .request(
+ "observability/children",
+ serde_json::to_value(query)
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,
+ )
+ .await
+ }
+ #[cfg(feature = "recording")]
+ pub async fn timeline(&self, query: TraceQuery) -> crate::Result<Timeline> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ let v = self
+ .client
+ .request(
+ "observability/timeline",
+ serde_json::to_value(query)
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,
+ )
+ .await?;
+ serde_json::from_value(v).map_err(|e| crate::Error::Remote(e.to_string()))
+ }
+ #[cfg(feature = "recording")]
+ pub async fn summary(&self, query: TraceQuery) -> crate::Result<TurnSummary> {
+ query
+ .validate()
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;
+ let v = self
+ .client
+ .request(
+ "observability/summary",
+ serde_json::to_value(query)
+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,
+ )
+ .await?;
+ serde_json::from_value(v).map_err(|e| crate::Error::Remote(e.to_string()))
+ }
+}
+
+/// Local evidence access is a separate opt-in from remote telemetry capture.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct EvidencePolicy {
+ #[serde(default)]
+ pub user_input: bool,
+ #[serde(default)]
+ pub assistant_output: bool,
+ #[serde(default)]
+ pub tool_arguments: bool,
+ #[serde(default)]
+ pub tool_results: bool,
+ pub max_bytes: usize,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct EvaluationSink {
+ pub name: String,
+ pub url: String,
+ pub authorization_env: String,
+ #[serde(default)]
+ pub include_explanation: bool,
+}
+impl EvaluationSink {
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ !self.name.is_empty()
+ && self.name.len() <= 64
+ && self
+ .name
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),
+ "Invalid evaluation destination name"
+ );
+ ensure!(
+ !self.authorization_env.is_empty()
+ && self.authorization_env.len() <= 100
+ && self
+ .authorization_env
+ .bytes()
+ .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
+ "Invalid evaluation credential reference"
+ );
+ let u = reqwest::Url::parse(&self.url)?;
+ ensure!(
+ u.username().is_empty()
+ && u.password().is_none()
+ && u.query().is_none()
+ && u.fragment().is_none()
+ && u.path() == "/"
+ && (u.scheme() == "https"
+ || (u.scheme() == "http"
+ && u.host_str().is_some_and(|h| h == "localhost"
+ || h.trim_matches(['[', ']'])
+ .parse::<std::net::IpAddr>()
+ .is_ok_and(|ip| ip.is_loopback())))),
+ "Evaluation destination must be HTTPS or explicit loopback origin"
+ );
+ Ok(())
+ }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RunIdentity {
+ pub session_id: String,
+ pub turn_id: String,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RecordedTurn {
+ pub session_id: String,
+ pub turn_id: String,
+ pub status: Option<String>,
+ pub agent: String,
+ pub name: String,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct TurnPage {
+ pub turns: Vec<RecordedTurn>,
+ pub next_cursor: Option<i64>,
+ pub through: i64,
+}
+#[cfg(feature = "recording")]
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Timeline {
+ pub run: RunIdentity,
+ pub page: openworker_observability::query::Page,
+ pub coverage: serde_json::Value,
+}
+#[cfg(feature = "recording")]
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TurnSummary {
+ pub run: RunIdentity,
+ pub summary: openworker_observability::summary::Summary,
+}
diff --git a/harness/harness-sdk/src/service.rs b/harness/harness-sdk/src/service.rs
index 23191598..b81adfb4 100644
--- a/harness/harness-sdk/src/service.rs
+++ b/harness/harness-sdk/src/service.rs
@@ -19,6 +19,12 @@ pub use openworker_runtime::coding_handoff::config::{
pub use openworker_runtime::coding_handoff::ObjectFormat as GitObjectFormat;
pub use openworker_runtime::team_definition::Sandbox as TeamSandbox;
+#[cfg(feature = "recording")]
+#[path = "service_observability.rs"]
+mod observability;
+#[cfg(feature = "recording")]
+pub use observability::Observability;
+
pub fn new_idempotency_key() -> String {
openworker_runtime::id("idem")
}
diff --git a/harness/harness-sdk/src/service_observability.rs b/harness/harness-sdk/src/service_observability.rs
new file mode 100644
index 00000000..5f62b9e6
--- /dev/null
+++ b/harness/harness-sdk/src/service_observability.rs
@@ -0,0 +1,51 @@
+//! Typed read-only service queries. Authorization remains on the service.
+use super::{error, Result, Service};
+use crate::recording::{Timeline, TurnSummary};
+use crate::TraceQuery;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+
+pub struct Observability<'a> {
+ service: &'a Service,
+}
+impl Service {
+ pub fn observability(&self) -> Observability<'_> {
+ Observability { service: self }
+ }
+}
+impl Observability<'_> {
+ async fn read<T: DeserializeOwned>(&self, query: TraceQuery, kind: &str) -> Result<T> {
+ query.validate().map_err(|_| error("InvalidInput"))?;
+ let mut path = format!(
+ "/v1/sessions/{}/observability/{}/{}",
+ query.session_id, query.turn_id, kind
+ );
+ let mut params = Vec::new();
+ if kind == "timeline" {
+ params.push(format!("after={}&limit={}", query.after, query.limit));
+ }
+ if kind != "children" {
+ if let Some(through) = query.through {
+ params.push(format!("through={through}"));
+ }
+ }
+ if !params.is_empty() {
+ path.push('?');
+ path.push_str(&params.join("&"));
+ }
+ let value = self.service.json("GET", &path, None, None).await?;
+ serde_json::from_value(value).map_err(|_| error("InvalidResponse"))
+ }
+ pub async fn timeline(&self, query: TraceQuery) -> Result<Timeline> {
+ self.read(query, "timeline").await
+ }
+ pub async fn summary(&self, query: TraceQuery) -> Result<TurnSummary> {
+ self.read(query, "summary").await
+ }
+ pub async fn children(&self, query: TraceQuery) -> Result<Value> {
+ self.read(query, "children").await
+ }
+ pub async fn subtree_summary(&self, query: TraceQuery) -> Result<Value> {
+ self.read(query, "subtree-summary").await
+ }
+}
diff --git a/harness/harness-sdk/tests/agents_regressions.rs b/harness/harness-sdk/tests/agents_regressions.rs
index 536b3035..b405921c 100644
--- a/harness/harness-sdk/tests/agents_regressions.rs
+++ b/harness/harness-sdk/tests/agents_regressions.rs
@@ -93,6 +93,7 @@ async fn exercise(postgres: bool, process: bool) {
sandbox: Sandbox::Off,
scripted: true,
telemetry: None,
+ recording: None,
agents: vec![
Agent::new("assistant")
.instructions("Use tools.")
diff --git a/harness/harness-sdk/tests/dynamic_agents.rs b/harness/harness-sdk/tests/dynamic_agents.rs
index 39f42dc0..5c15538e 100644
--- a/harness/harness-sdk/tests/dynamic_agents.rs
+++ b/harness/harness-sdk/tests/dynamic_agents.rs
@@ -51,6 +51,7 @@ async fn exercise(postgres: bool, process: bool) {
sandbox: Sandbox::Off,
scripted: true,
telemetry: None,
+ recording: None,
agents: vec![],
};
let h = connect(&config, process).await;
@@ -244,6 +245,7 @@ async fn live_configuration_requires_explicit_model_without_calling_provider() {
sandbox: Sandbox::Off,
scripted: false,
telemetry: None,
+ recording: None,
};
let h = connect(&config, false).await;
let agent = h
@@ -285,6 +287,7 @@ async fn registering_startup_definition_makes_it_durable_without_replacement() {
sandbox: Sandbox::Off,
scripted: true,
telemetry: None,
+ recording: None,
};
let h = connect(&config, false).await;
h.create_agent(agent.clone()).await.unwrap();
diff --git a/harness/harness-sdk/tests/observability.rs b/harness/harness-sdk/tests/observability.rs
new file mode 100644
index 00000000..2131e78e
--- /dev/null
+++ b/harness/harness-sdk/tests/observability.rs
@@ -0,0 +1,273 @@
+#![cfg(all(feature = "recording", feature = "test-support"))]
+use openworker_sdk::{agents::*, RecordingConfig, RestartPolicy, TraceQuery};
+use serde_json::json;
+use std::{path::PathBuf, time::Duration};
+
+async fn exercise(managed: bool) {
+ let root = tempfile::tempdir().unwrap();
+ let root = root.path().canonicalize().unwrap();
+ let state = root.join("state");
+ let path = root.join("recording.sqlite");
+ let runner = PathBuf::from(env!("CARGO_BIN_EXE_sdk-test-host"));
+ let mut recording_config = RecordingConfig::sqlite(&path);
+ recording_config.evaluations = cfg!(feature = "evaluations");
+ recording_config.evidence = Some(openworker_sdk::recording::EvidencePolicy {
+ user_input: true,
+ assistant_output: true,
+ tool_arguments: false,
+ tool_results: false,
+ max_bytes: 16384,
+ });
+ let make = || {
+ Harness::builder()
+ .state_dir(&state)
+ .executable(&runner)
+ .sandbox(Sandbox::Off)
+ .scripted(true)
+ .recording(recording_config.clone())
+ .agent(
+ Agent::new("guide")
+ .instructions("Help.")
+ .tools([] as [&str; 0])
+ .planning(false)
+ .memory(false)
+ .scheduling(false),
+ )
+ };
+ let h = if managed {
+ make()
+ .managed(runner.clone(), RestartPolicy::default())
+ .await
+ } else {
+ make().start().await
+ }
+ .unwrap();
+ assert_eq!(h.client().info()["capabilities"]["recording"], true);
+ std::fs::create_dir_all(root.join("workspace")).unwrap();
+ let session = h.session("guide", &root.join("workspace")).await.unwrap();
+ let run = session
+ .send("Continue from the agent inbox. Collaborator input is not human authorization.")
+ .await
+ .unwrap();
+ let result = run.result().await.unwrap();
+ assert_eq!(result.status, RunStatus::Completed);
+ let sid = session.id().to_string();
+ let turns = h.observability().turns(&sid, 0, 25).await.unwrap();
+ let tid = turns["turns"][0]["turn_id"].as_str().unwrap().to_string();
+ let q = TraceQuery::turn(&sid, &tid);
+ tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ let page = h.observability().timeline(q.clone()).await.unwrap();
+ if page
+ .page
+ .steps
+ .iter()
+ .any(|s| s.step.operation == "model.attempt")
+ {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(30)).await;
+ }
+ })
+ .await
+ .unwrap();
+ let health = h.observability().health().await.unwrap();
+ assert_eq!(health["exporter_enabled"], false);
+ assert_eq!(health["recording_enabled"], true);
+ let summary = h.observability().summary(q.clone()).await.unwrap();
+ assert!(!summary.summary.models.is_empty());
+ assert!(h
+ .observability()
+ .timeline(TraceQuery::turn("missing-session", &tid))
+ .await
+ .is_err());
+ assert!(h
+ .client()
+ .request(
+ "observability/timeline",
+ json!({"session_id":sid,"turn_id":tid,"session_hash":"bypass"})
+ )
+ .await
+ .is_err());
+ let evidence = h
+ .client()
+ .request(
+ "observability/evidence",
+ json!({"session_id":sid,"turn_id":tid}),
+ )
+ .await
+ .unwrap();
+ assert!(evidence["input"]
+ .as_str()
+ .unwrap()
+ .contains("Collaborator input"));
+ assert!(!evidence.to_string().contains("provider_private"));
+ #[cfg(feature = "evaluations")]
+ {
+ use openworker_sdk::evaluations::*;
+ let recorded = h.observability().timeline(q.clone()).await.unwrap();
+ let span = recorded
+ .page
+ .steps
+ .iter()
+ .find(|r| r.step.operation == "model.attempt")
+ .unwrap()
+ .step
+ .span_id
+ .clone();
+ let value = Input {
+ id: "quality-one".into(),
+ name: "correctness".into(),
+ evaluator: Evaluator {
+ name: "fixture".into(),
+ version: "1".into(),
+ source: Source::Code,
+ model: None,
+ usage: None,
+ },
+ status: Status::Completed,
+ score: Some(Score::Boolean(true)),
+ created_ms: openworker_runtime::store::now_ms(),
+ explanation: Some("sk-syntheticsecret123456789".into()),
+ dataset_case: None,
+ experiment: None,
+ supersedes: None,
+ };
+ let result = h
+ .observability()
+ .record_evaluation(q.clone(), value.clone(), Some(span.clone()))
+ .await
+ .unwrap();
+ assert_eq!(result["duplicate"], false);
+ assert!(!result.to_string().contains("syntheticsecret"));
+ assert_eq!(
+ h.observability()
+ .record_evaluation(q.clone(), value.clone(), Some(span.clone()))
+ .await
+ .unwrap()["duplicate"],
+ true
+ );
+ assert!(h
+ .observability()
+ .record_evaluation(
+ TraceQuery::turn(&sid, "missing-turn"),
+ value.clone(),
+ Some(span.clone())
+ )
+ .await
+ .is_err());
+ h.observability().flush(5000).await.unwrap();
+ rusqlite::Connection::open(&path)
+ .unwrap()
+ .execute("DELETE FROM ow_observation_steps WHERE span_id=?", [&span])
+ .unwrap();
+ assert_eq!(
+ h.observability()
+ .record_evaluation(q.clone(), value.clone(), Some(span))
+ .await
+ .unwrap()["duplicate"],
+ true
+ );
+ let mut read = q.clone();
+ read.limit = 25;
+ assert_eq!(
+ h.observability()
+ .evaluations(read)
+ .await
+ .unwrap()
+ .results
+ .len(),
+ 1
+ );
+ }
+ h.shutdown().await.unwrap();
+ let reopened = make().start().await.unwrap();
+ let page = reopened.observability().timeline(q).await.unwrap();
+ assert!(!page.page.steps.is_empty());
+ assert!(!serde_json::to_string(&page)
+ .unwrap()
+ .contains("Collaborator input"));
+ reopened.shutdown().await.unwrap();
+}
+#[tokio::test]
+async fn embedded_records_queries_and_reopens() {
+ exercise(false).await;
+}
+#[tokio::test]
+async fn managed_records_queries_and_reopens() {
+ exercise(true).await;
+}
+#[tokio::test]
+async fn disabled_recording_does_not_create_a_database() {
+ let root = tempfile::tempdir().unwrap();
+ let state = root.path().join("state");
+ let h = Harness::builder()
+ .state_dir(&state)
+ .executable(env!("CARGO_BIN_EXE_sdk-test-host"))
+ .sandbox(Sandbox::Off)
+ .start()
+ .await
+ .unwrap();
+ assert_eq!(
+ h.observability().health().await.unwrap()["recording_enabled"],
+ false
+ );
+ assert!(h
+ .observability()
+ .timeline(TraceQuery::turn("s", "t"))
+ .await
+ .is_err());
+ assert!(!state.join("recording.sqlite").exists());
+ h.shutdown().await.unwrap();
+}
+
+#[cfg(feature = "recording-postgres")]
+#[tokio::test]
+#[ignore = "disposable loopback PostgreSQL fixture required"]
+async fn postgres_recording_is_independent_from_sqlite_execution() {
+ let root = tempfile::tempdir().unwrap();
+ let schema = format!("ow_sdk_obs_{}", std::process::id());
+ let config = RecordingConfig {
+ evaluations: false,
+ max_pending_evaluations: 10_000,
+ evaluation_retention_days: 30,
+ evidence: None,
+ evaluation_sinks: vec![],
+ database: openworker_sdk::RecordingDatabase::Postgres {
+ connection_env: "OW_OBSERVABILITY_TEST_URL".into(),
+ schema: schema.clone(),
+ ca_file: None,
+ loopback_plaintext: true,
+ },
+ deployment: "sdk".into(),
+ retention_days: 7,
+ queue_capacity: 16,
+ };
+ config.migrate_postgres().await.unwrap();
+ let h = Harness::builder()
+ .state_dir(root.path().join("state"))
+ .executable(env!("CARGO_BIN_EXE_sdk-test-host"))
+ .sandbox(Sandbox::Off)
+ .recording(config)
+ .start()
+ .await
+ .unwrap();
+ assert_eq!(
+ h.observability().health().await.unwrap()["recording_enabled"],
+ true
+ );
+ assert!(root.path().join("state/runtime.sqlite3").exists());
+ h.shutdown().await.unwrap();
+ tokio::task::spawn_blocking(move || {
+ let mut client = postgres::Client::connect(
+ &std::env::var("OW_OBSERVABILITY_TEST_URL").unwrap(),
+ postgres::NoTls,
+ )
+ .unwrap();
+ client
+ .batch_execute(&format!("DROP SCHEMA {schema} CASCADE"))
+ .unwrap();
+ })
+ .await
+ .unwrap();
+}
diff --git a/harness/harness-sdk/tests/standalone_subagents.rs b/harness/harness-sdk/tests/standalone_subagents.rs
index f4f29eda..86d88918 100644
--- a/harness/harness-sdk/tests/standalone_subagents.rs
+++ b/harness/harness-sdk/tests/standalone_subagents.rs
@@ -32,6 +32,10 @@ async fn scenario(managed: bool, execution: &str, approval: u8) {
})
.allow_subagents(true),
);
+ #[cfg(feature = "recording")]
+ let builder = builder.recording(openworker_sdk::RecordingConfig::sqlite(
+ root.join("traces.sqlite"),
+ ));
let h = if managed {
builder
.managed(executable, Default::default())
@@ -91,6 +95,31 @@ async fn scenario(managed: bool, execution: &str, approval: u8) {
.await
.unwrap();
assert_eq!(run.status, RunStatus::Completed, "{run:?} {events}");
+ #[cfg(feature = "recording")]
+ {
+ let turns = h.observability().turns(parent.id(), 0, 25).await.unwrap();
+ let tid = turns["turns"][0]["turn_id"].as_str().unwrap();
+ let children = h
+ .observability()
+ .children(openworker_sdk::TraceQuery::turn(parent.id(), tid))
+ .await
+ .unwrap();
+ assert_eq!(children["children"].as_array().unwrap().len(), 1);
+ assert_eq!(children["children"][0]["agent_id"], "reader");
+ assert_eq!(children["children"][0]["origin"]["parent_turn_id"], tid);
+ assert!(children["children"][0]["origin"]["parent_call_id"].is_string());
+ assert!(!children.to_string().contains("CHILD_FILE_EVIDENCE"));
+ assert!(children["children"][0]["child_turn_id"].is_string());
+ let tree = h
+ .observability()
+ .subtree_summary(openworker_sdk::TraceQuery::turn(parent.id(), tid))
+ .await
+ .unwrap();
+ assert_eq!(tree["selection"]["runs"].as_array().unwrap().len(), 2);
+ assert_eq!(tree["selection"]["selection_truncated"], false);
+ assert!(tree["summary"]["wall_elapsed_us"].is_null());
+ }
+
let doc = h
.client()
.request(
diff --git a/harness/harness-sdk/tests/storage_teams.rs b/harness/harness-sdk/tests/storage_teams.rs
index c1c625df..95ce690a 100644
--- a/harness/harness-sdk/tests/storage_teams.rs
+++ b/harness/harness-sdk/tests/storage_teams.rs
@@ -106,6 +106,7 @@ async fn exercise(postgres: bool, process: bool) {
sandbox: Sandbox::Off,
scripted: true,
telemetry: None,
+ recording: None,
agents: vec![],
};
let c = client(&config, process).await;
diff --git a/harness/observability/Cargo.toml b/harness/observability/Cargo.toml
index 19bcb4ca..492c96cc 100644
--- a/harness/observability/Cargo.toml
+++ b/harness/observability/Cargo.toml
@@ -1,18 +1,23 @@
[package]
name = "openworker-observability"
-version = "0.1.0"
+version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
-publish = false
+publish = ["crates-io"]
+readme = "README.md"
+repository = "https://github.com/andrewyng/openworker"
+description = "Optional bounded recording, trace queries and evaluation storage for OpenWorker"
[features]
default = []
+evaluations = ["dep:openworker-compliance"]
postgres = ["dep:postgres", "dep:postgres-native-tls", "dep:native-tls", "dep:rustls-pemfile", "dep:zeroize"]
otlp = ["openworker-telemetry/otlp"]
[dependencies]
-openworker-telemetry = { path = "../telemetry" }
+openworker-compliance = { path = "../compliance", version = "=0.1.0-beta.2", optional = true }
+openworker-telemetry = { path = "../telemetry", version = "=0.1.0-beta.2" }
anyhow = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
diff --git a/harness/observability/LICENSE b/harness/observability/LICENSE
new file mode 100644
index 00000000..60d38246
--- /dev/null
+++ b/harness/observability/LICENSE
@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2024 Andrew Ng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/harness/observability/src/evaluations.rs b/harness/observability/src/evaluations.rs
new file mode 100644
index 00000000..46c7ccf1
--- /dev/null
+++ b/harness/observability/src/evaluations.rs
@@ -0,0 +1,295 @@
+//! Optional, acknowledged evaluation records. Never sent through the lossy trace queue.
+use anyhow::{ensure, Result};
+pub const DEFAULT_MAX_PENDING: u32 = 10_000;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Target {
+ pub session_hash: String,
+ pub turn_hash: String,
+ pub trace_id: Option<String>,
+ pub span_id: Option<String>,
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Evaluator {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub model: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub usage: Option<crate::Usage>,
+ pub name: String,
+ pub version: String,
+ pub source: Source,
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Source {
+ Human,
+ Code,
+ Model,
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(
+ tag = "type",
+ content = "value",
+ rename_all = "snake_case",
+ deny_unknown_fields
+)]
+pub enum Score {
+ Numeric(f64),
+ Boolean(bool),
+ Categorical(String),
+ Text(String),
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Status {
+ Completed,
+ Skipped,
+ Error,
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Input {
+ pub id: String,
+ pub name: String,
+ pub evaluator: Evaluator,
+ pub status: Status,
+ pub score: Option<Score>,
+ pub created_ms: u64,
+ pub explanation: Option<String>,
+ pub dataset_case: Option<String>,
+ pub experiment: Option<String>,
+ /// Corrections append a new ID, preserving the original evaluation.
+ pub supersedes: Option<String>,
+}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Evaluation {
+ pub version: u8,
+ pub deployment: String,
+ pub target: Target,
+ pub result: Input,
+}
+impl Evaluation {
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ self.version == 1 && crate::identifier(&self.deployment),
+ "Unsupported evaluation schema or deployment"
+ );
+ ensure!(
+ crate::hex_id(&self.target.session_hash, 64)
+ && crate::hex_id(&self.target.turn_hash, 64),
+ "Invalid evaluation scope"
+ );
+ ensure!(
+ self.target
+ .trace_id
+ .as_ref()
+ .is_none_or(|v| crate::hex_id(v, 32))
+ && self
+ .target
+ .span_id
+ .as_ref()
+ .is_none_or(|v| crate::hex_id(v, 16))
+ && (self.target.span_id.is_none() || self.target.trace_id.is_some()),
+ "Invalid evaluation trace target"
+ );
+ for id in [
+ &self.result.id,
+ &self.result.name,
+ &self.result.evaluator.name,
+ &self.result.evaluator.version,
+ ]
+ .into_iter()
+ .chain(self.result.dataset_case.iter())
+ .chain(self.result.experiment.iter())
+ .chain(self.result.supersedes.iter())
+ {
+ ensure!(crate::label(id), "Invalid evaluation identifier");
+ }
+ ensure!(
+ self.result
+ .evaluator
+ .model
+ .as_ref()
+ .is_none_or(|m| crate::label(m)),
+ "Invalid evaluator model"
+ );
+ ensure!(
+ (self.result.evaluator.model.is_none() && self.result.evaluator.usage.is_none())
+ || self.result.evaluator.source == Source::Model,
+ "Only model evaluators carry model accounting"
+ );
+ if let Some(u) = &self.result.evaluator.usage {
+ ensure!(
+ openworker_telemetry::TokenUsage {
+ input: u.input,
+ output: u.output,
+ cache_read: u.cache_read,
+ cache_write: u.cache_write,
+ reasoning: u.reasoning
+ }
+ .validate(),
+ "Invalid evaluator usage"
+ );
+ }
+ ensure!(
+ self.result.created_ms > 0 && self.result.created_ms <= 253402300799000,
+ "Invalid evaluation timestamp"
+ );
+ ensure!(
+ (self.result.status == Status::Completed) == self.result.score.is_some(),
+ "Only completed evaluations carry a score"
+ );
+ ensure!(
+ self.result.supersedes.as_ref() != Some(&self.result.id),
+ "An evaluation cannot supersede itself"
+ );
+ if let Some(score) = &self.result.score {
+ match score {
+ Score::Numeric(n) => ensure!(n.is_finite(), "Numeric score must be finite"),
+ Score::Categorical(s) => {
+ ensure!(!s.is_empty() && s.len() <= 256, "Invalid category")
+ }
+ Score::Text(s) => ensure!(s.len() <= 4000, "Score text exceeds limit"),
+ Score::Boolean(_) => {}
+ }
+ }
+ ensure!(
+ self.result
+ .explanation
+ .as_ref()
+ .is_none_or(|s| s.len() <= 4000),
+ "Evaluation explanation exceeds limit"
+ );
+ ensure!(
+ serde_json::to_vec(self)?.len() <= 16384,
+ "Evaluation exceeds record limit"
+ );
+ Ok(())
+ }
+ /// Best-effort secret redaction precedes durable storage and export.
+ pub fn redacted(mut self) -> Self {
+ if let Some(text) = &mut self.result.explanation {
+ *text = openworker_telemetry::redact(text);
+ }
+ if let Some(Score::Text(text) | Score::Categorical(text)) = &mut self.result.score {
+ *text = openworker_telemetry::redact(text);
+ }
+ self
+ }
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Row {
+ pub cursor: u64,
+ pub evaluation: Evaluation,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Page {
+ pub results: Vec<Row>,
+ pub next: Option<u64>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Query {
+ pub deployment: String,
+ pub session_hash: String,
+ pub turn_hash: String,
+ pub after: u64,
+ pub limit: u16,
+}
+impl Query {
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ crate::identifier(&self.deployment)
+ && crate::hex_id(&self.session_hash, 64)
+ && crate::hex_id(&self.turn_hash, 64)
+ && self.after <= i64::MAX as u64
+ && (1..=50).contains(&self.limit),
+ "Invalid evaluation query"
+ );
+ Ok(())
+ }
+}
+pub(crate) fn page(mut results: Vec<Row>, q: &Query) -> Result<Page> {
+ for row in &results {
+ row.evaluation.validate()?;
+ ensure!(
+ row.cursor > q.after
+ && row.evaluation.deployment == q.deployment
+ && row.evaluation.target.session_hash == q.session_hash
+ && row.evaluation.target.turn_hash == q.turn_hash,
+ "Evaluation scope mismatch"
+ );
+ }
+ let more = results.len() > usize::from(q.limit);
+ results.truncate(q.limit.into());
+ let next = more.then(|| results.last().unwrap().cursor);
+ Ok(Page { results, next })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ pub fn sample() -> Evaluation {
+ Evaluation {
+ version: 1,
+ deployment: "sdk".into(),
+ target: Target {
+ session_hash: format!("{:064x}", 1),
+ turn_hash: format!("{:064x}", 2),
+ trace_id: None,
+ span_id: None,
+ },
+ result: Input {
+ id: "e1".into(),
+ name: "correctness".into(),
+ evaluator: Evaluator {
+ name: "fixture".into(),
+ version: "1".into(),
+ source: Source::Code,
+ model: None,
+ usage: None,
+ },
+ status: Status::Completed,
+ score: Some(Score::Boolean(true)),
+ created_ms: 1,
+ explanation: None,
+ dataset_case: None,
+ experiment: None,
+ supersedes: None,
+ },
+ }
+ }
+ #[test]
+ fn status_and_target_cannot_misrepresent_results() {
+ let mut e = sample();
+ assert!(e.validate().is_ok());
+ e.result.status = Status::Error;
+ assert!(e.validate().is_err());
+ e.result.score = None;
+ assert!(e.validate().is_ok());
+ e.target.span_id = Some(format!("{:016x}", 1));
+ assert!(e.validate().is_err());
+ }
+}
+
+/// Reuses the compliance outbox lease/backoff contract; only evaluation delivery
+/// is retried. No runtime action is addressed by this checkpoint.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Delivery {
+ pub token: String,
+ pub row: Row,
+}
+pub fn delivery_key(deployment: &str, sink: &str, binding: &str, token: &str) -> Result<()> {
+ ensure!(
+ crate::identifier(deployment)
+ && crate::identifier(sink)
+ && crate::hex_id(binding, 64)
+ && crate::identifier(token),
+ "Invalid evaluation delivery scope"
+ );
+ Ok(())
+}
diff --git a/harness/observability/src/lib.rs b/harness/observability/src/lib.rs
index 60404c30..200a6194 100644
--- a/harness/observability/src/lib.rs
+++ b/harness/observability/src/lib.rs
@@ -4,6 +4,8 @@ use anyhow::{ensure, Result};
use openworker_telemetry::Record;
use serde::{Deserialize, Serialize};
use std::time::UNIX_EPOCH;
+#[cfg(feature = "evaluations")]
+pub mod evaluations;
#[cfg(feature = "postgres")]
pub mod postgres;
pub mod query;
@@ -340,6 +342,74 @@ pub struct StoredStep {
/// Trusted host interface only. A service must authorize the canonical session
/// before querying; the observation database is never an authorization authority.
pub trait Store: Send + 'static {
+ #[cfg(feature = "evaluations")]
+ fn register_evaluation_sink(
+ &mut self,
+ _deployment: &str,
+ _sink: &str,
+ _binding: &str,
+ ) -> Result<()> {
+ anyhow::bail!("Evaluation export unavailable")
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn prune_evaluations(
+ &mut self,
+ _deployment: &str,
+ _before_ms: u64,
+ _limit: u16,
+ ) -> Result<u64> {
+ anyhow::bail!("Evaluation pruning unsupported")
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn claim_evaluation_export(
+ &mut self,
+ _deployment: &str,
+ _sink: &str,
+ _binding: &str,
+ _token: &str,
+ _now: u64,
+ ) -> Result<Option<evaluations::Delivery>> {
+ anyhow::bail!("Evaluation export unavailable")
+ }
+ #[cfg(feature = "evaluations")]
+ fn settle_evaluation_export(
+ &mut self,
+ _deployment: &str,
+ _sink: &str,
+ _token: &str,
+ _now: u64,
+ _success: bool,
+ ) -> Result<()> {
+ anyhow::bail!("Evaluation export unavailable")
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn evaluation(
+ &mut self,
+ _deployment: &str,
+ _id: &str,
+ ) -> Result<Option<evaluations::Evaluation>> {
+ anyhow::bail!("Evaluation storage unavailable")
+ }
+ #[cfg(feature = "evaluations")]
+ fn record_evaluation(&mut self, value: &evaluations::Evaluation) -> Result<Write> {
+ self.record_evaluation_bounded(value, evaluations::DEFAULT_MAX_PENDING)
+ }
+ #[cfg(feature = "evaluations")]
+ fn record_evaluation_bounded(
+ &mut self,
+ _value: &evaluations::Evaluation,
+ _max_pending: u32,
+ ) -> Result<Write> {
+ anyhow::bail!("Evaluation storage unsupported")
+ }
+ #[cfg(feature = "evaluations")]
+ fn evaluations(&mut self, _query: &evaluations::Query) -> Result<evaluations::Page> {
+ anyhow::bail!("Evaluation storage unsupported")
+ }
+
/// Read-only bounded session/turn projection; no authorization authority.
fn timeline(&mut self, _query: &query::Query) -> Result<query::Page> {
anyhow::bail!("Recording queries unsupported by this adapter")
diff --git a/harness/observability/src/postgres.rs b/harness/observability/src/postgres.rs
index 4b902ec9..601af64e 100644
--- a/harness/observability/src/postgres.rs
+++ b/harness/observability/src/postgres.rs
@@ -136,6 +136,8 @@ impl Postgres {
INSERT INTO observation_version VALUES(TRUE,1);")?;
}
tx.batch_execute("CREATE INDEX IF NOT EXISTS observation_turn ON observation_steps(deployment,session_hash,(data->>'turn_hash'),cursor)")?;
+ #[cfg(feature="evaluations")]
+ tx.batch_execute("CREATE TABLE IF NOT EXISTS observation_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data JSONB NOT NULL,PRIMARY KEY(deployment,sink)); CREATE TABLE IF NOT EXISTS observation_evaluations(cursor BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,deployment TEXT NOT NULL,id TEXT NOT NULL,session_hash TEXT NOT NULL,turn_hash TEXT NOT NULL,data JSONB NOT NULL,recorded_ms BIGINT NOT NULL DEFAULT ((extract(epoch FROM clock_timestamp())*1000)::bigint),UNIQUE(deployment,id)); CREATE INDEX IF NOT EXISTS observation_evaluation_scope ON observation_evaluations(deployment,session_hash,turn_hash,cursor)")?;
tx.commit()?;
Ok(())
}
@@ -156,6 +158,223 @@ impl Postgres {
}
}
impl Store for Postgres {
+ #[cfg(feature = "evaluations")]
+ fn register_evaluation_sink(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ binding: &str,
+ ) -> Result<()> {
+ crate::evaluations::delivery_key(deployment, sink, binding, "registration")?;
+ let mut tx = self.client.transaction()?;
+ tx.execute(
+ "INSERT INTO observation_evaluation_delivery VALUES($1,$2,$3) ON CONFLICT DO NOTHING",
+ &[
+ &deployment,
+ &sink,
+ &serde_json::to_value(openworker_compliance::delivery::Checkpoint::new(
+ binding.into(),
+ ))?,
+ ],
+ )?;
+ let row=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;
+ let c: openworker_compliance::delivery::Checkpoint = serde_json::from_value(row.get(0))?;
+ ensure!(
+ c.binding == binding,
+ "Evaluation destination changed; choose a new sink name"
+ );
+ tx.commit()?;
+ Ok(())
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn prune_evaluations(&mut self, deployment: &str, before_ms: u64, limit: u16) -> Result<u64> {
+ crate::query(deployment, None, 0, limit)?;
+ ensure!(before_ms <= i64::MAX as u64, "Invalid cutoff");
+ Ok(self.client.execute("DELETE FROM observation_evaluations WHERE cursor IN(SELECT e.cursor FROM observation_evaluations e WHERE e.deployment=$1 AND e.recorded_ms<$2 AND NOT EXISTS(SELECT 1 FROM observation_evaluation_delivery d WHERE d.deployment=e.deployment AND (d.data->>'cursor')::bigint<e.cursor) ORDER BY e.cursor LIMIT $3)",&[&deployment,&(before_ms as i64),&i64::from(limit)])?)
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn claim_evaluation_export(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ binding: &str,
+ token: &str,
+ now: u64,
+ ) -> Result<Option<crate::evaluations::Delivery>> {
+ use openworker_compliance::delivery::Checkpoint;
+ crate::evaluations::delivery_key(deployment, sink, binding, token)?;
+ let mut tx = self.client.transaction()?;
+ tx.execute(
+ "INSERT INTO observation_evaluation_delivery VALUES($1,$2,$3) ON CONFLICT DO NOTHING",
+ &[
+ &deployment,
+ &sink,
+ &serde_json::to_value(Checkpoint::new(binding.into()))?,
+ ],
+ )?;
+ let raw=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;
+ let mut checkpoint: Checkpoint = serde_json::from_value(raw.get(0))?;
+ ensure!(
+ checkpoint.binding == binding,
+ "Export destination changed; choose a new sink name"
+ );
+ let row=tx.query_opt("SELECT cursor,data FROM observation_evaluations WHERE deployment=$1 AND cursor>$2 ORDER BY cursor LIMIT 1",&[&deployment,&checkpoint.cursor])?;
+ let Some(row) = row else {
+ tx.commit()?;
+ return Ok(None);
+ };
+ let cursor: i64 = row.get(0);
+ let claimed = checkpoint.claim(binding, token, now, cursor)?;
+ let evaluation: crate::evaluations::Evaluation = serde_json::from_value(row.get(1))?;
+ evaluation.validate()?;
+ ensure!(
+ evaluation.deployment == deployment,
+ "Evaluation export scope mismatch"
+ );
+ tx.execute(
+ "UPDATE observation_evaluation_delivery SET data=$1 WHERE deployment=$2 AND sink=$3",
+ &[&serde_json::to_value(checkpoint)?, &deployment, &sink],
+ )?;
+ tx.commit()?;
+ Ok(claimed.then(|| crate::evaluations::Delivery {
+ token: token.into(),
+ row: crate::evaluations::Row {
+ cursor: cursor as u64,
+ evaluation,
+ },
+ }))
+ }
+ #[cfg(feature = "evaluations")]
+ fn settle_evaluation_export(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ token: &str,
+ now: u64,
+ success: bool,
+ ) -> Result<()> {
+ let mut tx = self.client.transaction()?;
+ let raw=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;
+ let mut c: openworker_compliance::delivery::Checkpoint =
+ serde_json::from_value(raw.get(0))?;
+ c.settle(token, now, success)?;
+ tx.execute(
+ "UPDATE observation_evaluation_delivery SET data=$1 WHERE deployment=$2 AND sink=$3",
+ &[&serde_json::to_value(c)?, &deployment, &sink],
+ )?;
+ tx.commit()?;
+ Ok(())
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn evaluation(
+ &mut self,
+ deployment: &str,
+ id: &str,
+ ) -> Result<Option<crate::evaluations::Evaluation>> {
+ ensure!(
+ crate::identifier(deployment) && crate::label(id),
+ "Invalid evaluation identity"
+ );
+ self.client
+ .query_opt(
+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",
+ &[&deployment, &id],
+ )?
+ .map(|r| {
+ let e: crate::evaluations::Evaluation = serde_json::from_value(r.get(0))?;
+ e.validate()?;
+ ensure!(
+ e.deployment == deployment && e.result.id == id,
+ "Evaluation identity mismatch"
+ );
+ Ok(e)
+ })
+ .transpose()
+ }
+ #[cfg(feature = "evaluations")]
+ fn record_evaluation_bounded(
+ &mut self,
+ value: &crate::evaluations::Evaluation,
+ max_pending: u32,
+ ) -> Result<Write> {
+ ensure!(
+ (1..=1_000_000).contains(&max_pending),
+ "Invalid evaluation export backlog limit"
+ );
+ value.validate()?;
+ let value = value.clone().redacted();
+ value.validate()?;
+ let data = serde_json::to_value(&value)?;
+ if let Some(old) = self.evaluation(&value.deployment, &value.result.id)? {
+ ensure!(old == value, "Evaluation identity conflict");
+ return Ok(Write::Duplicate);
+ }
+ let mut tx = self.client.transaction()?;
+ tx.query_one(
+ "SELECT pg_advisory_xact_lock(hashtext($1))",
+ &[&format!("eval-capacity:{}", value.deployment)],
+ )?;
+ let retained: i64 = tx
+ .query_one(
+ "SELECT count(*) FROM observation_evaluations e WHERE deployment=$1 AND EXISTS(SELECT 1 FROM observation_evaluation_delivery d WHERE d.deployment=e.deployment AND (d.data->>'cursor')::bigint<e.cursor)",
+ &[&value.deployment],
+ )?
+ .get(0);
+ let exists:bool=tx.query_one("SELECT EXISTS(SELECT 1 FROM observation_evaluations WHERE deployment=$1 AND id=$2)",&[&value.deployment,&value.result.id])?.get(0);
+ ensure!(
+ exists || retained < i64::from(max_pending),
+ "Pending evaluation export limit reached; restore delivery or raise the configured limit"
+ );
+ if let Some(id) = &value.result.supersedes {
+ let prior = tx
+ .query_opt(
+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",
+ &[&value.deployment, id],
+ )?
+ .context("Superseded evaluation missing")?;
+ let prior: crate::evaluations::Evaluation = serde_json::from_value(prior.get(0))?;
+ ensure!(
+ prior.target == value.target,
+ "Correction must retain its target"
+ );
+ }
+ let inserted=tx.execute("INSERT INTO observation_evaluations(deployment,id,session_hash,turn_hash,data) VALUES($1,$2,$3,$4,$5) ON CONFLICT(deployment,id) DO NOTHING",&[&value.deployment,&value.result.id,&value.target.session_hash,&value.target.turn_hash,&data])?;
+ if inserted == 0 {
+ let old = tx.query_one(
+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",
+ &[&value.deployment, &value.result.id],
+ )?;
+ ensure!(
+ serde_json::from_value::<crate::evaluations::Evaluation>(old.get(0))? == value,
+ "Evaluation identity conflict"
+ );
+ }
+ tx.commit()?;
+ Ok(if inserted == 0 {
+ Write::Duplicate
+ } else {
+ Write::Inserted
+ })
+ }
+ #[cfg(feature = "evaluations")]
+ fn evaluations(&mut self, q: &crate::evaluations::Query) -> Result<crate::evaluations::Page> {
+ q.validate()?;
+ let rows=self.client.query("SELECT cursor,data FROM observation_evaluations WHERE deployment=$1 AND session_hash=$2 AND turn_hash=$3 AND cursor>$4 ORDER BY cursor LIMIT $5",&[&q.deployment,&q.session_hash,&q.turn_hash,&(q.after as i64),&(i64::from(q.limit)+1)])?;
+ let rows = rows
+ .into_iter()
+ .map(|r| {
+ Ok(crate::evaluations::Row {
+ cursor: r.get::<_, i64>(0) as u64,
+ evaluation: serde_json::from_value(r.get(1))?,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ crate::evaluations::page(rows, q)
+ }
+
fn timeline(&mut self, q: &crate::query::Query) -> Result<crate::query::Page> {
q.validate()?;
let sessions: Vec<_> = q.scopes.iter().map(|s| s.session_hash.as_str()).collect();
diff --git a/harness/observability/src/query.rs b/harness/observability/src/query.rs
index 26371133..a2c326a5 100644
--- a/harness/observability/src/query.rs
+++ b/harness/observability/src/query.rs
@@ -44,7 +44,8 @@ pub struct Page {
}
pub(crate) fn page(rows: Vec<(i64, Step)>, through: u64, q: &Query) -> Result<Page> {
let limit = q.limit;
- let more = rows.len() > usize::from(limit);
+ let mut more = rows.len() > usize::from(limit);
+ let mut bytes = 0usize;
let mut steps = vec![];
for (cursor, step) in rows.into_iter().take(limit.into()) {
ensure!(cursor > 0, "Invalid recording cursor");
@@ -57,6 +58,12 @@ pub(crate) fn page(rows: Vec<(i64, Step)>, through: u64, q: &Query) -> Result<Pa
&& step.turn_hash.as_ref() == Some(&s.turn_hash)),
"Recording scope mismatch"
);
+ let size = serde_json::to_vec(&step)?.len();
+ if q.limit <= 200 && !steps.is_empty() && bytes + size > 512_000 {
+ more = true;
+ break;
+ }
+ bytes += size;
steps.push(StoredStep {
cursor: cursor as u64,
step,
diff --git a/harness/observability/src/sqlite.rs b/harness/observability/src/sqlite.rs
index a9d0196d..3c168cb5 100644
--- a/harness/observability/src/sqlite.rs
+++ b/harness/observability/src/sqlite.rs
@@ -41,6 +41,7 @@ impl Sqlite {
| OpenFlags::SQLITE_OPEN_NOFOLLOW,
)?;
db.busy_timeout(Duration::from_millis(500))?;
+ ensure!(db.query_row("SELECT count(*) FROM sqlite_master WHERE type='table' AND name IN ('sessions','events','submissions')", [], |r| r.get::<_, i64>(0))? == 0, "Recording database must be separate from execution storage");
db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
CREATE TABLE IF NOT EXISTS ow_observation_version(singleton INTEGER PRIMARY KEY CHECK(singleton=1), version INTEGER NOT NULL);")?;
let version: Option<i64> = db
@@ -75,6 +76,299 @@ impl Sqlite {
}
}
impl Store for Sqlite {
+ #[cfg(feature = "evaluations")]
+ fn register_evaluation_sink(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ binding: &str,
+ ) -> Result<()> {
+ crate::evaluations::delivery_key(deployment, sink, binding, "registration")?;
+ let tx = self
+ .db
+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;
+ tx.execute(
+ "INSERT INTO ow_evaluation_delivery VALUES(?,?,?) ON CONFLICT DO NOTHING",
+ params![
+ deployment,
+ sink,
+ serde_json::to_string(&openworker_compliance::delivery::Checkpoint::new(
+ binding.into()
+ ))?
+ ],
+ )?;
+ let raw: String = tx.query_row(
+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",
+ params![deployment, sink],
+ |r| r.get(0),
+ )?;
+ let c: openworker_compliance::delivery::Checkpoint = serde_json::from_str(&raw)?;
+ ensure!(
+ c.binding == binding,
+ "Evaluation destination changed; choose a new sink name"
+ );
+ tx.commit()?;
+ Ok(())
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn prune_evaluations(&mut self, deployment: &str, before_ms: u64, limit: u16) -> Result<u64> {
+ crate::query(deployment, None, 0, limit)?;
+ ensure!(before_ms <= i64::MAX as u64, "Invalid cutoff");
+ if self.db.query_row(
+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",
+ [],
+ |r| r.get::<_, i64>(0),
+ )? == 0
+ {
+ return Ok(0);
+ }
+ self.db.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;
+ Ok(self.db.execute("DELETE FROM ow_observation_evaluations WHERE cursor IN(SELECT e.cursor FROM ow_observation_evaluations e WHERE e.deployment=?1 AND e.recorded_ms<?2 AND NOT EXISTS(SELECT 1 FROM ow_evaluation_delivery d WHERE d.deployment=e.deployment AND json_extract(d.data,'$.cursor')<e.cursor) ORDER BY e.cursor LIMIT ?3)",params![deployment,before_ms as i64,i64::from(limit)])? as u64)
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn claim_evaluation_export(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ binding: &str,
+ token: &str,
+ now: u64,
+ ) -> Result<Option<crate::evaluations::Delivery>> {
+ use openworker_compliance::delivery::Checkpoint;
+ crate::evaluations::delivery_key(deployment, sink, binding, token)?;
+ let tx = self
+ .db
+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;
+ let raw: Option<String> = tx
+ .query_row(
+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",
+ params![deployment, sink],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let mut checkpoint = raw
+ .map(|s| serde_json::from_str::<Checkpoint>(&s))
+ .transpose()?
+ .unwrap_or_else(|| Checkpoint::new(binding.into()));
+ ensure!(
+ checkpoint.binding == binding,
+ "Export destination changed; choose a new sink name"
+ );
+ tx.execute(
+ "INSERT INTO ow_evaluation_delivery VALUES(?,?,?) ON CONFLICT DO NOTHING",
+ params![deployment, sink, serde_json::to_string(&checkpoint)?],
+ )?;
+ if tx.query_row(
+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",
+ [],
+ |r| r.get::<_, i64>(0),
+ )? == 0
+ {
+ tx.commit()?;
+ return Ok(None);
+ }
+ let row:Option<(i64,String)>=tx.query_row("SELECT cursor,data FROM ow_observation_evaluations WHERE deployment=? AND cursor>? ORDER BY cursor LIMIT 1",params![deployment,checkpoint.cursor],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((cursor, data)) = row else {
+ tx.commit()?;
+ return Ok(None);
+ };
+ let claimed = checkpoint.claim(binding, token, now, cursor)?;
+ let evaluation: crate::evaluations::Evaluation = serde_json::from_str(&data)?;
+ evaluation.validate()?;
+ ensure!(
+ evaluation.deployment == deployment,
+ "Evaluation export scope mismatch"
+ );
+ tx.execute(
+ "UPDATE ow_evaluation_delivery SET data=? WHERE deployment=? AND sink=?",
+ params![serde_json::to_string(&checkpoint)?, deployment, sink],
+ )?;
+ tx.commit()?;
+ Ok(claimed.then(|| crate::evaluations::Delivery {
+ token: token.into(),
+ row: crate::evaluations::Row {
+ cursor: cursor as u64,
+ evaluation,
+ },
+ }))
+ }
+ #[cfg(feature = "evaluations")]
+ fn settle_evaluation_export(
+ &mut self,
+ deployment: &str,
+ sink: &str,
+ token: &str,
+ now: u64,
+ success: bool,
+ ) -> Result<()> {
+ let tx = self
+ .db
+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+ let raw: String = tx.query_row(
+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",
+ params![deployment, sink],
+ |r| r.get(0),
+ )?;
+ let mut c: openworker_compliance::delivery::Checkpoint = serde_json::from_str(&raw)?;
+ c.settle(token, now, success)?;
+ tx.execute(
+ "UPDATE ow_evaluation_delivery SET data=? WHERE deployment=? AND sink=?",
+ params![serde_json::to_string(&c)?, deployment, sink],
+ )?;
+ tx.commit()?;
+ Ok(())
+ }
+
+ #[cfg(feature = "evaluations")]
+ fn evaluation(
+ &mut self,
+ deployment: &str,
+ id: &str,
+ ) -> Result<Option<crate::evaluations::Evaluation>> {
+ ensure!(
+ crate::identifier(deployment) && crate::label(id),
+ "Invalid evaluation identity"
+ );
+ if self.db.query_row(
+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",
+ [],
+ |r| r.get::<_, i64>(0),
+ )? == 0
+ {
+ return Ok(None);
+ }
+ let raw: Option<String> = self
+ .db
+ .query_row(
+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",
+ params![deployment, id],
+ |r| r.get(0),
+ )
+ .optional()?;
+ raw.map(|raw| {
+ let e: crate::evaluations::Evaluation = serde_json::from_str(&raw)?;
+ e.validate()?;
+ ensure!(
+ e.deployment == deployment && e.result.id == id,
+ "Evaluation identity mismatch"
+ );
+ Ok(e)
+ })
+ .transpose()
+ }
+ #[cfg(feature = "evaluations")]
+ fn record_evaluation_bounded(
+ &mut self,
+ value: &crate::evaluations::Evaluation,
+ max_pending: u32,
+ ) -> Result<Write> {
+ ensure!(
+ (1..=1_000_000).contains(&max_pending),
+ "Invalid evaluation export backlog limit"
+ );
+ use rusqlite::OptionalExtension;
+ value.validate()?;
+ let value = value.clone().redacted();
+ value.validate()?;
+ let data = serde_json::to_string(&value)?;
+ // Existing corrections remain idempotent even after their predecessor expires.
+ if let Some(old) = self.evaluation(&value.deployment, &value.result.id)? {
+ ensure!(old == value, "Evaluation identity conflict");
+ return Ok(Write::Duplicate);
+ }
+ let tx = self
+ .db
+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink)); CREATE TABLE IF NOT EXISTS ow_observation_evaluations(cursor INTEGER PRIMARY KEY AUTOINCREMENT,deployment TEXT NOT NULL,id TEXT NOT NULL,session_hash TEXT NOT NULL,turn_hash TEXT NOT NULL,data TEXT NOT NULL,recorded_ms INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)*1000),UNIQUE(deployment,id)); CREATE INDEX IF NOT EXISTS ow_evaluation_scope ON ow_observation_evaluations(deployment,session_hash,turn_hash,cursor)")?;
+ let retained: i64 = tx.query_row(
+ "SELECT count(*) FROM ow_observation_evaluations e WHERE deployment=? AND EXISTS(SELECT 1 FROM ow_evaluation_delivery d WHERE d.deployment=e.deployment AND json_extract(d.data,'$.cursor')<e.cursor)",
+ [&value.deployment],
+ |r| r.get(0),
+ )?;
+ let exists: bool = tx.query_row(
+ "SELECT EXISTS(SELECT 1 FROM ow_observation_evaluations WHERE deployment=? AND id=?)",
+ params![value.deployment, value.result.id],
+ |r| r.get(0),
+ )?;
+ ensure!(
+ exists || retained < i64::from(max_pending),
+ "Pending evaluation export limit reached; restore delivery or raise the configured limit"
+ );
+ if let Some(id) = &value.result.supersedes {
+ let prior: Option<String> = tx
+ .query_row(
+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",
+ params![value.deployment, id],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let prior: crate::evaluations::Evaluation =
+ serde_json::from_str(&prior.context("Superseded evaluation missing")?)?;
+ ensure!(
+ prior.target == value.target,
+ "Correction must retain its target"
+ );
+ }
+ let inserted=tx.execute("INSERT INTO ow_observation_evaluations(deployment,id,session_hash,turn_hash,data) VALUES(?,?,?,?,?) ON CONFLICT(deployment,id) DO NOTHING",params![value.deployment,value.result.id,value.target.session_hash,value.target.turn_hash,data])?;
+ if inserted == 0 {
+ let old: String = tx.query_row(
+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",
+ params![value.deployment, value.result.id],
+ |r| r.get(0),
+ )?;
+ ensure!(
+ serde_json::from_str::<crate::evaluations::Evaluation>(&old)? == value,
+ "Evaluation identity conflict"
+ );
+ }
+ tx.commit()?;
+ Ok(if inserted == 0 {
+ Write::Duplicate
+ } else {
+ Write::Inserted
+ })
+ }
+ #[cfg(feature = "evaluations")]
+ fn evaluations(&mut self, q: &crate::evaluations::Query) -> Result<crate::evaluations::Page> {
+ q.validate()?;
+ if self.db.query_row(
+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",
+ [],
+ |r| r.get::<_, i64>(0),
+ )? == 0
+ {
+ return Ok(crate::evaluations::Page {
+ results: vec![],
+ next: None,
+ });
+ }
+ let mut stmt=self.db.prepare("SELECT cursor,data FROM ow_observation_evaluations WHERE deployment=? AND session_hash=? AND turn_hash=? AND cursor>? ORDER BY cursor LIMIT ?")?;
+ let rows = stmt.query_map(
+ params![
+ q.deployment,
+ q.session_hash,
+ q.turn_hash,
+ q.after as i64,
+ i64::from(q.limit) + 1
+ ],
+ |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),
+ )?;
+ let rows = rows
+ .map(|r| {
+ let (cursor, data) = r?;
+ Ok(crate::evaluations::Row {
+ cursor: cursor as u64,
+ evaluation: serde_json::from_str(&data)?,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ crate::evaluations::page(rows, q)
+ }
+
fn timeline(&mut self, q: &crate::query::Query) -> Result<crate::query::Page> {
q.validate()?;
let tx = self.db.transaction()?;
diff --git a/harness/observability/src/summary.rs b/harness/observability/src/summary.rs
index 80bc2ca5..3fb9db8d 100644
--- a/harness/observability/src/summary.rs
+++ b/harness/observability/src/summary.rs
@@ -1,11 +1,11 @@
//! Projection v1, rebuilt from bounded immutable raw records. It is not a bill,
//! audit ledger, or proof that absent spans/counters never existed.
use crate::{query::Page, Step};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
pub const VERSION: u8 = 1;
pub const MAX_GROUPS: usize = 128;
-#[derive(Clone, Debug, Default, Serialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Measure {
pub total: Option<u64>,
pub known: u64,
@@ -30,7 +30,7 @@ impl Measure {
}
}
}
-#[derive(Clone, Debug, Default, Serialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Usage {
pub input: Measure,
pub output: Measure,
@@ -47,7 +47,7 @@ impl Usage {
self.reasoning.add(u.and_then(|u| u.reasoning));
}
}
-#[derive(Debug, Default, Serialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Group {
pub name: Option<String>,
pub purpose: Option<String>,
@@ -97,9 +97,9 @@ pub struct Canonical {
pub last_span_id: Option<String>,
pub pending_waits: Option<u64>,
}
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Deserialize)]
pub struct Coverage {
- pub guarantee: &'static str,
+ pub guarantee: String,
pub scanned_steps: usize,
pub scan_truncated: bool,
pub group_truncated: bool,
@@ -114,7 +114,7 @@ pub struct Coverage {
pub clock_or_range_anomaly: bool,
pub hierarchy_incomplete: bool,
}
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Deserialize)]
pub struct Summary {
pub projection_version: u8,
pub through: u64,
@@ -124,7 +124,7 @@ pub struct Summary {
pub active_segments_accumulated_us: Measure,
pub model_accumulated_us: Measure,
pub tool_accumulated_us: Measure,
- pub exclusive_observed_us: BTreeMap<&'static str, u64>,
+ pub exclusive_observed_us: BTreeMap<String, u64>,
pub usage: Usage,
pub models: Vec<Group>,
pub tools: Vec<Group>,
@@ -332,7 +332,7 @@ pub fn build(page: &Page, canonical: Canonical) -> Summary {
for (at, delta) in changes {
if let Some(prev) = previous {
let length = at - prev;
- *exclusive.entry(phase(&active)).or_insert(0) += length;
+ *exclusive.entry(phase(&active).to_owned()).or_insert(0) += length;
window += length;
}
for n in 0..6 {
@@ -386,7 +386,7 @@ pub fn build(page: &Page, canonical: Canonical) -> Summary {
projection_version: VERSION,
through: page.through,
coverage: Coverage {
- guarantee: "best_effort",
+ guarantee: "best_effort".into(),
scanned_steps: page
.steps
.len()
diff --git a/harness/observability/tests/evaluations.rs b/harness/observability/tests/evaluations.rs
new file mode 100644
index 00000000..842d5da9
--- /dev/null
+++ b/harness/observability/tests/evaluations.rs
@@ -0,0 +1,206 @@
+#![cfg(feature = "evaluations")]
+use openworker_observability::{evaluations::*, sqlite::Sqlite, Store, Write};
+fn sample(id: &str) -> Evaluation {
+ Evaluation {
+ version: 1,
+ deployment: "sdk".into(),
+ target: Target {
+ session_hash: format!("{:064x}", 1),
+ turn_hash: format!("{:064x}", 2),
+ trace_id: None,
+ span_id: None,
+ },
+ result: Input {
+ id: id.into(),
+ name: "correctness".into(),
+ evaluator: Evaluator {
+ name: "fixture".into(),
+ version: "1".into(),
+ source: Source::Code,
+ model: None,
+ usage: None,
+ },
+ status: Status::Completed,
+ score: Some(Score::Boolean(true)),
+ created_ms: 1,
+ explanation: None,
+ dataset_case: None,
+ experiment: None,
+ supersedes: None,
+ },
+ }
+}
+#[test]
+fn scores_survive_reopen_and_do_not_rewrite_prior_evidence() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("traces.sqlite");
+ let mut store = Sqlite::open(&path).unwrap();
+ let e = sample("e1");
+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Inserted);
+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Duplicate);
+ let mut conflict = e.clone();
+ conflict.result.score = Some(Score::Boolean(false));
+ assert!(store.record_evaluation(&conflict).is_err());
+ let mut correction = sample("e2");
+ correction.result.supersedes = Some("e1".into());
+ correction.result.score = Some(Score::Boolean(false));
+ store.record_evaluation(&correction).unwrap();
+ drop(store);
+ let mut store = Sqlite::open(&path).unwrap();
+ let mut query = Query {
+ deployment: "sdk".into(),
+ session_hash: e.target.session_hash.clone(),
+ turn_hash: e.target.turn_hash.clone(),
+ after: 0,
+ limit: 1,
+ };
+ let page = store.evaluations(&query).unwrap();
+ assert_eq!(page.results[0].evaluation, e);
+ query.after = page.next.unwrap();
+ let page = store.evaluations(&query).unwrap();
+ assert_eq!(page.results[0].evaluation, correction);
+ assert!(page.next.is_none());
+ query.after = 0;
+ query.turn_hash = format!("{:064x}", 3);
+ assert!(store.evaluations(&query).unwrap().results.is_empty());
+ assert_eq!(
+ store.prune_evaluations("sdk", i64::MAX as u64, 1).unwrap(),
+ 1
+ );
+ assert!(store.evaluation("sdk", "e1").unwrap().is_none());
+ assert_eq!(
+ store.record_evaluation(&correction).unwrap(),
+ Write::Duplicate
+ );
+}
+
+#[cfg(feature = "postgres")]
+#[test]
+#[ignore = "disposable loopback PostgreSQL fixture required"]
+fn postgres_evaluations_are_idempotent_scoped_and_durable() {
+ use openworker_observability::postgres::{Config, Postgres, Tls};
+ let url = std::env::var("OW_OBSERVABILITY_TEST_URL").unwrap();
+ let schema = format!(
+ "ow_eval_{}",
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ );
+ let mut c = Config::new(url.clone(), schema.clone());
+ c.tls = Tls::LoopbackPlaintext;
+ Postgres::migrate(&c).unwrap();
+ let mut store = Postgres::open(&c).unwrap();
+ let e = sample("pg-score");
+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Inserted);
+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Duplicate);
+ drop(store);
+ let mut store = Postgres::open(&c).unwrap();
+ let mut q = Query {
+ deployment: "sdk".into(),
+ session_hash: e.target.session_hash.clone(),
+ turn_hash: e.target.turn_hash.clone(),
+ after: 0,
+ limit: 25,
+ };
+ assert_eq!(store.evaluations(&q).unwrap().results[0].evaluation, e);
+ q.turn_hash = format!("{:064x}", 3);
+ assert!(store.evaluations(&q).unwrap().results.is_empty());
+ let mut correction = e.clone();
+ correction.result.id = "pg-correction".into();
+ correction.result.supersedes = Some(e.result.id.clone());
+ store.record_evaluation(&correction).unwrap();
+ assert_eq!(
+ store.prune_evaluations("sdk", i64::MAX as u64, 1).unwrap(),
+ 1
+ );
+ assert_eq!(
+ store.record_evaluation(&correction).unwrap(),
+ Write::Duplicate
+ );
+ drop(store);
+ let mut outbox = Postgres::open(&c).unwrap();
+ // A distinct deployment keeps the earlier storage contract out of this queue.
+ outbox
+ .prune_evaluations("sdk", i64::MAX as u64, 100)
+ .unwrap();
+ delivery_contract(&mut outbox);
+ drop(outbox);
+ let mut cleanup = postgres::Client::connect(&url, postgres::NoTls).unwrap();
+ cleanup
+ .batch_execute(&format!("DROP SCHEMA {schema} CASCADE"))
+ .unwrap();
+}
+
+fn delivery_contract(store: &mut dyn Store) {
+ let binding = "a".repeat(64);
+ store
+ .register_evaluation_sink("sdk", "langfuse", &binding)
+ .unwrap();
+ let first = sample("delivery-one");
+ let second = sample("delivery-two");
+ store.record_evaluation_bounded(&first, 1).unwrap();
+ assert_eq!(
+ store.record_evaluation_bounded(&first, 1).unwrap(),
+ Write::Duplicate
+ );
+ assert!(store.record_evaluation_bounded(&second, 1).is_err());
+ let claimed = store
+ .claim_evaluation_export("sdk", "langfuse", &binding, "first", 100)
+ .unwrap()
+ .unwrap();
+ assert_eq!(claimed.row.evaluation, first);
+ assert!(store
+ .claim_evaluation_export("sdk", "langfuse", &binding, "other", 101)
+ .unwrap()
+ .is_none());
+ assert_eq!(
+ store
+ .prune_evaluations("sdk", i64::MAX as u64, 100)
+ .unwrap(),
+ 0
+ );
+ assert!(store
+ .settle_evaluation_export("sdk", "langfuse", "forged", 102, true)
+ .is_err());
+ store
+ .settle_evaluation_export("sdk", "langfuse", "first", 102, false)
+ .unwrap();
+ assert!(store
+ .claim_evaluation_export("sdk", "langfuse", &binding, "retry", 103)
+ .unwrap()
+ .is_none());
+ let retry = store
+ .claim_evaluation_export("sdk", "langfuse", &binding, "retry", 10000)
+ .unwrap()
+ .unwrap();
+ assert_eq!(retry.row.evaluation, first);
+ store
+ .settle_evaluation_export("sdk", "langfuse", "retry", 10001, true)
+ .unwrap();
+ // Retained, delivered evaluations must not consume pending admission capacity.
+ store.record_evaluation_bounded(&second, 1).unwrap();
+ assert_eq!(
+ store
+ .prune_evaluations("sdk", i64::MAX as u64, 100)
+ .unwrap(),
+ 1
+ );
+ assert!(store
+ .register_evaluation_sink("sdk", "langfuse", &"b".repeat(64))
+ .is_err());
+}
+#[test]
+fn sqlite_outbox_backoff_binding_and_pending_retention() {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("trace.sqlite");
+ let mut store = Sqlite::open(&path).unwrap();
+ delivery_contract(&mut store);
+ drop(store);
+ let mut reopened = Sqlite::open(&path).unwrap();
+ let next = reopened
+ .claim_evaluation_export("sdk", "langfuse", &"a".repeat(64), "restart", 20000)
+ .unwrap()
+ .unwrap();
+ assert_eq!(next.row.evaluation.result.id, "delivery-two");
+}
diff --git a/harness/observability/tests/storage.rs b/harness/observability/tests/storage.rs
index 59bf6a38..6f8e4cd2 100644
--- a/harness/observability/tests/storage.rs
+++ b/harness/observability/tests/storage.rs
@@ -290,3 +290,22 @@ fn postgres_connection_validation() {
c.schema = "ow_obs;DROP SCHEMA public".into();
assert!(c.validate().is_err());
}
+
+#[test]
+fn recording_refuses_execution_database_without_modifying_schema() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("runtime.sqlite3");
+ let db = rusqlite::Connection::open(&path).unwrap();
+ db.execute_batch("CREATE TABLE sessions(id TEXT PRIMARY KEY)")
+ .unwrap();
+ assert!(Sqlite::open(&path).is_err());
+ assert_eq!(
+ db.query_row(
+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_version'",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+}
diff --git a/harness/release/README.md b/harness/release/README.md
index e47b1e34..4648baad 100644
--- a/harness/release/README.md
+++ b/harness/release/README.md
@@ -10,7 +10,7 @@ See [versions and compatibility](../../docs/harness/guides/releases.md).
npm --prefix harness/typescript-sdk ci
npm --prefix harness/typescript-sdk run build
cargo build --manifest-path harness/Cargo.toml --locked --release \
- -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export
+ -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export,recording-postgres,evaluation-export
node harness/release/prepare.mjs \
/absolute/target/release/openworker-harness aarch64-apple-darwin /absolute/new-candidate
```
@@ -42,14 +42,14 @@ hook or fetches an executable during startup.
## Rust archives
-Package the nine crates listed in `manifest.json` together, in dependency order.
+Package the crates listed in `manifest.json` together, in dependency order.
Cargo rewrites path+version dependencies into registry dependencies. For example:
```sh
cargo package --manifest-path harness/Cargo.toml --no-verify \
-p openworker-artifacts -p openworker-compliance -p openworker-guardrails \
-p openworker-telemetry -p openworker-tools -p openworker-runtime \
- -p openworker-security-adapters -p openworker-store-postgres -p openworker-sdk
+ -p openworker-security-adapters -p openworker-store-postgres -p openworker-observability -p openworker-sdk
node harness/release/check-examples.mjs /absolute/new-examples
python3 harness/release/verify-crates.py \
--packages /absolute/target/package --out /absolute/new-consumer \
diff --git a/harness/release/manifest.json b/harness/release/manifest.json
index d7d08356..16d4250f 100644
--- a/harness/release/manifest.json
+++ b/harness/release/manifest.json
@@ -34,6 +34,10 @@
"name": "openworker-store-postgres",
"directory": "store-postgres"
},
+ {
+ "name": "openworker-observability",
+ "directory": "observability"
+ },
{
"name": "openworker-sdk",
"directory": "harness-sdk"
diff --git a/harness/release/smoke-npm.mjs b/harness/release/smoke-npm.mjs
index 1e635c29..fffab1d3 100644
--- a/harness/release/smoke-npm.mjs
+++ b/harness/release/smoke-npm.mjs
@@ -19,5 +19,15 @@ try {
assert.equal(JSON.parse(await readFile(join(dir,'new-app/openworker.json'),'utf8')).sdkRoot,null);
// The async fixture must keep its own event loop running while the app executes.
execFileSync(process.execPath,[join(root,'release/run-examples.mjs'),dir,'-',process.execPath],{cwd:dir,stdio:'inherit'});
+ await writeFile(join(dir,'observability.mjs'), `
+ import assert from 'node:assert/strict';
+ import {Harness,langfuseTelemetry} from '@openworkerai/harness-sdk';
+ import {evidenceJsonl} from '@openworkerai/harness-sdk/evaluation-export';
+ import {join} from 'node:path';
+ const h=await Harness.start({stateDir:join(process.cwd(),'observation-state'),sandbox:'off',recording:{database:{type:'sqlite',path:join(process.cwd(),'observation.sqlite')},evaluations:true}});
+ try {assert.equal((await h.observability.health()).recording_enabled,true);assert.equal((await h.observability.flush()).recording_drained,true);assert.equal(langfuseTelemetry('http://127.0.0.1:1','AUTH').acknowledgement,'langfuse_v4');assert(evidenceJsonl({selection:{sessionId:'s',turnId:'t'},watermark:0,timeline:{},summary:{}}).includes('selected_pages_only'));}
+ finally {await h.shutdown();}
+ `);
+ execFileSync(process.execPath,[join(dir,'observability.mjs')],{cwd:dir,stdio:'inherit'});
console.log('Packed npm client, native runtime, starter and exact TypeScript guide verified');
} finally {await rm(dir,{recursive:true,force:true});}
diff --git a/harness/release/verify-crates.py b/harness/release/verify-crates.py
index de4207a9..cd7c2f20 100644
--- a/harness/release/verify-crates.py
+++ b/harness/release/verify-crates.py
@@ -10,7 +10,7 @@ for item in m['rustPackages']:
name=item['name'];filename=f'{name}-{m["version"]}'
with tarfile.open(pathlib.Path(a.packages)/(filename+'.crate')) as archive:archive.extractall(out/'crates',filter='data')
patch.append(f'{name} = {{ path = {json.dumps(str(out/"crates"/filename))} }}')
-(out/'Cargo.toml').write_text('[package]\nname="openworker-registry-consumer"\nversion="0.0.0"\nedition="2021"\n[workspace]\n[dependencies]\nopenworker-sdk = {version="='+m['version']+'", features=["security-adapters","compliance-export"]}\ntokio = {version="1", features=["macros","rt-multi-thread"]}\nfutures-util="0.3"\nanyhow="1"\n[patch.crates-io]\n'+'\n'.join(patch)+'\n')
+(out/'Cargo.toml').write_text('[package]\nname="openworker-registry-consumer"\nversion="0.0.0"\nedition="2021"\n[workspace]\n[dependencies]\nopenworker-sdk = {version="='+m['version']+'", features=["security-adapters","compliance-export","recording-postgres","evaluation-export"]}\ntokio = {version="1", features=["macros","rt-multi-thread"]}\nfutures-util="0.3"\nanyhow="1"\n[patch.crates-io]\n'+'\n'.join(patch)+'\n')
(out/'src/main.rs').write_text(pathlib.Path(a.example).read_text() if a.example else 'fn main() { let _ = openworker_sdk::agents::Harness::builder(); }\n')
if a.fetch:
subprocess.run(['cargo','fetch','--manifest-path',str(out/'Cargo.toml')],check=True)
diff --git a/harness/runtime/src/agent_team.rs b/harness/runtime/src/agent_team.rs
index 192366d8..f19f7fc7 100644
--- a/harness/runtime/src/agent_team.rs
+++ b/harness/runtime/src/agent_team.rs
@@ -26,6 +26,8 @@ pub struct Wait {
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Document {
+ #[serde(default)]
+ pub trace_origins: BTreeMap<String, crate::observation_links::SpawnOrigin>,
pub id: String,
pub lead: String,
pub roster: rules::Roster,
@@ -365,6 +367,7 @@ pub fn apply(
}
}
let doc = Document {
+ trace_origins: BTreeMap::new(),
id: id.clone(),
lead: lead_id.clone(),
roster,
@@ -1037,6 +1040,26 @@ pub fn apply(
};
let reservation = doc.roster.reserve(request.clone(), || crate::id("s"))?;
if reservation.created {
+ if let Some(turn) = &s.turn {
+ doc.trace_origins.insert(
+ request.agent_id.clone(),
+ crate::observation_links::SpawnOrigin {
+ parent_session_id: s.id.clone(),
+ parent_turn_id: turn.id.clone(),
+ parent_call_id: turn
+ .queue
+ .first()
+ .filter(|c| {
+ matches!(
+ c["name"].as_str(),
+ Some("team_spawn" | "create_sub_agent")
+ )
+ })
+ .and_then(|c| c["call_id"].as_str())
+ .map(str::to_owned),
+ },
+ );
+ }
doc.sandbox_choices
.insert(request.agent_id.clone(), sandbox.clone());
if let Some(binding) = binding {
@@ -1420,6 +1443,7 @@ pub fn public_snapshot(mut value: Value) -> Value {
if let Some(object) = value.as_object_mut() {
for key in [
"standalone",
+ "trace_origins",
"commands",
"tasks",
"charges",
diff --git a/harness/runtime/src/connections.rs b/harness/runtime/src/connections.rs
index 96e26466..a58c389a 100644
--- a/harness/runtime/src/connections.rs
+++ b/harness/runtime/src/connections.rs
@@ -160,7 +160,7 @@ impl Manager {
bail!("Credential database cannot be a link");
}
let db = Connection::open(path)?;
- db.execute_batch("PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL; PRAGMA secure_delete=ON;
+ db.execute_batch("PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL; PRAGMA secure_delete=ON; BEGIN IMMEDIATE;
CREATE TABLE IF NOT EXISTS connections(id TEXT PRIMARY KEY,label TEXT NOT NULL,identity TEXT NOT NULL,persistence TEXT NOT NULL,secret BLOB);
CREATE TABLE IF NOT EXISTS settings(id INTEGER PRIMARY KEY CHECK(id=1),value TEXT NOT NULL);")?;
let has_provider = db
@@ -198,6 +198,10 @@ impl Manager {
"ALTER TABLE endpoints ADD COLUMN prompt_caching INTEGER NOT NULL DEFAULT 1 CHECK(prompt_caching IN (0,1))",
)?;
}
+ // Multiple service controllers may prepare the same immutable child type.
+ // Serialize check-and-alter migrations so a duplicate initializer cannot
+ // turn valid child provisioning into a permanent definition failure.
+ db.execute_batch("COMMIT")?;
Ok(Self {
scope: root.canonicalize()?.to_string_lossy().into(),
state: Mutex::new(State {
diff --git a/harness/runtime/src/lib.rs b/harness/runtime/src/lib.rs
index 4148484e..d8894a41 100644
--- a/harness/runtime/src/lib.rs
+++ b/harness/runtime/src/lib.rs
@@ -71,3 +71,5 @@ pub use openworker_compliance as compliance;
pub mod compliance_export;
pub mod subagents;
+
+pub mod observation_links;
diff --git a/harness/runtime/src/observation_links.rs b/harness/runtime/src/observation_links.rs
new file mode 100644
index 00000000..79d4ed54
--- /dev/null
+++ b/harness/runtime/src/observation_links.rs
@@ -0,0 +1,118 @@
+//! Durable spawn provenance. Contains identifiers only, never a child brief.
+use serde::{Deserialize, Serialize};
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SpawnOrigin {
+ pub parent_session_id: String,
+ pub parent_turn_id: String,
+ pub parent_call_id: Option<String>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ChildLink {
+ pub origin: SpawnOrigin,
+ pub child_session_id: String,
+ /// First accepted task only; later inbox turns are not charged to the spawn.
+ #[serde(default)]
+ pub child_turn_id: Option<String>,
+ pub agent_id: String,
+ pub phase: crate::team_coordination::Phase,
+}
+pub fn children(doc: &crate::agent_team::Document, session: &str, turn: &str) -> Vec<ChildLink> {
+ doc.trace_origins
+ .iter()
+ .filter(|(_, o)| o.parent_session_id == session && o.parent_turn_id == turn)
+ .filter_map(|(id, origin)| {
+ doc.roster.instance(id).map(|i| ChildLink {
+ origin: origin.clone(),
+ child_session_id: i.session_id.clone(),
+ child_turn_id: None,
+ agent_id: id.clone(),
+ phase: i.phase,
+ })
+ })
+ .collect()
+}
+
+/// A bounded selection of causally linked initial tasks. No transcript is read.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Run {
+ pub session_id: String,
+ pub turn_id: String,
+}
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Subtree {
+ pub runs: Vec<Run>,
+ pub selection_truncated: bool,
+ pub scope: String,
+}
+pub fn subtree(
+ root: Run,
+ mut children: impl FnMut(&Run) -> anyhow::Result<Vec<ChildLink>>,
+) -> anyhow::Result<Subtree> {
+ let mut runs = vec![root.clone()];
+ let mut seen = std::collections::BTreeSet::from([root]);
+ let mut truncated = false;
+ let mut index = 0;
+ while index < runs.len() {
+ for child in children(&runs[index])? {
+ let Some(turn_id) = child.child_turn_id else {
+ continue;
+ };
+ let run = Run {
+ session_id: child.child_session_id,
+ turn_id,
+ };
+ if seen.contains(&run) {
+ continue;
+ }
+ if runs.len() == 25 {
+ truncated = true;
+ continue;
+ }
+ seen.insert(run.clone());
+ runs.push(run);
+ }
+ index += 1;
+ }
+ Ok(Subtree {
+ runs,
+ selection_truncated: truncated,
+ scope: "root_and_visible_spawned_initial_turns".into(),
+ })
+}
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn nested_links_are_bounded_and_cycles_do_not_double_count() {
+ let root = Run {
+ session_id: "s0".into(),
+ turn_id: "t".into(),
+ };
+ let result = subtree(root, |r| {
+ Ok((0..40)
+ .map(|n| ChildLink {
+ origin: SpawnOrigin {
+ parent_session_id: r.session_id.clone(),
+ parent_turn_id: r.turn_id.clone(),
+ parent_call_id: None,
+ },
+ child_session_id: format!("s{n}"),
+ child_turn_id: Some("t".into()),
+ agent_id: format!("a{n}"),
+ phase: crate::team_coordination::Phase::Ready,
+ })
+ .collect())
+ })
+ .unwrap();
+ assert_eq!(result.runs.len(), 25);
+ assert!(result.selection_truncated);
+ assert_eq!(
+ result
+ .runs
+ .iter()
+ .collect::<std::collections::BTreeSet<_>>()
+ .len(),
+ 25
+ );
+ }
+}
diff --git a/harness/runtime/src/scoped_store.rs b/harness/runtime/src/scoped_store.rs
index 6338e17f..89819174 100644
--- a/harness/runtime/src/scoped_store.rs
+++ b/harness/runtime/src/scoped_store.rs
@@ -82,6 +82,10 @@ impl StateStore for ScopedStore {
.unwrap()
.assistant_call(sid, tid, call, args)
}
+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {
+ self.metadata(sid)?;
+ self.inner.lock().unwrap().has_turn(sid, tid)
+ }
fn agent_run(&self, sid: &str, submission: Option<&str>) -> Result<Option<Value>> {
self.metadata(sid)?;
self.inner.lock().unwrap().agent_run(sid, submission)
diff --git a/harness/runtime/src/store.rs b/harness/runtime/src/store.rs
index 3459ce3a..42a29e72 100644
--- a/harness/runtime/src/store.rs
+++ b/harness/runtime/src/store.rs
@@ -213,6 +213,17 @@ pub struct Event {
/// the runtime safe for shared workers. `recover` may only touch sessions owned
/// exclusively by this store instance. No automatic replay of tool effects.
pub trait StateStore: Send {
+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {
+ Ok(self.metadata(sid)?.turn.is_some_and(|t| t.id == tid))
+ }
+
+ fn observation_children(
+ &self,
+ _sid: &str,
+ _tid: &str,
+ ) -> Result<Vec<crate::observation_links::ChildLink>> {
+ bail!("Spawn trace queries unsupported")
+ }
fn pending_observers(&self, _sid: &str) -> Result<Vec<Value>> {
Ok(vec![])
}
@@ -1068,6 +1079,40 @@ impl Store {
}
impl StateStore for Store {
+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {
+ self.metadata(sid)?;
+ Ok(self.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM submissions WHERE session_id=? AND turn_id=?)",
+ params![sid, tid],
+ |r| r.get(0),
+ )?)
+ }
+
+ fn observation_children(
+ &self,
+ sid: &str,
+ tid: &str,
+ ) -> Result<Vec<crate::observation_links::ChildLink>> {
+ let session = self.metadata(sid)?;
+ let Some(member) = session.agent_team else {
+ return Ok(vec![]);
+ };
+ let raw: String = self.db.query_row(
+ "SELECT data FROM agent_teams WHERE id=?",
+ [member.team_id],
+ |r| r.get(0),
+ )?;
+ let mut children =
+ crate::observation_links::children(&serde_json::from_str(&raw)?, sid, tid);
+ for child in &mut children {
+ child.child_turn_id = self.db.query_row(
+ "SELECT turn_id FROM events WHERE session_id=? AND kind='turn_accepted' ORDER BY id LIMIT 1",
+ [&child.child_session_id], |r| r.get(0),
+ ).optional()?.flatten();
+ }
+ Ok(children)
+ }
+
fn pending_observers(&self, sid: &str) -> Result<Vec<Value>> {
Store::pending_observers(self, sid)
}
diff --git a/harness/runtime/tests/connections.rs b/harness/runtime/tests/connections.rs
index 3d37a1fa..8ef0f600 100644
--- a/harness/runtime/tests/connections.rs
+++ b/harness/runtime/tests/connections.rs
@@ -465,3 +465,29 @@ fn macos_keychain_roundtrip_and_removal() {
.any(|w| w == b"keychain-synthetic-test-key"));
assert!(m.key(&selected).is_err());
}
+
+#[test]
+fn concurrent_controller_connection_cache_initialization() {
+ // Two service controllers can prepare the same immutable child definition.
+ // Their local connection-cache schema must be initialized atomically.
+ for _ in 0..12 {
+ let root = tempfile::tempdir().unwrap();
+ let barrier = std::sync::Arc::new(std::sync::Barrier::new(4));
+ std::thread::scope(|scope| {
+ let tasks = (0..4)
+ .map(|_| {
+ let barrier = barrier.clone();
+ let path = root.path();
+ scope.spawn(move || {
+ barrier.wait();
+ openworker_runtime::connections::Manager::open(path)
+ .map(|m| m.list().unwrap())
+ })
+ })
+ .collect::<Vec<_>>();
+ for task in tasks {
+ task.join().unwrap().unwrap();
+ }
+ });
+ }
+}
diff --git a/harness/server-reference/src/recording_query.rs b/harness/server-reference/src/recording_query.rs
index f9ef1285..112dd0ea 100644
--- a/harness/server-reference/src/recording_query.rs
+++ b/harness/server-reference/src/recording_query.rs
@@ -48,7 +48,7 @@ pub(crate) fn authority(method: &str, path: &str) -> bool {
_,
"observability",
_,
- "timeline" | "summary"
+ "timeline" | "summary" | "children" | "subtree-summary"
]
)
}
@@ -70,6 +70,14 @@ pub(crate) fn router(prefix: &str) -> Router<App> {
&format!("{prefix}/sessions/:sid/observability/:turn/timeline"),
get(timeline),
)
+ .route(
+ &format!("{prefix}/sessions/:sid/observability/:turn/children"),
+ get(children),
+ )
+ .route(
+ &format!("{prefix}/sessions/:sid/observability/:turn/subtree-summary"),
+ get(subtree_summary),
+ )
.route(
&format!("{prefix}/sessions/:sid/observability/:turn/summary"),
get(summary),
@@ -267,6 +275,45 @@ async fn summary(
let summary = summary::build(&page, canonical(&run));
Ok(Json(json!({"run":run,"summary":summary})))
}
+async fn subtree_summary(
+ Extension(app): Extension<App>,
+ Path((sid, turn)): Path<(String, String)>,
+ Query(q): Query<Summary>,
+) -> Result<Json<Value>, ApiError> {
+ use openworker_runtime::observation_links::{subtree, Run as LinkRun};
+ app.store.observation_run(&sid, &turn)?;
+ let selection = subtree(
+ LinkRun {
+ session_id: sid,
+ turn_id: turn,
+ },
+ |r| app.store.observation_children(&r.session_id, &r.turn_id),
+ )?;
+ for run in &selection.runs {
+ app.store.observation_run(&run.session_id, &run.turn_id)?;
+ }
+ let page = read(
+ &app,
+ Read {
+ deployment: app.config.schema.clone(),
+ scopes: selection
+ .runs
+ .iter()
+ .map(|r| scope(&r.session_id, &r.turn_id))
+ .collect(),
+ after: 0,
+ through: q.through,
+ limit: openworker_observability::query::MAX_SUMMARY_STEPS,
+ },
+ )
+ .await?;
+ for run in &selection.runs {
+ app.store.observation_run(&run.session_id, &run.turn_id)?;
+ }
+ Ok(Json(
+ json!({"selection":selection,"summary":summary::build(&page,Canonical::default())}),
+ ))
+}
async fn grouped(
Extension(app): Extension<App>,
Query(q): Query<Search>,
@@ -329,6 +376,18 @@ async fn grouped(
))
}
+async fn children(
+ Extension(app): Extension<App>,
+ Path((sid, turn)): Path<(String, String)>,
+) -> Result<Json<Value>, ApiError> {
+ app.store.observation_run(&sid, &turn)?;
+ let children = app.store.observation_children(&sid, &turn)?;
+ app.store.observation_run(&sid, &turn)?;
+ Ok(Json(
+ json!({"children":children,"coverage":"authorized_captured_spawn_origins_only"}),
+ ))
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -341,6 +400,8 @@ mod tests {
"/observability/health",
"/sessions/s/observability/t/timeline",
"/sessions/s/observability/t/summary",
+ "/sessions/s/observability/t/children",
+ "/sessions/s/observability/t/subtree-summary",
] {
let path = format!("{prefix}{path}");
assert!(authority("GET", &path));
diff --git a/harness/server-reference/tests/service_v2.rs b/harness/server-reference/tests/service_v2.rs
index a0e5fa22..2431c1fb 100644
--- a/harness/server-reference/tests/service_v2.rs
+++ b/harness/server-reference/tests/service_v2.rs
@@ -165,6 +165,13 @@ async fn qualify(standalone: bool, live: bool) {
.unwrap();
let mut value = json!({"service":{"id":"m2-test","namespace":"test","agent":"builder"},"database_env":"OW_PG_TEST_URL","schema":format!("ow_service_{}",uuid::Uuid::new_v4().simple()),"observability":{"schema":format!("ow_record_{}",uuid::Uuid::new_v4().simple())},"database_loopback_plaintext":true,"state_root":root.path().join("state"),"bundle":bundle,"runner":std::env::var("OW_PROBE_RUNNER").unwrap(),"model":"fixture","scripted":true,"api_token_file":token_file,"actor":"tester","openshell":{"identity":"m2-test","endpoint":std::env::var("OW_PROBE_ENDPOINT").unwrap(),"certificates":std::env::var("OW_PROBE_CERTS").unwrap(),"tls_server_name":std::env::var("OW_PROBE_TLS_NAME").ok(),"workspace":"default","image":std::env::var("OW_PROBE_IMAGE").unwrap()}});
+ if let Ok(endpoint) = std::env::var("OW_TEST_OTLP_ENDPOINT") {
+ assert!(
+ endpoint.starts_with("http://127.0.0.1:"),
+ "Qualification exporter must be loopback"
+ );
+ value["telemetry"] = json!({"endpoint":endpoint,"signals":["traces"],"acknowledgement":"langfuse_v4","headers":[{"name":"Authorization","value_env":"OW_TEST_OTLP_AUTH"},{"name":"x-langfuse-ingestion-version","value":"4"}],"content":{"capture":"metadata"}});
+ }
if live {
value["model"] = json!(
std::env::var("OW_STANDALONE_LIVE_MODEL").expect("Provide an authorized model ID")
diff --git a/harness/server-reference/tests/support/service_subagents.rs b/harness/server-reference/tests/support/service_subagents.rs
index f03db1ec..3734636c 100644
--- a/harness/server-reference/tests/support/service_subagents.rs
+++ b/harness/server-reference/tests/support/service_subagents.rs
@@ -51,6 +51,52 @@ pub(super) async fn qualify(
.await
.unwrap();
assert_eq!(run.result().await.unwrap().status, "completed");
+ let links = service
+ .json(
+ "GET",
+ &format!(
+ "/v1/sessions/{}/observability/{}/children",
+ parent.id(),
+ run.id
+ ),
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(links["children"].as_array().unwrap().len(), 3);
+ for link in links["children"].as_array().unwrap() {
+ assert_eq!(link["origin"]["parent_turn_id"], run.id);
+ assert!(link["origin"]["parent_call_id"].is_string());
+ }
+ // A parent-only read credential cannot discover child identifiers or usage.
+ let cfg: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
+ let admin_token = std::fs::read_to_string(cfg["api_token_file"].as_str().unwrap()).unwrap();
+ let read_token = uuid::Uuid::new_v4().simple().to_string().repeat(2);
+ _db.authorized(admin_token.trim()).unwrap().auth_issue("trace-parent-only",serde_json::from_value(json!({"principal":"trace-reader","role":"observer","scope":{"kind":"sessions","ids":[parent.id()]}})).unwrap(),&read_token).unwrap();
+ let restricted = _db.authorized(&read_token).unwrap();
+ assert!(restricted
+ .observation_children(parent.id(), &run.id)
+ .unwrap()
+ .is_empty());
+ let initial_scope = format!("/v1/sessions/{}/observability/{}", parent.id(), run.id);
+ let initial_tree = service
+ .json(
+ "GET",
+ &format!("{initial_scope}/subtree-summary"),
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ initial_tree["selection"]["runs"].as_array().unwrap().len(),
+ 4
+ );
+ assert!(initial_tree["summary"]["wall_elapsed_us"].is_null());
+ for link in links["children"].as_array().unwrap() {
+ assert!(link["child_turn_id"].is_string());
+ }
let original = doc(config, parent.id());
assert!(original.standalone.is_some());
assert_eq!(original.roster.instances().count(), 4);
@@ -181,6 +227,50 @@ pub(super) async fn qualify(
.unwrap()
.iter()
.any(|e| e["type"] == "tool_started"));
+ // Follow-up messages must not change the initial spawn selection.
+ let after_tree = service
+ .json(
+ "GET",
+ &format!("{initial_scope}/subtree-summary"),
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(after_tree["selection"], initial_tree["selection"]);
+ if let Ok(destination) = std::env::var("OW_OBSERVABILITY_EVIDENCE") {
+ let timeline = service
+ .json(
+ "GET",
+ &format!("{initial_scope}/timeline?limit=200"),
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ let summary = service
+ .json("GET", &format!("{initial_scope}/summary"), None, None)
+ .await
+ .unwrap();
+ let mut child_traces = Vec::new();
+ for link in links["children"].as_array().unwrap() {
+ let base = format!(
+ "/v1/sessions/{}/observability/{}",
+ link["child_session_id"].as_str().unwrap(),
+ link["child_turn_id"].as_str().unwrap()
+ );
+ let timeline = service
+ .json("GET", &format!("{base}/timeline?limit=200"), None, None)
+ .await
+ .unwrap();
+ let summary = service
+ .json("GET", &format!("{base}/summary"), None, None)
+ .await
+ .unwrap();
+ child_traces.push(json!({"timeline":timeline,"summary":summary}));
+ }
+ std::fs::write(destination,serde_json::to_vec_pretty(&json!({"timeline":timeline,"summary":summary,"children":links,"subtree":after_tree,"childTraces":child_traces})).unwrap()).unwrap();
+ }
eprintln!("PASS standalone service releases: two async builders, sync restricted verifier, shared OpenShell, worker restart, durable follow-up");
}
diff --git a/harness/service-ui/public/app.js b/harness/service-ui/public/app.js
index 3608b984..253854a1 100644
--- a/harness/service-ui/public/app.js
+++ b/harness/service-ui/public/app.js
@@ -563,6 +563,7 @@ async function render() {
request: (path, signal) =>
api(path, "GET", undefined, undefined, signal),
isAdmin: can.admin(),
+ children: true,
});
return;
}
diff --git a/harness/service-ui/public/observability.js b/harness/service-ui/public/observability.js
index ccd78ea6..9573abf3 100644
--- a/harness/service-ui/public/observability.js
+++ b/harness/service-ui/public/observability.js
@@ -116,7 +116,11 @@ const stepKind = (s) =>
? "question_wait"
: "other_active";
-export function mountObservability(root, { request, isAdmin = false }) {
+let viewerSequence = 0;
+export function mountObservability(root, { request, isAdmin = false, initialRun = null, directory = true, openSession = true, children = false }) {
+ const instance = `ow-trace-${++viewerSequence}`;
+ const domId = (id) => `${instance}-${id}`;
+ const ids = (html) => html.replace(/(id|for|aria-controls|aria-labelledby)="(obs[^" ]*)"/g, (_, attr, id) => `${attr}="${domId(id)}"`);
const lifetime = new AbortController();
let stopped = false,
serial = 0,
@@ -124,7 +128,8 @@ export function mountObservability(root, { request, isAdmin = false }) {
timer,
loadController,
searchController;
- let chosen = null,
+ let childLinks=[], ancestors=[], subtree=null;
+ let chosen = initialRun,
tab = "usage",
summary = null,
selection = null,
@@ -155,8 +160,10 @@ export function mountObservability(root, { request, isAdmin = false }) {
controllers.delete(c);
}
}
- root.innerHTML = `<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder &amp; exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`;
- const $ = (s) => root.querySelector(s);
+ root.innerHTML = ids(`<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder &amp; exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`);
+ const $ = (s) => root.querySelector(s.replace(/#(obs[\w-]*)/g, (_, id) => `#${domId(id)}`));
+ if (!directory) $(".obs-picker").hidden = true;
+ root.addEventListener("ow-trace-refresh", () => load(), {signal:lifetime.signal});
const content = $("#obs-content"),
input = $("#obs-run-search"),
menu = $("#obs-run-menu");
@@ -168,7 +175,7 @@ export function mountObservability(root, { request, isAdmin = false }) {
}
function scope() {
$("#obs-scope").innerHTML = chosen
- ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc(chosen.agent)} · ${esc(chosen.status)} · ${esc(new Date(chosen.created_ms).toLocaleString())}</span></div><button class="obs-text" data-obs="recent">Recent runs</button><a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a></div>`
+ ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc([chosen.agent, chosen.status, known(chosen.created_ms) ? new Date(chosen.created_ms).toLocaleString() : null].filter(Boolean).join(" · "))}</span></div>${directory ? `<button class="obs-text" data-obs="recent">Recent runs</button>` : ""}${openSession ? `<a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a>` : ""}</div>`
: `<p class="obs-hint obs-scope">${selection ? `${selection.selected_runs} recent authorized runs${selection.selection_truncated ? " · more available" : ""}. Summary covers this page only.` : "Recent authorized runs"}</p>`;
if (summary?.coverage.scan_truncated || summary?.coverage.group_truncated) {
$("#obs-scope").insertAdjacentHTML(
@@ -176,13 +183,14 @@ export function mountObservability(root, { request, isAdmin = false }) {
'<span class="obs-partial">Partial summary</span>',
);
}
- content.setAttribute("aria-labelledby", `obs-${tab}-tab`);
+ content.setAttribute("aria-labelledby", domId(`obs-${tab}-tab`));
for (const b of root.querySelectorAll(
"[data-obs=usage],[data-obs=timeline]",
)) {
const active = b.dataset.obs === tab;
b.classList.toggle("active", active);
b.setAttribute("aria-selected", String(active));
+ b.tabIndex = active ? 0 : -1;
}
}
function draw() {
@@ -219,7 +227,7 @@ export function mountObservability(root, { request, isAdmin = false }) {
)
.join(
"",
- )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}`;
+ )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}${chosen&&children?`<section class="obs-panel"><h2>Sub-agents</h2>${ancestors.length?`<button class="button" data-obs="parent">Back to parent turn</button>`:""}${childLinks.map((child,i)=>`<button class="button" data-obs-child="${i}">${esc(child.agent_id)} · ${esc(child.phase)}</button>`).join(" ")||hint("No visible recorded child links for this turn.")}${subtree ? hint(`Linked work: ${count(subtree.selection.runs.length)} visible initial tasks${subtree.selection.selection_truncated ? " · truncated" : ""} · ${measurement(subtree.summary.usage.input)} input / ${measurement(subtree.summary.usage.output)} output tokens. Later inbox turns are separate.`) : ""}${hint("Usage above covers this turn. Open a child to inspect its own requests and usage.")}</section>`:""}`;
widths();
}
function usageView(s) {
@@ -274,7 +282,7 @@ export function mountObservability(root, { request, isAdmin = false }) {
}))
.filter((r) => r.value > 0),
"Run phase duration",
- )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="obs-step-detail">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="obs-timeline-error" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;
+ )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="${domId("obs-step-detail")}">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="${domId("obs-timeline-error")}" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;
}
function stepDetail(r) {
const s = r.step;
@@ -338,6 +346,12 @@ export function mountObservability(root, { request, isAdmin = false }) {
selection = value;
summary = selection.summary;
}
+ childLinks=[];subtree=null;
+ if(chosen && children) {
+ try {const links=await read(perRun(chosen)+"/children",c); if(stopped||n!==serial)return;childLinks=links.children||[];} catch {childLinks=[];}
+ if (stopped || n !== serial) return;
+ try {const value=await read(perRun(chosen)+"/subtree-summary?"+query({through:timelineThrough}),c);if(stopped||n!==serial)return;subtree=value;} catch {subtree=null;}
+ }
draw();
} catch (e) {
if (stopped || n !== serial || c.signal.aborted) return;
@@ -356,12 +370,12 @@ export function mountObservability(root, { request, isAdmin = false }) {
$("#obs-run-options").innerHTML = results
.map(
(r, i) =>
- `<button type="button" class="obs-run-option" role="option" id="obs-option-${i}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,
+ `<button type="button" class="obs-run-option" role="option" id="${domId(`obs-option-${i}`)}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,
)
.join("");
$("#obs-search-more").hidden = !resultNext;
if (focusedIndex >= 0)
- input.setAttribute("aria-activedescendant", `obs-option-${focusedIndex}`);
+ input.setAttribute("aria-activedescendant", domId(`obs-option-${focusedIndex}`));
else input.removeAttribute("aria-activedescendant");
}
async function search(more = false) {
@@ -401,7 +415,8 @@ export function mountObservability(root, { request, isAdmin = false }) {
$("#obs-search-state").textContent = "Unable to search runs. Try again.";
}
}
- function choose(run) {
+ function choose(run, keepParents=false) {
+ if(!keepParents) ancestors=[];
if (!run) return;
chosen = run;
tool = null;
@@ -498,14 +513,29 @@ export function mountObservability(root, { request, isAdmin = false }) {
draw();
return;
}
+ const childButton=e.target.closest("[data-obs-child]");
+ if(childButton){
+ const child=childLinks[Number(childButton.dataset.obsChild)];if(!child)return;
+ const selectedSerial=serial;const selectedParent=chosen;
+ if(child.child_turn_id){ancestors.push(selectedParent);choose({session_id:child.child_session_id,turn_id:child.child_turn_id},true);return;}
+ childButton.disabled=true;
+ const c=controller();
+ read("/v1/observability/runs?"+query({q:child.child_session_id,limit:100}),c).then(page=>{
+ if(stopped||serial!==selectedSerial)return;
+ const run=child.child_turn_id ? {session_id:child.child_session_id,turn_id:child.child_turn_id} : page.runs.find(r=>r.session_id===child.child_session_id);
+ if(run){ancestors.push(selectedParent);choose(run,true);}else{content.prepend(notice("No accessible child turn is available yet."));}
+ }).catch(()=>{if(!stopped)content.prepend(notice("Child traces are unavailable."));}).finally(()=>{childButton.disabled=false;});
+ return;
+ }
const action = e.target.closest("[data-obs]")?.dataset.obs;
+ if(action==="parent" && ancestors.length){choose(ancestors.pop(),true);return;}
if (action === "refresh") {
health = null;
healthError = false;
if ($("#obs-health")?.open) loadHealth();
load();
}
- if (action === "recent") {
+ if (action === "recent" && directory) {
chosen = null;
tool = null;
tab = "usage";
@@ -534,6 +564,15 @@ export function mountObservability(root, { request, isAdmin = false }) {
},
{ signal: lifetime.signal },
);
+ root.addEventListener("keydown", e => {
+ if (e.target.getAttribute?.("role") !== "tab" || !["ArrowLeft", "ArrowRight", "Home", "End"].includes(e.key)) return;
+ const tabs = [...root.querySelectorAll('[role="tab"]')];
+ let index = tabs.indexOf(e.target);
+ if (e.key === "Home") index = 0;
+ else if (e.key === "End") index = tabs.length - 1;
+ else index = (index + (e.key === "ArrowRight" ? 1 : -1) + tabs.length) % tabs.length;
+ e.preventDefault(); tab = tabs[index].dataset.obs; draw(); tabs[index].focus();
+ }, {signal:lifetime.signal});
input.addEventListener(
"input",
() => {
diff --git a/harness/service-ui/server.mjs b/harness/service-ui/server.mjs
index c757f506..92e6b2d9 100644
--- a/harness/service-ui/server.mjs
+++ b/harness/service-ui/server.mjs
@@ -104,8 +104,10 @@ function allowedRecording(path) {
`^/v1/sessions/${id}/observability/${id}/timeline$`,
).test(u.pathname);
const summary = new RegExp(
- `^/v1/sessions/${id}/observability/${id}/summary$`,
+ `^/v1/sessions/${id}/observability/${id}/(?:summary|subtree-summary)$`,
).test(u.pathname);
+ const children = new RegExp(`^/v1/sessions/${id}/observability/${id}/children$`).test(u.pathname);
+ if(children) return !u.search;
const health = u.pathname === "/v1/observability/health";
if (!listing && !timeline && !summary && !health) return false;
try {
diff --git a/harness/store-postgres/src/db.rs b/harness/store-postgres/src/db.rs
index 9a43b713..badeae80 100644
--- a/harness/store-postgres/src/db.rs
+++ b/harness/store-postgres/src/db.rs
@@ -509,6 +509,23 @@ impl Database {
if tx.execute("INSERT INTO sessions(id,data,owner,epoch,expires) VALUES($1,'{}',$2,1,clock_timestamp()+$3::bigint*interval '1 millisecond') ON CONFLICT DO NOTHING", &[&s.id,&self.owner,&self.lease_ms])?!=1 {bail!("Session already exists; claim it explicitly");}
1
};
+ // Controllers acquire team -> environment. Terminal reconciliation also
+ // needs the team row, so take it BEFORE write_metadata locks the bound
+ // environment. Otherwise a child save and a reception controller can
+ // deadlock (environment -> team versus team -> environment). This is a
+ // lock-order fix, not permission to retry any uncertain external action.
+ if s.closed
+ || s.turn
+ .as_ref()
+ .is_some_and(|t| openworker_runtime::store::terminal(&t.status))
+ {
+ if let Some(member) = &s.agent_team {
+ tx.query_one(
+ "SELECT data FROM agent_teams WHERE id=$1 FOR UPDATE",
+ &[&member.team_id],
+ )?;
+ }
+ }
append_history(&mut tx, s)?;
write_metadata(&mut tx, s)?;
openworker_runtime::subagents::terminal_saved(&mut crate::agent_team::Tx(&mut tx), s)?;
diff --git a/harness/store-postgres/src/lib.rs b/harness/store-postgres/src/lib.rs
index 8e63150e..7e216238 100644
--- a/harness/store-postgres/src/lib.rs
+++ b/harness/store-postgres/src/lib.rs
@@ -583,6 +583,16 @@ impl StateStore for PostgresStore {
) -> Result<Value> {
PostgresStore::compliance_page(self, sid, q)
}
+ fn observation_children(
+ &self,
+ sid: &str,
+ tid: &str,
+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {
+ PostgresStore::observation_children(self, sid, tid)
+ }
+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {
+ self.observation_run(sid, tid).map(|_| true)
+ }
fn supports_agent_teams(&self) -> bool {
true
}
diff --git a/harness/store-postgres/src/run_query.rs b/harness/store-postgres/src/run_query.rs
index 026ebe30..3ab8ae91 100644
--- a/harness/store-postgres/src/run_query.rs
+++ b/harness/store-postgres/src/run_query.rs
@@ -209,3 +209,52 @@ impl PostgresStore {
self.call(move |db| db.observation_run(&sid, &tid))
}
}
+
+impl Database {
+ pub(crate) fn observation_children(
+ &mut self,
+ sid: &str,
+ tid: &str,
+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {
+ let mut tx = self.client.transaction()?;
+ auth::guard(&mut tx, self.auth.as_ref(), Action::Read, Some(sid), None)?;
+ let row=tx.query_opt("SELECT t.data FROM sessions s JOIN agent_teams t ON t.id=s.data->'agent_team'->>'team_id' WHERE s.id=$1",&[&sid])?;
+ let mut visible = Vec::new();
+ if let Some(row) = row {
+ let doc = serde_json::from_value(row.get(0))?;
+ for mut child in openworker_runtime::observation_links::children(&doc, sid, tid) {
+ // Parent visibility never grants access to a child's identifiers or usage.
+ match auth::guard(
+ &mut tx,
+ self.auth.as_ref(),
+ Action::Read,
+ Some(&child.child_session_id),
+ None,
+ ) {
+ Ok(_) => {
+ child.child_turn_id=tx.query_opt("SELECT turn_id FROM submissions WHERE session_id=$1 AND turn_id IS NOT NULL ORDER BY sequence LIMIT 1", &[&child.child_session_id])?.and_then(|r|r.get(0));
+ visible.push(child);
+ }
+ Err(e)
+ if matches!(
+ e.downcast_ref::<auth::Error>(),
+ Some(auth::Error::Forbidden)
+ ) => {}
+ Err(e) => return Err(e),
+ }
+ }
+ }
+ tx.commit()?;
+ Ok(visible)
+ }
+}
+impl PostgresStore {
+ pub fn observation_children(
+ &self,
+ sid: &str,
+ tid: &str,
+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {
+ let (sid, tid) = (sid.to_owned(), tid.to_owned());
+ self.call(move |db| db.observation_children(&sid, &tid))
+ }
+}
diff --git a/harness/store-postgres/tests/environments.rs b/harness/store-postgres/tests/environments.rs
index 2581d7fc..fc7fb7e2 100644
--- a/harness/store-postgres/tests/environments.rs
+++ b/harness/store-postgres/tests/environments.rs
@@ -421,3 +421,68 @@ async fn resilience_recovered_runtime_shutdown_retires_old_attachments_without_d
);
assert!(!next.get("recovered").unwrap().closed);
}
+
+#[test]
+#[ignore = "requires isolated OW_PG_TEST_URL"]
+fn terminal_team_save_does_not_hold_environment_while_waiting_for_team() {
+ let c = config();
+ let mut store = PostgresStore::open(c.clone()).unwrap();
+ let binding = ready(&store, Sharing::Shared, Management::Harness);
+ let lead = session("lock-order-lead", binding.clone());
+ store.save(&lead, "created", json!({})).unwrap();
+ let team = store
+ .agent_team(
+ &lead.id,
+ "create",
+ &json!({"agent_id":"lead","agent_types":["builder"]}),
+ None,
+ )
+ .unwrap();
+ let team_id = team["id"].as_str().unwrap().to_owned();
+ let mut closed = store.get(&lead.id).unwrap();
+ closed.closed = true;
+ let mut controller = admin(&c);
+ let mut monitor = admin(&c);
+ let mut tx = controller.transaction().unwrap();
+ tx.query_one(
+ "SELECT data FROM agent_teams WHERE id=$1 FOR UPDATE",
+ &[&team_id],
+ )
+ .unwrap();
+ let worker = std::thread::spawn(move || store.save(&closed, "session_closed", json!({})));
+ // Wait until this exact schema's worker is blocked on the held team row.
+ let relation = format!("{}.agent_teams", c.schema);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let waiting = loop {
+ let blocked:bool=monitor.query_one("SELECT EXISTS(SELECT 1 FROM pg_stat_activity a JOIN pg_locks l ON l.pid=a.pid WHERE l.relation=$1::text::regclass AND a.wait_event_type='Lock' AND a.query LIKE '%agent_teams%FOR UPDATE%')",&[&relation]).unwrap().get(0);
+ if blocked {
+ break true;
+ }
+ if std::time::Instant::now() > deadline {
+ break false;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ };
+ tx.batch_execute("SET LOCAL lock_timeout='500ms'").unwrap();
+ let can_read_environment = tx
+ .query_one(
+ "SELECT data FROM environments WHERE id=$1 FOR SHARE",
+ &[&binding.environment_id],
+ )
+ .is_ok();
+ // Always release test locks before joining or asserting, even on failure.
+ tx.rollback().unwrap();
+ let saved = worker.join().unwrap();
+ drop(controller);
+ drop(monitor);
+ postgres::Client::connect(&c.connection, postgres::NoTls)
+ .unwrap()
+ .batch_execute(&format!("DROP SCHEMA {} CASCADE", c.schema))
+ .unwrap();
+ assert!(waiting, "Worker never reached the team lock");
+ assert!(
+ can_read_environment,
+ "Worker held environment while waiting for team: reversed lock order"
+ );
+ saved.unwrap();
+}
diff --git a/harness/telemetry/src/config.rs b/harness/telemetry/src/config.rs
new file mode 100644
index 00000000..300e8756
--- /dev/null
+++ b/harness/telemetry/src/config.rs
@@ -0,0 +1,108 @@
+//! Export configuration remains available without linking any exporter.
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Config {
+ /// Collector or vendor OTLP base URL. Signal paths (`/v1/traces` ...) are
+ /// appended. A base path is allowed, e.g. `https://cloud.langfuse.com/api/public/otel`.
+ pub endpoint: String,
+ #[serde(default = "default_deployment")]
+ pub deployment: String,
+ /// Extra request headers, typically authentication. Requires HTTPS unless
+ /// the endpoint is loopback.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub headers: Vec<Header>,
+ /// Signals to export. Defaults to all three; vendors that only ingest
+ /// traces (e.g. Langfuse) should select `traces` only.
+ #[serde(default = "all_signals")]
+ pub signals: Vec<Signal>,
+ /// Explicit receiver dialect; standard OTLP remains the default.
+ #[serde(default)]
+ pub acknowledgement: Acknowledgement,
+ /// Opt-in content capture. Off by default: records stay content-free.
+ #[serde(default)]
+ pub content: crate::ContentPolicy,
+}
+impl Default for Config {
+ fn default() -> Self {
+ Self {
+ endpoint: String::new(),
+ deployment: default_deployment(),
+ headers: vec![],
+ signals: all_signals(),
+ acknowledgement: Acknowledgement::default(),
+ content: crate::ContentPolicy::default(),
+ }
+ }
+}
+fn default_deployment() -> String {
+ "default".into()
+}
+fn all_signals() -> Vec<Signal> {
+ vec![Signal::Traces, Signal::Metrics, Signal::Logs]
+}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Signal {
+ Traces,
+ Metrics,
+ Logs,
+}
+/// Langfuse v4 currently returns a queue-job JSON receipt for nonempty batches.
+/// Select it explicitly; arbitrary HTTP 200 bodies are never assumed successful.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Acknowledgement {
+ #[default]
+ Otlp,
+ LangfuseV4,
+}
+/// One request header. Exactly one of `value` (non-secret literal, e.g. a
+/// protocol version) or `value_env` (name of an environment variable holding
+/// the secret, e.g. `Basic <base64>` or an API key) must be set.
+#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Header {
+ pub name: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub value: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub value_env: Option<String>,
+}
+impl Header {
+ /// Secret header whose value is read from environment variable `env`.
+ pub fn secret(name: impl Into<String>, env: impl Into<String>) -> Self {
+ Self {
+ name: name.into(),
+ value_env: Some(env.into()),
+ ..Self::default()
+ }
+ }
+ /// Non-secret literal header.
+ pub fn literal(name: impl Into<String>, value: impl Into<String>) -> Self {
+ Self {
+ name: name.into(),
+ value: Some(value.into()),
+ ..Self::default()
+ }
+ }
+}
+
+impl Config {
+ /// Direct Langfuse v4 export. The trusted host resolves the credential environment variable.
+ pub fn langfuse(endpoint: impl Into<String>, authorization_env: impl Into<String>) -> Self {
+ Self {
+ endpoint: endpoint.into(),
+ signals: vec![Signal::Traces],
+ acknowledgement: Acknowledgement::LangfuseV4,
+ headers: vec![
+ Header::secret("Authorization", authorization_env),
+ Header::literal("x-langfuse-ingestion-version", "4"),
+ ],
+ content: crate::ContentPolicy {
+ capture: crate::Capture::Metadata,
+ ..Default::default()
+ },
+ ..Default::default()
+ }
+ }
+}
diff --git a/harness/telemetry/src/lib.rs b/harness/telemetry/src/lib.rs
index 1fa3aa8e..ad82ef8a 100644
--- a/harness/telemetry/src/lib.rs
+++ b/harness/telemetry/src/lib.rs
@@ -12,6 +12,7 @@ use std::{
};
mod accounting;
pub mod attributes;
+pub mod config;
mod observation;
pub use accounting::{ModelPurpose, TokenUsage};
#[cfg(feature = "otel")]
@@ -391,6 +392,7 @@ pub trait Sink: Send + 'static {
}
}
enum Message {
+ Flush(mpsc::SyncSender<bool>),
Record(Record),
Observations([Observation; Gauge::ALL.len()], usize),
}
@@ -485,11 +487,13 @@ impl Telemetry {
break;
}
if !batch.is_empty() {
+ let mut acknowledgements = Vec::new();
let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut records = Vec::new();
let mut observations = Vec::new();
for message in batch.drain(..) {
match message {
+ Message::Flush(ack) => acknowledgements.push(ack),
Message::Record(r) => records.push(r),
Message::Observations(o, n) => {
observations.extend_from_slice(&o[..n])
@@ -505,6 +509,9 @@ impl Telemetry {
if !ok {
s.failures.fetch_add(1, Ordering::Relaxed);
}
+ for ack in acknowledgements {
+ let _ = ack.try_send(ok);
+ }
batch.clear();
}
if t.load(Ordering::Acquire) {
@@ -532,6 +539,19 @@ impl Telemetry {
},
))
}
+ /// Bounded barrier for records already enqueued. Success is not durable or
+ /// lossless delivery; inspect counters for earlier drops/rejections. Never retries.
+ pub fn flush(&self, budget: Duration) -> bool {
+ let Some(inner) = &self.0 else {
+ return true;
+ };
+ let (ack, rx) = mpsc::sync_channel(1);
+ if inner.stop.load(Ordering::Acquire) || inner.tx.try_send(Message::Flush(ack)).is_err() {
+ return false;
+ }
+ rx.recv_timeout(budget.min(Duration::from_secs(5)))
+ .unwrap_or(false)
+ }
pub fn enabled(&self) -> bool {
self.0.is_some()
}
diff --git a/harness/telemetry/src/otlp.rs b/harness/telemetry/src/otlp.rs
index a2da5ec7..815b242d 100644
--- a/harness/telemetry/src/otlp.rs
+++ b/harness/telemetry/src/otlp.rs
@@ -14,92 +14,8 @@ use opentelemetry_sdk::{
};
use std::{sync::Arc, time::Duration};
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct Config {
- /// Collector or vendor OTLP base URL. Signal paths (`/v1/traces` ...) are
- /// appended. A base path is allowed, e.g. `https://cloud.langfuse.com/api/public/otel`.
- pub endpoint: String,
- #[serde(default = "default_deployment")]
- pub deployment: String,
- /// Extra request headers, typically authentication. Requires HTTPS unless
- /// the endpoint is loopback.
- #[serde(default, skip_serializing_if = "Vec::is_empty")]
- pub headers: Vec<Header>,
- /// Signals to export. Defaults to all three; vendors that only ingest
- /// traces (e.g. Langfuse) should select `traces` only.
- #[serde(default = "all_signals")]
- pub signals: Vec<Signal>,
- /// Explicit receiver dialect; standard OTLP remains the default.
- #[serde(default)]
- pub acknowledgement: Acknowledgement,
- /// Opt-in content capture. Off by default: records stay content-free.
- #[serde(default)]
- pub content: crate::ContentPolicy,
-}
-impl Default for Config {
- fn default() -> Self {
- Self {
- endpoint: String::new(),
- deployment: default_deployment(),
- headers: vec![],
- signals: all_signals(),
- acknowledgement: Acknowledgement::default(),
- content: crate::ContentPolicy::default(),
- }
- }
-}
-fn default_deployment() -> String {
- "default".into()
-}
-fn all_signals() -> Vec<Signal> {
- vec![Signal::Traces, Signal::Metrics, Signal::Logs]
-}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum Signal {
- Traces,
- Metrics,
- Logs,
-}
-/// Langfuse v4 currently returns a queue-job JSON receipt for nonempty batches.
-/// Select it explicitly; arbitrary HTTP 200 bodies are never assumed successful.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum Acknowledgement {
- #[default]
- Otlp,
- LangfuseV4,
-}
-/// One request header. Exactly one of `value` (non-secret literal, e.g. a
-/// protocol version) or `value_env` (name of an environment variable holding
-/// the secret, e.g. `Basic <base64>` or an API key) must be set.
-#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct Header {
- pub name: String,
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub value: Option<String>,
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub value_env: Option<String>,
-}
+pub use crate::config::{Acknowledgement, Config, Header, Signal};
impl Header {
- /// Secret header whose value is read from environment variable `env`.
- pub fn secret(name: impl Into<String>, env: impl Into<String>) -> Self {
- Self {
- name: name.into(),
- value_env: Some(env.into()),
- ..Self::default()
- }
- }
- /// Non-secret literal header.
- pub fn literal(name: impl Into<String>, value: impl Into<String>) -> Self {
- Self {
- name: name.into(),
- value: Some(value.into()),
- ..Self::default()
- }
- }
fn resolve(&self) -> Result<(http::HeaderName, http::HeaderValue)> {
let name = http::HeaderName::from_bytes(self.name.trim().to_ascii_lowercase().as_bytes())
.context("Invalid telemetry header name")?;
diff --git a/harness/telemetry/tests/boundaries.rs b/harness/telemetry/tests/boundaries.rs
index 7415b492..dbd403bb 100644
--- a/harness/telemetry/tests/boundaries.rs
+++ b/harness/telemetry/tests/boundaries.rs
@@ -232,3 +232,19 @@ fn shared_registry_bounds_names_and_content_and_reserves_typed_fields() {
.iter()
.all(|(k, v)| export_attribute(k, v) && v.len() <= 67));
}
+
+#[test]
+fn flush_is_a_bounded_barrier_without_stopping_instrumentation() {
+ let rows = Arc::new(Mutex::new(vec![]));
+ let (t, c) = Telemetry::start(Capture(rows.clone()), 32).unwrap();
+ t.event(Operation::Command, Outcome::Ok, None, None);
+ assert!(t.flush(Duration::from_secs(2)));
+ assert_eq!(rows.lock().unwrap().len(), 1);
+ t.event(Operation::Command, Outcome::Ok, None, None);
+ assert!(t.flush(Duration::from_secs(2)));
+ assert_eq!(rows.lock().unwrap().len(), 2);
+ assert_eq!(t.snapshot().accepted, 2);
+ assert!(c.shutdown(Duration::from_secs(2)));
+ assert!(!t.flush(Duration::from_millis(10)));
+ assert!(Telemetry::default().flush(Duration::ZERO));
+}
diff --git a/harness/tests/sdk-storage-shard.mjs b/harness/tests/sdk-storage-shard.mjs
index a635fba9..8b2c41ad 100644
--- a/harness/tests/sdk-storage-shard.mjs
+++ b/harness/tests/sdk-storage-shard.mjs
@@ -27,7 +27,9 @@ if (process.argv.includes("--list")) {
if (result.error) throw result.error;
if (result.status !== 0) process.exit(result.status ?? 1);
};
- run(["test", "--locked", "-p", "openworker-sdk", "--features", "test-support",
+ process.env.OW_OBSERVABILITY_TEST_URL = process.env.OW_PG_TEST_URL;
+ const features = shard === "core" ? "test-support,recording-postgres,evaluation-export" : shard === "teams" ? "test-support,recording" : "test-support";
+ run(["test", "--locked", "-p", "openworker-sdk", "--features", features,
...(shard === "core" ? ["--lib", "--bins"] : []),
...groups[shard].flatMap((name) => ["--test", name]),
"--", "--include-ignored", "--test-threads=1"]);
diff --git a/harness/trace-viewer/LICENSE b/harness/trace-viewer/LICENSE
new file mode 100644
index 00000000..60d38246
--- /dev/null
+++ b/harness/trace-viewer/LICENSE
@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2024 Andrew Ng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/harness/trace-viewer/README.md b/harness/trace-viewer/README.md
new file mode 100644
index 00000000..8ccb084d
--- /dev/null
+++ b/harness/trace-viewer/README.md
@@ -0,0 +1,50 @@
+# Optional turn trace viewer
+
+This standalone browser component uses the management UI's timeline, usage and
+model/tool summaries. It adds no dependency to the Rust harness or TypeScript SDK.
+It is a local source package pending release review, not a published npm package.
+
+```js
+import {mountTurnTrace} from './src/index.js';
+import './src/style.css';
+
+const view = mountTurnTrace(document.querySelector('#trace'), {
+ adapter, // timeline(query), summary(query); see src/index.d.ts
+ sessionId, turnId,
+ onChild(child) { openTrace(child.child_session_id, child.child_turn_id); },
+});
+// On route change: view.dispose(). To reload: view.refresh().
+```
+
+The browser adapter calls your application's authenticated API. Keep harness,
+Langfuse and database credentials in the trusted backend. Check session ownership
+on every request. Optional adapter methods add `children`, `subtreeSummary` and
+`evaluations` panels. The viewer never starts an agent or calls an evaluator.
+
+Each instance has independent DOM IDs, requests and disposal. Timeline pages load
+100 records at a time, to a browser cap of 1,000. Summaries use the server's separate
+bounded scan; missing counters and truncation remain visible. Evaluations show the
+first 25 results; later pages remain available through the SDK query API.
+
+Linked work means the root turn and authorized spawned initial turns (up to 25),
+not every later message in those child sessions. Tokens count model attempts once;
+cache/reasoning counts are subsets. Concurrent task durations are not wall time.
+
+`node sync.mjs` copies the canonical modules into the management UI's static
+assets. Run `node sync.mjs --check` to catch drift. These are generated copies;
+edit this package's source and sync it instead of maintaining two viewers.
+
+For a loopback preview of qualification data:
+
+```sh
+node examples/serve.mjs /absolute/path/to/qualification.json 59602
+```
+
+Browser qualification (using an existing Playwright installation):
+
+```sh
+node test/browser.mjs http://127.0.0.1:59602 /absolute/path/to/playwright/index.mjs /tmp/trace-screenshots
+```
+
+This checks keyboard tabs, exact child navigation, independent mounted instances,
+HTML escaping, disposal, and the 1,000-record pagination limit.
diff --git a/harness/trace-viewer/examples/index.html b/harness/trace-viewer/examples/index.html
new file mode 100644
index 00000000..8906d860
--- /dev/null
+++ b/harness/trace-viewer/examples/index.html
@@ -0,0 +1,6 @@
+<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>OpenWorker · Turn trace</title><link rel="stylesheet" href="/src/style.css"><style>body{margin:0;background:#f6f7f9}header{padding:20px 40px;background:white;border-bottom:1px solid #e7e9ee;font:15px -apple-system,BlinkMacSystemFont,sans-serif;color:#20242b}header span{color:#737b89;margin-left:16px;font-size:13px}main{margin:32px auto;border-radius:16px;max-width:1120px;overflow:hidden;box-shadow:0 4px 24px #17243508}</style><header><strong>OpenWorker</strong><span>SDK turn trace</span></header><main id="trace"></main><script type="module">
+import {mountTurnTrace} from '/src/index.js';
+const evidence=await fetch('/trace.json').then(r=>r.json());
+const adapter={timeline:async()=>evidence.timeline,summary:async()=>evidence.summary,evaluations:async()=>({results:evidence.evaluationRecord?[{cursor:1,evaluation:evidence.evaluationRecord}]:[],next:null})};
+window.viewer=mountTurnTrace(document.querySelector('#trace'),{adapter,sessionId:evidence.timeline.run.session_id,turnId:evidence.timeline.run.turn_id});
+</script></html>
diff --git a/harness/trace-viewer/examples/serve.mjs b/harness/trace-viewer/examples/serve.mjs
new file mode 100644
index 00000000..49adec6b
--- /dev/null
+++ b/harness/trace-viewer/examples/serve.mjs
@@ -0,0 +1,10 @@
+// Loopback-only viewer for a selected synthetic qualification result.
+import {createServer} from 'node:http';
+import {readFile} from 'node:fs/promises';
+const [evidence,port='59602']=process.argv.slice(2);
+if(!evidence)throw Error('Usage: node examples/serve.mjs EVIDENCE_JSON [PORT]');
+const files=new Map([['/',new URL('index.html',import.meta.url)],...['index.js','observability.js','observability-data.js','style.css'].map(f=>[`/src/${f}`,new URL(`../src/${f}`,import.meta.url)])]);
+const types={js:'text/javascript',html:'text/html',css:'text/css',json:'application/json'};
+createServer(async(req,res)=>{
+ try {const path=new URL(req.url,'http://local').pathname;const file=path==='/trace.json'?evidence:files.get(path);if(!file){res.writeHead(404);res.end();return;}const body=await readFile(file);res.writeHead(200,{'content-type':path==='/'?'text/html':types[path.split('.').pop()]??'text/plain','cache-control':'no-store'});res.end(body);}catch{res.writeHead(500);res.end('Unavailable');}
+}).listen(Number(port),'127.0.0.1',()=>console.log(`Viewer: http://127.0.0.1:${port}`));
diff --git a/harness/trace-viewer/package.json b/harness/trace-viewer/package.json
new file mode 100644
index 00000000..785f5a03
--- /dev/null
+++ b/harness/trace-viewer/package.json
@@ -0,0 +1,25 @@
+{
+ "name": "@openworkerai/harness-trace-viewer",
+ "version": "0.1.0-beta.2",
+ "private": true,
+ "type": "module",
+ "description": "Optional browser trace viewer for OpenWorker SDK applications",
+ "exports": {
+ ".": {
+ "types": "./src/index.d.ts",
+ "import": "./src/index.js"
+ },
+ "./style.css": "./src/style.css"
+ },
+ "files": [
+ "src",
+ "README.md",
+ "LICENSE"
+ ],
+ "scripts": {
+ "sync": "node sync.mjs",
+ "check": "node sync.mjs --check"
+ },
+ "types": "./src/index.d.ts",
+ "license": "MIT"
+}
diff --git a/harness/trace-viewer/src/index.d.ts b/harness/trace-viewer/src/index.d.ts
new file mode 100644
index 00000000..ef2b7af7
--- /dev/null
+++ b/harness/trace-viewer/src/index.d.ts
@@ -0,0 +1,10 @@
+export interface TraceAdapter {
+ timeline(query: {sessionId:string;turnId:string;after?:number;through?:number;limit?:number}): Promise<unknown>;
+ summary(query: {sessionId:string;turnId:string;through?:number}): Promise<unknown>;
+ children?(query: {sessionId:string;turnId:string}): Promise<unknown>;
+ subtreeSummary?(query: {sessionId:string;turnId:string}): Promise<unknown>;
+ evaluations?(query: {sessionId:string;turnId:string;limit?:number}): Promise<unknown>;
+}
+export interface ChildLink {child_session_id:string;child_turn_id:string|null;agent_id:string;phase:string}
+export function mountTurnTrace(root:HTMLElement, options:{adapter:TraceAdapter;sessionId:string;turnId:string;onChild?:(child:ChildLink)=>void}): {refresh():void;dispose():void};
+export function mountObservability(root:HTMLElement, options:{request:(path:string,signal?:AbortSignal)=>Promise<any>;isAdmin?:boolean;initialRun?:{session_id:string;turn_id:string}|null;directory?:boolean;openSession?:boolean;children?:boolean}):()=>void;
diff --git a/harness/trace-viewer/src/index.js b/harness/trace-viewer/src/index.js
new file mode 100644
index 00000000..968d61f2
--- /dev/null
+++ b/harness/trace-viewer/src/index.js
@@ -0,0 +1,75 @@
+import {mountObservability} from './observability.js';
+import {count, measurement} from './observability-data.js';
+export {mountObservability};
+
+/** Mount one turn. The trusted application owns authorization and transport. */
+export function mountTurnTrace(root, {adapter, sessionId, turnId, onChild}) {
+ root.classList.add('ow-trace-viewer');
+ const query = {sessionId, turnId};
+ const request = async path => {
+ const url = new URL(path, 'https://viewer.invalid');
+ const match = /^\/v1\/sessions\/([^/]+)\/observability\/([^/]+)\/(timeline|summary)$/.exec(url.pathname);
+ if (!match || decodeURIComponent(match[1]) !== sessionId || decodeURIComponent(match[2]) !== turnId)
+ throw Error('Unsupported trace scope');
+ const q = {...query};
+ for (const key of ['after', 'through', 'limit'])
+ if (url.searchParams.has(key)) q[key] = Number(url.searchParams.get(key));
+ return adapter[match[3]](q);
+ };
+ const unmount = mountObservability(root, {request, initialRun: {session_id: sessionId, turn_id: turnId}, directory: false, openSession: false});
+ const extra = document.createElement('div'); root.append(extra);
+ let disposed = false, generation = 0;
+ const paragraph = text => { const p = document.createElement('p'); p.textContent = text; return p; };
+ const section = title => {
+ const node = document.createElement('section'); node.className = 'obs-panel';
+ const h = document.createElement('h2'); h.textContent = title; node.append(h); return node;
+ };
+ async function details() {
+ const mine = ++generation;
+ extra.replaceChildren();
+ const tasks = [];
+ if (adapter.children) tasks.push(['Sub-agents', async () => {
+ const page = await adapter.children(query), node = section('Sub-agents');
+ for (const child of page.children) {
+ if (onChild && child.child_turn_id) {
+ const b = document.createElement('button'); b.className = 'button';
+ b.textContent = `${child.agent_id} · ${child.phase}`;
+ b.addEventListener('click', () => onChild(child)); node.append(b);
+ } else node.append(paragraph(`${child.agent_id} · ${child.phase}`));
+ }
+ if (!page.children.length) node.append(paragraph('No visible recorded child links for this turn.'));
+ return node;
+ }]);
+ if (adapter.subtreeSummary) tasks.push(['Linked work', async () => {
+ const tree = await adapter.subtreeSummary(query), node = section('Linked work');
+ node.append(paragraph(`${count(tree.selection.runs.length)} visible initial tasks${tree.selection.selection_truncated ? ' · selection limit reached' : ''}`));
+ node.append(paragraph(`Input: ${measurement(tree.summary.usage.input)} · Output: ${measurement(tree.summary.usage.output)} tokens`));
+ node.append(paragraph('Includes this turn and spawned initial tasks. Later messages are separate turns; concurrent durations are not elapsed time.'));
+ return node;
+ }]);
+ if (adapter.evaluations) tasks.push(['Evaluations', async () => {
+ const page = await adapter.evaluations({...query, limit: 25}), node = section('Evaluations');
+ for (const row of page.results) {
+ const e = row.evaluation.result;
+ const details = document.createElement('details'); details.className = 'obs-details';
+ const summary = document.createElement('summary');
+ summary.textContent = `${e.name} · ${e.status === 'completed' ? String(e.score?.value) : e.status}`;
+ details.append(summary, paragraph(`${e.evaluator.name} · ${e.evaluator.version} · ${e.evaluator.source}`));
+ if (e.explanation) details.append(paragraph(e.explanation));
+ if (e.supersedes) details.append(paragraph(`Corrects ${e.supersedes}; original evidence retained.`));
+ node.append(details);
+ }
+ if (!page.results.length) node.append(paragraph('No recorded evaluations.'));
+ if (page.next !== null) node.append(paragraph('Showing the first 25 evaluations. Use the query API to read later pages.'));
+ return node;
+ }]);
+ const results = await Promise.all(tasks.map(async ([title, task]) => {
+ try {return await task();} catch {const node = section(title); node.append(paragraph('Unavailable or not enabled for this application.')); return node;}
+ }));
+ if (!disposed && mine === generation) extra.replaceChildren(...results);
+ }
+ const refresh = () => {root.dispatchEvent(new Event('ow-trace-refresh')); details();};
+ const click = e => {if (e.target.closest('[data-obs="refresh"]')) details();};
+ root.addEventListener('click', click); details();
+ return {refresh, dispose() {disposed = true; generation++; root.removeEventListener('click', click); unmount(); root.replaceChildren(); root.classList.remove('ow-trace-viewer');}};
+}
diff --git a/harness/trace-viewer/src/observability-data.js b/harness/trace-viewer/src/observability-data.js
new file mode 100644
index 00000000..0ce44467
--- /dev/null
+++ b/harness/trace-viewer/src/observability-data.js
@@ -0,0 +1,153 @@
+// Pure presentation helpers. Provider-reported values stay optional; they are
+// neither a bill nor inferred per-tool token costs.
+export const esc = (s) =>
+ String(s ?? "").replace(
+ /[&<>"']/g,
+ (c) =>
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[
+ c
+ ],
+ );
+export const known = (n) =>
+ typeof n === "number" &&
+ Number.isFinite(n) &&
+ n >= 0 &&
+ n <= Number.MAX_SAFE_INTEGER;
+export const count = (n) =>
+ known(n) ? n.toLocaleString("en-US", { maximumFractionDigits: 0 }) : "—";
+export function duration(us) {
+ if (!known(us)) return "—";
+ if (us === 0) return "0 ms";
+ if (us < 1000) return "<1 ms";
+ if (us < 1000000)
+ return `${(us / 1000).toLocaleString("en-US", { maximumFractionDigits: 1 })} ms`;
+ if (us < 60000000)
+ return `${(us / 1000000).toLocaleString("en-US", { maximumFractionDigits: 1 })} s`;
+ return `${(us / 60000000).toLocaleString("en-US", { maximumFractionDigits: 1 })} min`;
+}
+export const measurement = (m, formatter = count) =>
+ `${formatter(m?.total)}${m?.unknown > 0 && known(m.total) ? " + unknown" : ""}`;
+export const average = (m) =>
+ m?.known > 0 && known(m.total) ? m.total / m.known : null;
+export const runKey = (r) => `${r.session_id}/${r.turn_id}`;
+export const purpose = (p) =>
+ ({
+ main: "Main agent",
+ reviewer: "Auto-approver",
+ auto_approver: "Auto-approver",
+ compaction: "Compaction",
+ summary: "Compaction",
+ helper: "Helper",
+ title: "Title",
+ })[p] ||
+ p ||
+ "Unspecified";
+export const phases = [
+ ["model", "Model requests"],
+ ["tool", "Tool execution"],
+ ["policy", "Policy checks"],
+ ["approval_wait", "Waiting for approval"],
+ ["question_wait", "Waiting for answer"],
+ ["other_active", "Other activity"],
+ ["parallel", "Parallel work"],
+ ["unobserved_gap", "Unobserved gaps"],
+];
+export function usageTotal(u) {
+ // A combined token number is only useful if both counters were reported.
+ if (!known(u?.input?.total) || !known(u?.output?.total)) return null;
+ const total = u.input.total + u.output.total;
+ return Number.isSafeInteger(total) ? total : null;
+}
+export function purposeUsage(models = []) {
+ const groups = new Map();
+ for (const m of models) {
+ const name = purpose(m.purpose),
+ current = groups.get(name) || { name, total: 0, missing: 0, known: 0 };
+ const total = usageTotal(m.usage);
+ if (known(total)) {
+ current.total += total;
+ current.known++;
+ }
+ if (!known(total) || m.usage?.input?.unknown || m.usage?.output?.unknown)
+ current.missing++;
+ groups.set(name, current);
+ }
+ return [...groups.values()].map((g) => ({
+ ...g,
+ total: g.known && Number.isSafeInteger(g.total) ? g.total : null,
+ }));
+}
+export function coverageNotes(c = {}) {
+ const notes = [];
+ if (!c.scanned_steps) notes.push("No recorded steps for this selection.");
+ if (c.scan_truncated) notes.push("The 4,096-step summary limit was reached.");
+ if (c.group_truncated)
+ notes.push(
+ "Only the first 128 model/tool groups are listed; group sums may differ from totals.",
+ );
+ if (known(c.expected_segments) && c.recorded_segments !== c.expected_segments)
+ notes.push(
+ `${count(c.recorded_segments)} of ${count(c.expected_segments)} execution segments recorded.`,
+ );
+ if (c.scanned_steps && !c.all_visible_segments_closed)
+ notes.push("Visible execution segments are unfinished or missing.");
+ if (c.pending_waits > 0)
+ notes.push(`${count(c.pending_waits)} pending human response(s).`);
+ if (c.requests_without_completion > 0)
+ notes.push(
+ `${count(c.requests_without_completion)} model request(s) without a completion record.`,
+ );
+ if (c.completions_without_start > 0)
+ notes.push(
+ `${count(c.completions_without_start)} model completion(s) without a start record.`,
+ );
+ if (c.unknown_timing_steps > 0)
+ notes.push(
+ `${count(c.unknown_timing_steps)} step(s) without measured duration.`,
+ );
+ if (c.unknown_outcome_steps > 0)
+ notes.push(
+ `${count(c.unknown_outcome_steps)} step(s) with unknown outcome.`,
+ );
+ if (c.clock_or_range_anomaly)
+ notes.push("Clock or timing range anomalies were detected.");
+ if (c.hierarchy_incomplete)
+ notes.push(
+ "Parent relationships are missing or cyclic; phase attribution may be incomplete.",
+ );
+ return notes;
+}
+export function timelineLayout(records) {
+ const steps = records
+ .filter((r) => known(r.step.started_ms))
+ .slice()
+ .sort(
+ (a, b) => a.step.started_ms - b.step.started_ms || a.cursor - b.cursor,
+ );
+ const starts = steps.map((r) => r.step.started_ms * 1000);
+ const from = starts.length ? Math.min(...starts) : 0;
+ const to = steps.length
+ ? Math.max(
+ ...steps.map(
+ (r) =>
+ r.step.started_ms * 1000 +
+ (known(r.step.duration_us) ? r.step.duration_us : 0),
+ ),
+ )
+ : from;
+ const span = Math.max(1, to - from);
+ return {
+ from,
+ to,
+ rows: steps.map((r) => ({
+ ...r,
+ left: Math.max(
+ 0,
+ Math.min(100, ((r.step.started_ms * 1000 - from) / span) * 100),
+ ),
+ width: known(r.step.duration_us)
+ ? Math.max(0, Math.min(100, (r.step.duration_us / span) * 100))
+ : 0,
+ })),
+ };
+}
diff --git a/harness/trace-viewer/src/observability.js b/harness/trace-viewer/src/observability.js
new file mode 100644
index 00000000..9573abf3
--- /dev/null
+++ b/harness/trace-viewer/src/observability.js
@@ -0,0 +1,685 @@
+import {
+ esc,
+ known,
+ count,
+ duration,
+ measurement,
+ average,
+ runKey,
+ purpose,
+ phases,
+ usageTotal,
+ purposeUsage,
+ coverageNotes,
+ timelineLayout,
+} from "./observability-data.js";
+const query = (params) =>
+ new URLSearchParams(
+ Object.entries(params).filter(
+ ([, v]) => v !== null && v !== undefined && v !== "",
+ ),
+ ).toString();
+const perRun = (r) =>
+ `/v1/sessions/${encodeURIComponent(r.session_id)}/observability/${encodeURIComponent(r.turn_id)}`;
+const cell = (value) => `<td>${esc(value)}</td>`;
+const hint = (text) => `<p class="obs-hint">${esc(text)}</p>`;
+const notice = (text) =>
+ `<div class="obs-notice" role="status">${esc(text)}</div>`;
+function failure(e) {
+ if (e.status === 403) return "This credential cannot read the selected run.";
+ if (e.status === 404) return "This run is no longer available.";
+ if (e.status === 429)
+ return "Recording readers are busy. Try Refresh shortly.";
+ if (e.status === 503)
+ return "Recordings are unavailable or disabled. You can still search runs.";
+ return "Unable to load recordings. Try Refresh.";
+}
+const rowTable = (headers, rows, label) =>
+ `<div class="obs-table-wrap"><table class="table" aria-label="${label}"><thead><tr>${headers.map((h) => `<th>${h}</th>`).join("")}</tr></thead><tbody>${rows.join("") || `<tr><td colspan="${headers.length}" class="obs-empty-cell">No recorded data</td></tr>`}</tbody></table></div>`;
+function bars(rows, label, formatter = duration, type = "phase") {
+ const max = Math.max(
+ 1,
+ ...rows.filter((r) => known(r.value)).map((r) => r.value),
+ );
+ return `<div class="obs-bars" aria-label="${label}">${rows.map((r) => `<div class="obs-bar-row"><span>${esc(r.name)}</span><div class="obs-track"><i class="obs-fill obs-${r.key || type}" data-obs-width="${known(r.value) ? Math.min(100, (r.value / max) * 100) : 0}"></i></div><span class="obs-number">${esc(formatter(r.value))}${r.missing ? "*" : ""}</span></div>`).join("") || hint("No recorded data")}</div>`;
+}
+function tokensByPurpose(s) {
+ const rows = purposeUsage(s.models).map((r) => ({
+ ...r,
+ value: r.total,
+ key: "model",
+ }));
+ return (
+ bars(rows, "Tokens by purpose", count) +
+ (s.coverage.group_truncated ? hint("Group list is truncated.") : "") +
+ (rows.some((r) => r.missing)
+ ? hint("* Some request counters are unknown.")
+ : "") +
+ hint(
+ `Cached input: ${measurement(s.usage.cache_read)} · already included in input.`,
+ )
+ );
+}
+function modelTable(s) {
+ return rowTable(
+ ["Model / purpose", "Calls", "Input", "Output", "First output · avg."],
+ s.models.map(
+ (m) =>
+ `<tr><td>${esc(m.name || "Unspecified")}<span class="cell-subtitle">${esc(purpose(m.purpose))}</span></td>${cell(count(m.calls))}${cell(measurement(m.usage?.input))}${cell(measurement(m.usage?.output))}<td>${esc(duration(average(m.first_output_us)))}${m.first_output_us.unknown ? `<span class="cell-subtitle">${count(m.first_output_us.unknown)} unknown</span>` : ""}</td></tr>`,
+ ),
+ "Model usage",
+ );
+}
+function toolTable(s) {
+ return rowTable(
+ ["Tool", "Calls", "Accumulated", "Avg. known", "Result bytes"],
+ [...s.tools]
+ .sort(
+ (a, b) =>
+ (b.accumulated_us.total ?? -1) - (a.accumulated_us.total ?? -1),
+ )
+ .map(
+ (t) =>
+ `<tr><td><button class="obs-text" data-obs-tool="${esc(t.name || "")}">${esc(t.name || "Unspecified")}</button></td>${cell(count(t.calls))}${cell(measurement(t.accumulated_us, duration))}${cell(duration(average(t.accumulated_us)))}${cell(measurement(t.returned_bytes))}</tr>`,
+ ),
+ "Tool summary",
+ );
+}
+function coverage(s) {
+ const c = s.coverage,
+ notes = coverageNotes(c);
+ return `<details class="obs-details"><summary>Coverage${notes.length ? ` · ${notes.length} ${notes.length === 1 ? "note" : "notes"}` : ""}</summary><p>${count(c.scanned_steps)} steps scanned. Recording is best effort; missing counters are unknown, not zero.</p>${notes.length ? `<ul>${notes.map((n) => `<li>${esc(n)}</li>`).join("")}</ul>` : ""}<p>Late commits and retention can change a refreshed view. A closed run is not proof of lossless accounting.</p></details>`;
+}
+const stepName = (s) =>
+ s.operation === "model.attempt"
+ ? s.model || "Model request"
+ : s.operation === "model.requested"
+ ? "Request started"
+ : s.operation === "tool.execute"
+ ? s.tool || "Tool execution"
+ : {
+ "agent.execute": "Execution segment",
+ "policy.review": "Policy check",
+ "approval.wait": "Approval wait",
+ "question.wait": "Question wait",
+ }[s.operation] || s.operation;
+const stepKind = (s) =>
+ s.operation.startsWith("model.")
+ ? "model"
+ : s.operation.startsWith("tool.")
+ ? "tool"
+ : s.operation === "policy.review"
+ ? "policy"
+ : s.operation === "approval.wait"
+ ? "approval_wait"
+ : s.operation === "question.wait"
+ ? "question_wait"
+ : "other_active";
+
+let viewerSequence = 0;
+export function mountObservability(root, { request, isAdmin = false, initialRun = null, directory = true, openSession = true, children = false }) {
+ const instance = `ow-trace-${++viewerSequence}`;
+ const domId = (id) => `${instance}-${id}`;
+ const ids = (html) => html.replace(/(id|for|aria-controls|aria-labelledby)="(obs[^" ]*)"/g, (_, attr, id) => `${attr}="${domId(id)}"`);
+ const lifetime = new AbortController();
+ let stopped = false,
+ serial = 0,
+ searchSerial = 0,
+ timer,
+ loadController,
+ searchController;
+ let childLinks=[], ancestors=[], subtree=null;
+ let chosen = initialRun,
+ tab = "usage",
+ summary = null,
+ selection = null,
+ timeline = [],
+ timelinePage = null,
+ timelineThrough = null,
+ selectedStep = null,
+ tool = null;
+ let results = [],
+ resultNext = null,
+ searchText = "",
+ focusedIndex = -1,
+ pageCursor = null,
+ pageStack = [];
+ let health = null,
+ healthError = false;
+ const controllers = new Set();
+ function controller() {
+ const c = new AbortController();
+ controllers.add(c);
+ return c;
+ }
+ async function read(path, c) {
+ controllers.add(c);
+ try {
+ return await request(path, c.signal);
+ } finally {
+ controllers.delete(c);
+ }
+ }
+ root.innerHTML = ids(`<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder &amp; exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`);
+ const $ = (s) => root.querySelector(s.replace(/#(obs[\w-]*)/g, (_, id) => `#${domId(id)}`));
+ if (!directory) $(".obs-picker").hidden = true;
+ root.addEventListener("ow-trace-refresh", () => load(), {signal:lifetime.signal});
+ const content = $("#obs-content"),
+ input = $("#obs-run-search"),
+ menu = $("#obs-run-menu");
+ function widths() {
+ for (const el of root.querySelectorAll("[data-obs-width]"))
+ el.style.width = `${el.dataset.obsWidth}%`;
+ for (const el of root.querySelectorAll("[data-obs-left]"))
+ el.style.left = `${el.dataset.obsLeft}%`;
+ }
+ function scope() {
+ $("#obs-scope").innerHTML = chosen
+ ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc([chosen.agent, chosen.status, known(chosen.created_ms) ? new Date(chosen.created_ms).toLocaleString() : null].filter(Boolean).join(" · "))}</span></div>${directory ? `<button class="obs-text" data-obs="recent">Recent runs</button>` : ""}${openSession ? `<a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a>` : ""}</div>`
+ : `<p class="obs-hint obs-scope">${selection ? `${selection.selected_runs} recent authorized runs${selection.selection_truncated ? " · more available" : ""}. Summary covers this page only.` : "Recent authorized runs"}</p>`;
+ if (summary?.coverage.scan_truncated || summary?.coverage.group_truncated) {
+ $("#obs-scope").insertAdjacentHTML(
+ "beforeend",
+ '<span class="obs-partial">Partial summary</span>',
+ );
+ }
+ content.setAttribute("aria-labelledby", domId(`obs-${tab}-tab`));
+ for (const b of root.querySelectorAll(
+ "[data-obs=usage],[data-obs=timeline]",
+ )) {
+ const active = b.dataset.obs === tab;
+ b.classList.toggle("active", active);
+ b.setAttribute("aria-selected", String(active));
+ b.tabIndex = active ? 0 : -1;
+ }
+ }
+ function draw() {
+ if (stopped) return;
+ scope();
+ if (!summary) return;
+ const s = summary,
+ tokenTotal = usageTotal(s.usage),
+ tokensMissing = s.usage.input.unknown + s.usage.output.unknown;
+ content.innerHTML = `<div class="stats obs-stats">${[
+ [
+ chosen ? "Run elapsed" : "Observed window",
+ duration(chosen ? s.wall_elapsed_us : s.observed_window_us),
+ ],
+ [
+ "Reported tokens",
+ count(tokenTotal) +
+ (known(tokenTotal) && tokensMissing ? " + unknown" : ""),
+ ],
+ [
+ "Model requests",
+ count(s.models.reduce((n, g) => n + g.calls, 0)) +
+ (s.coverage.group_truncated ? "+" : ""),
+ ],
+ [
+ "Tool calls",
+ count(s.tools.reduce((n, g) => n + g.calls, 0)) +
+ (s.coverage.group_truncated ? "+" : ""),
+ ],
+ ]
+ .map(
+ ([label, value]) =>
+ `<div class="stat"><div class="stat-label">${label}</div><div class="stat-main"><span class="stat-value">${esc(value)}</span></div></div>`,
+ )
+ .join(
+ "",
+ )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}${chosen&&children?`<section class="obs-panel"><h2>Sub-agents</h2>${ancestors.length?`<button class="button" data-obs="parent">Back to parent turn</button>`:""}${childLinks.map((child,i)=>`<button class="button" data-obs-child="${i}">${esc(child.agent_id)} · ${esc(child.phase)}</button>`).join(" ")||hint("No visible recorded child links for this turn.")}${subtree ? hint(`Linked work: ${count(subtree.selection.runs.length)} visible initial tasks${subtree.selection.selection_truncated ? " · truncated" : ""} · ${measurement(subtree.summary.usage.input)} input / ${measurement(subtree.summary.usage.output)} output tokens. Later inbox turns are separate.`) : ""}${hint("Usage above covers this turn. Open a child to inspect its own requests and usage.")}</section>`:""}`;
+ widths();
+ }
+ function usageView(s) {
+ return `<div class="obs-grid"><section class="obs-panel"><h2>Where time went</h2>${hint("Exclusive phases · observed time")}${bars(
+ phases
+ .map(([key, name]) => ({
+ key,
+ name,
+ value: s.exclusive_observed_us[key],
+ }))
+ .filter((r) => r.value > 0),
+ "Exclusive phase duration",
+ )}${hint(`Observed window: ${duration(s.observed_window_us)}${chosen ? "" : " · union across selected runs, not summed run time"}`)}</section><section class="obs-panel"><h2>Tokens by purpose</h2>${hint("Reported input + output")}${tokensByPurpose(s)}</section></div><div class="obs-grid"><section class="obs-panel"><h2>Accumulated work</h2>${hint("May exceed elapsed time when calls overlap")}${bars(
+ [
+ {
+ name: "Model requests",
+ value: s.model_accumulated_us.total,
+ key: "model",
+ },
+ {
+ name: "Tool executions",
+ value: s.tool_accumulated_us.total,
+ key: "tool",
+ },
+ ],
+ "Accumulated work",
+ )}${s.model_accumulated_us.unknown || s.tool_accumulated_us.unknown ? hint("Some durations are unknown; bars show measured work only.") : ""}</section><section class="obs-panel"><h2>Model response time</h2>${hint("Average request start → first output · known samples")}${bars(
+ s.models.map((m) => ({
+ name: `${m.name || "Unspecified"} · ${purpose(m.purpose)}`,
+ value: average(m.first_output_us),
+ missing: m.first_output_us.unknown,
+ key: "model",
+ })),
+ "Average model first output",
+ )}</section></div><section class="obs-panel"><h2>Tools</h2>${hint("Select a tool to inspect its calls in a run")}${toolTable(s)}</section><section class="obs-panel"><h2>Model usage</h2>${modelTable(s)}</section>`;
+ }
+ function timelineView(s) {
+ if (!chosen) return notice("Choose a run to see its timeline.");
+ const records = tool
+ ? timeline.filter((r) => r.step.tool === tool)
+ : timeline;
+ const layout = timelineLayout(records);
+ const selected = timeline.find(
+ (r) => String(r.cursor) === String(selectedStep),
+ );
+ return `<div class="obs-grid"><section class="obs-panel"><h2>Time by phase</h2>${bars(
+ phases
+ .map(([key, name]) => ({
+ key,
+ name,
+ value: s.exclusive_observed_us[key],
+ }))
+ .filter((r) => r.value > 0),
+ "Run phase duration",
+ )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="${domId("obs-step-detail")}">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="${domId("obs-timeline-error")}" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;
+ }
+ function stepDetail(r) {
+ const s = r.step;
+ return `<div class="obs-step-detail"><div><strong>${esc(stepName(s))}</strong><span>${esc(s.outcome)} · ${esc(s.operation)}</span></div><dl>${[
+ ["Duration", duration(s.duration_us)],
+ ["First output", duration(s.first_output_us)],
+ ["Input tokens", count(s.usage?.input)],
+ ["Output tokens", count(s.usage?.output)],
+ ["Result bytes", count(s.output_bytes)],
+ ["Attempt", count(s.attempt)],
+ ]
+ .map(([k, v]) => `<div><dt>${k}</dt><dd>${v}</dd></div>`)
+ .join(
+ "",
+ )}</dl><details><summary>Record details</summary><pre>${esc(JSON.stringify(r, null, 2))}</pre></details></div>`;
+ }
+ function runDirectory() {
+ if (chosen) return "";
+ const runs = selection?.selection?.runs || [];
+ return `<section class="obs-panel"><h2>Runs</h2>${rowTable(
+ ["Run", "Agent", "Status", "Submitted"],
+ runs.map(
+ (r, i) =>
+ `<tr><td><button class="obs-text obs-run-name" data-obs-run="${i}">${esc(r.name || r.turn_id)}</button><span class="cell-subtitle mono">${esc(r.turn_id)}</span></td>${cell(r.agent)}${cell(r.status)}${cell(new Date(r.created_ms).toLocaleString())}</tr>`,
+ ),
+ "Authorized runs",
+ )}<div class="obs-pagination">${pageStack.length ? '<button class="button" data-obs="previous">Previous page</button>' : ""}${selection?.selection?.next_cursor ? '<button class="button" data-obs="next">Next 25 runs</button>' : ""}</div></section>`;
+ }
+ async function load() {
+ const n = ++serial;
+ loadController?.abort();
+ const c = controller();
+ loadController = c;
+ summary = null;
+ content.innerHTML = notice("Loading recordings…");
+ scope();
+ try {
+ if (chosen) {
+ const page = await read(perRun(chosen) + "/timeline?limit=100", c);
+ if (stopped || n !== serial) return;
+ timeline = page.page.steps;
+ timelinePage = page.page;
+ timelineThrough = page.page.through;
+ selectedStep = null;
+ // One watermark aligns the first timeline page and summary. It is not
+ // a cross-request database snapshot; late commits/retention can differ.
+ const value = await read(
+ perRun(chosen) + "/summary?" + query({ through: timelineThrough }),
+ c,
+ );
+ if (stopped || n !== serial) return;
+ chosen = value.run;
+ summary = value.summary;
+ } else {
+ const value = await read(
+ "/v1/observability/summary?" +
+ query({ limit: 25, after: pageCursor }),
+ c,
+ );
+ if (stopped || n !== serial) return;
+ selection = value;
+ summary = selection.summary;
+ }
+ childLinks=[];subtree=null;
+ if(chosen && children) {
+ try {const links=await read(perRun(chosen)+"/children",c); if(stopped||n!==serial)return;childLinks=links.children||[];} catch {childLinks=[];}
+ if (stopped || n !== serial) return;
+ try {const value=await read(perRun(chosen)+"/subtree-summary?"+query({through:timelineThrough}),c);if(stopped||n!==serial)return;subtree=value;} catch {subtree=null;}
+ }
+ draw();
+ } catch (e) {
+ if (stopped || n !== serial || c.signal.aborted) return;
+ // Old successful metrics are removed before each read; never retain them
+ // as a silent fallback after scope change, logout or recording failure.
+ content.innerHTML = notice(failure(e));
+ scope();
+ }
+ }
+ function closeSearch() {
+ menu.hidden = true;
+ input.setAttribute("aria-expanded", "false");
+ input.removeAttribute("aria-activedescendant");
+ }
+ function drawResults() {
+ $("#obs-run-options").innerHTML = results
+ .map(
+ (r, i) =>
+ `<button type="button" class="obs-run-option" role="option" id="${domId(`obs-option-${i}`)}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,
+ )
+ .join("");
+ $("#obs-search-more").hidden = !resultNext;
+ if (focusedIndex >= 0)
+ input.setAttribute("aria-activedescendant", domId(`obs-option-${focusedIndex}`));
+ else input.removeAttribute("aria-activedescendant");
+ }
+ async function search(more = false) {
+ const n = ++searchSerial;
+ searchController?.abort();
+ const c = controller();
+ searchController = c;
+ menu.hidden = false;
+ input.setAttribute("aria-expanded", "true");
+ $("#obs-search-state").textContent = "Searching…";
+ try {
+ const page = await read(
+ "/v1/observability/runs?" +
+ query({ q: searchText, limit: 10, after: more ? resultNext : null }),
+ c,
+ );
+ if (stopped || n !== searchSerial) return;
+ results = more
+ ? [
+ ...results,
+ ...page.runs.filter(
+ (r) => !results.some((x) => runKey(x) === runKey(r)),
+ ),
+ ]
+ : page.runs;
+ resultNext = page.next_cursor;
+ focusedIndex = -1;
+ drawResults();
+ $("#obs-search-state").textContent = results.length
+ ? `${results.length} matching runs${resultNext ? " · more available" : ""}`
+ : "No matching runs.";
+ } catch (e) {
+ if (stopped || n !== searchSerial || c.signal.aborted) return;
+ results = [];
+ resultNext = null;
+ drawResults();
+ $("#obs-search-state").textContent = "Unable to search runs. Try again.";
+ }
+ }
+ function choose(run, keepParents=false) {
+ if(!keepParents) ancestors=[];
+ if (!run) return;
+ chosen = run;
+ tool = null;
+ tab = "timeline";
+ input.value = "";
+ searchText = "";
+ closeSearch();
+ ++searchSerial;
+ searchController?.abort();
+ clearTimeout(timer);
+ load();
+ }
+ async function moreTimeline(button) {
+ if (!chosen || !timelinePage?.next || button.disabled) return;
+ button.disabled = true;
+ const n = serial,
+ c = controller();
+ try {
+ const value = await read(
+ perRun(chosen) +
+ "/timeline?" +
+ query({
+ after: timelinePage.next,
+ through: timelineThrough,
+ limit: Math.min(100, 1000 - timeline.length),
+ }),
+ c,
+ );
+ if (stopped || n !== serial) return;
+ const cursors = new Set(timeline.map((r) => r.cursor));
+ timeline.push(...value.page.steps.filter((r) => !cursors.has(r.cursor)));
+ timelinePage = value.page;
+ draw();
+ } catch (e) {
+ if (!stopped && n === serial && !c.signal.aborted) {
+ $("#obs-timeline-error").textContent = failure(e);
+ button.disabled = false;
+ }
+ }
+ }
+ async function loadHealth() {
+ if (health || healthError || stopped) return;
+ const el = $("#obs-health-content");
+ el.textContent = "Loading…";
+ const c = controller();
+ try {
+ health = await read("/v1/observability/health", c);
+ if (stopped) return;
+ el.innerHTML =
+ hint(
+ "Serving API instance only. Worker and other API counters are not aggregated here.",
+ ) +
+ `<dl class="obs-health-values"><div><dt>Recorder</dt><dd>${health.recording_enabled ? (health.recording ? "Enabled" : "Not reporting") : "Disabled"}</dd></div><div><dt>Exporter</dt><dd>${health.exporter_enabled ? "Enabled" : "Disabled"}</dd></div><div><dt>Recording drops</dt><dd>${count(health.recording?.dropped)}</dd></div><div><dt>Write failures</dt><dd>${count(health.recording?.failed_writes)}</dd></div><div><dt>Export failures</dt><dd>${count(health.exporter?.export_failures)}</dd></div></dl><details><summary>Process counters</summary><pre>${esc(JSON.stringify(health, null, 2))}</pre></details>`;
+ } catch (e) {
+ if (!stopped && !c.signal.aborted) {
+ healthError = true;
+ el.textContent =
+ e.status === 403
+ ? "Administrator access required."
+ : "Process health unavailable. Refresh to try again.";
+ }
+ }
+ }
+ root.addEventListener(
+ "click",
+ (e) => {
+ const option = e.target.closest("[data-obs-option]");
+ if (option) {
+ choose(results[Number(option.dataset.obsOption)]);
+ return;
+ }
+ const run = e.target.closest("[data-obs-run]");
+ if (run) {
+ choose(selection?.selection.runs[Number(run.dataset.obsRun)]);
+ return;
+ }
+ const step = e.target.closest("[data-obs-step]");
+ if (step) {
+ selectedStep = step.dataset.obsStep;
+ draw();
+ root.querySelector(`[data-obs-step="${selectedStep}"]`)?.focus();
+ return;
+ }
+ const t = e.target.closest("[data-obs-tool]");
+ if (t) {
+ if (!chosen) {
+ content.prepend(
+ notice("Choose a run to inspect individual tool calls."),
+ );
+ return;
+ }
+ tool = t.dataset.obsTool;
+ tab = "timeline";
+ draw();
+ return;
+ }
+ const childButton=e.target.closest("[data-obs-child]");
+ if(childButton){
+ const child=childLinks[Number(childButton.dataset.obsChild)];if(!child)return;
+ const selectedSerial=serial;const selectedParent=chosen;
+ if(child.child_turn_id){ancestors.push(selectedParent);choose({session_id:child.child_session_id,turn_id:child.child_turn_id},true);return;}
+ childButton.disabled=true;
+ const c=controller();
+ read("/v1/observability/runs?"+query({q:child.child_session_id,limit:100}),c).then(page=>{
+ if(stopped||serial!==selectedSerial)return;
+ const run=child.child_turn_id ? {session_id:child.child_session_id,turn_id:child.child_turn_id} : page.runs.find(r=>r.session_id===child.child_session_id);
+ if(run){ancestors.push(selectedParent);choose(run,true);}else{content.prepend(notice("No accessible child turn is available yet."));}
+ }).catch(()=>{if(!stopped)content.prepend(notice("Child traces are unavailable."));}).finally(()=>{childButton.disabled=false;});
+ return;
+ }
+ const action = e.target.closest("[data-obs]")?.dataset.obs;
+ if(action==="parent" && ancestors.length){choose(ancestors.pop(),true);return;}
+ if (action === "refresh") {
+ health = null;
+ healthError = false;
+ if ($("#obs-health")?.open) loadHealth();
+ load();
+ }
+ if (action === "recent" && directory) {
+ chosen = null;
+ tool = null;
+ tab = "usage";
+ pageCursor = null;
+ pageStack = [];
+ load();
+ }
+ if (action === "usage" || action === "timeline") {
+ tab = action;
+ draw();
+ }
+ if (action === "next") {
+ pageStack.push(pageCursor);
+ pageCursor = selection.selection.next_cursor;
+ load();
+ }
+ if (action === "previous") {
+ pageCursor = pageStack.pop() ?? null;
+ load();
+ }
+ if (action === "more-timeline") moreTimeline(e.target.closest("button"));
+ if (action === "clear-tool") {
+ tool = null;
+ draw();
+ }
+ },
+ { signal: lifetime.signal },
+ );
+ root.addEventListener("keydown", e => {
+ if (e.target.getAttribute?.("role") !== "tab" || !["ArrowLeft", "ArrowRight", "Home", "End"].includes(e.key)) return;
+ const tabs = [...root.querySelectorAll('[role="tab"]')];
+ let index = tabs.indexOf(e.target);
+ if (e.key === "Home") index = 0;
+ else if (e.key === "End") index = tabs.length - 1;
+ else index = (index + (e.key === "ArrowRight" ? 1 : -1) + tabs.length) % tabs.length;
+ e.preventDefault(); tab = tabs[index].dataset.obs; draw(); tabs[index].focus();
+ }, {signal:lifetime.signal});
+ input.addEventListener(
+ "input",
+ () => {
+ searchText = input.value;
+ clearTimeout(timer);
+ ++searchSerial;
+ searchController?.abort();
+ results = [];
+ resultNext = null;
+ focusedIndex = -1;
+ drawResults();
+ menu.hidden = false;
+ input.setAttribute("aria-expanded", "true");
+ $("#obs-search-state").textContent = "Searching…";
+ timer = setTimeout(() => search(), 250);
+ },
+ { signal: lifetime.signal },
+ );
+ input.addEventListener(
+ "focus",
+ () => {
+ if (menu.hidden) {
+ searchText = input.value;
+ search();
+ }
+ },
+ { signal: lifetime.signal },
+ );
+ input.addEventListener(
+ "keydown",
+ (e) => {
+ if (e.key === "Escape") {
+ ++searchSerial;
+ searchController?.abort();
+ clearTimeout(timer);
+ closeSearch();
+ e.preventDefault();
+ }
+ if (["ArrowDown", "ArrowUp"].includes(e.key)) {
+ e.preventDefault();
+ if (menu.hidden) {
+ search();
+ return;
+ }
+ focusedIndex = Math.max(
+ 0,
+ Math.min(
+ results.length - 1,
+ focusedIndex + (e.key === "ArrowDown" ? 1 : -1),
+ ),
+ );
+ drawResults();
+ $("#obs-run-options")
+ .querySelector('[aria-selected="true"]')
+ ?.scrollIntoView({ block: "nearest" });
+ }
+ if (e.key === "Enter" && focusedIndex >= 0) {
+ e.preventDefault();
+ choose(results[focusedIndex]);
+ }
+ },
+ { signal: lifetime.signal },
+ );
+ $("#obs-search-more").addEventListener(
+ "click",
+ () => {
+ if (results.length >= 100) {
+ $("#obs-search-state").textContent =
+ "Narrow your search to see more matches.";
+ return;
+ }
+ search(true);
+ },
+ { signal: lifetime.signal },
+ );
+ document.addEventListener(
+ "click",
+ (e) => {
+ if (!root.querySelector(".obs-picker").contains(e.target)) {
+ closeSearch();
+ }
+ },
+ { signal: lifetime.signal },
+ );
+ document.addEventListener(
+ "focusin",
+ (e) => {
+ if (!root.querySelector(".obs-picker").contains(e.target)) closeSearch();
+ },
+ { signal: lifetime.signal },
+ );
+ $("#obs-health")?.addEventListener(
+ "toggle",
+ () => {
+ if ($("#obs-health").open) loadHealth();
+ },
+ { signal: lifetime.signal },
+ );
+ load();
+ return () => {
+ stopped = true;
+ serial++;
+ searchSerial++;
+ clearTimeout(timer);
+ lifetime.abort();
+ for (const c of controllers) c.abort();
+ controllers.clear();
+ root.replaceChildren();
+ };
+}
diff --git a/harness/trace-viewer/src/style.css b/harness/trace-viewer/src/style.css
new file mode 100644
index 00000000..6f91452b
--- /dev/null
+++ b/harness/trace-viewer/src/style.css
@@ -0,0 +1,508 @@
+/* Optional viewer styles; no global reset. */
+.ow-trace-viewer { color:#24272c;background:#fff;font:15px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;max-width:1120px;margin:auto;padding:28px;--line:#e5e7eb;--border:#e5e7eb;--muted:#667085;--surface:#f6f7f9;--text:#24272c;--accent:#405f99; }
+.ow-trace-viewer * {box-sizing:border-box}
+.ow-trace-viewer [hidden] {display:none!important}
+.ow-trace-viewer button,.ow-trace-viewer input {font:inherit}
+.ow-trace-viewer button {cursor:pointer;color:inherit}
+.ow-trace-viewer button:focus-visible,.ow-trace-viewer input:focus-visible {outline:2px solid #405f99;outline-offset:3px}
+.ow-trace-viewer .page-heading {display:flex;align-items:center;justify-content:space-between;margin-bottom:24px}
+.ow-trace-viewer h1 {font-size:27px;letter-spacing:-.7px;margin:0}
+.ow-trace-viewer h2 {font-size:17px;margin:0 0 16px}
+.ow-trace-viewer p {color:#667085}
+.ow-trace-viewer .button {background:white;border:1px solid #d9dde4;border-radius:8px;padding:8px 14px}
+.ow-trace-viewer .tabs {display:flex;gap:4px;background:#f1f2f4;border-radius:9px;padding:4px}
+.ow-trace-viewer .tab {border:0;background:transparent;border-radius:6px;padding:7px 15px}
+.ow-trace-viewer .tab.active {background:white;box-shadow:0 1px 4px #0001}
+.ow-trace-viewer .table {width:100%;border-collapse:collapse;text-align:left}
+.ow-trace-viewer th,.ow-trace-viewer td {padding:12px;border-bottom:1px solid #eee;font-size:13px}
+.ow-trace-viewer th {font-weight:500;color:#667085}
+.ow-trace-viewer .cell-subtitle {display:block;color:#667085;font-size:12px}
+.ow-trace-viewer pre {white-space:pre-wrap;overflow-wrap:anywhere;font-size:12px;padding:16px;background:#f6f7f9;border-radius:8px}
+/* Observability: shared typography, neutral surfaces and restrained phase color. */
+.observability {
+ --obs-model: #687fba;
+ --obs-tool: #6c958d;
+ --obs-policy: #9b8aae;
+ --obs-wait: #b39763;
+}
+.obs-toolbar {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 24px;
+ border-bottom: 1px solid var(--line);
+ margin-bottom: 20px;
+}
+.obs-tabs {
+ margin: 0;
+ border: 0;
+ align-self: stretch;
+}
+.obs-tabs .tab {
+ padding: 16px 2px;
+ margin-right: 22px;
+}
+.obs-picker {
+ position: relative;
+ display: flex;
+ align-items: center;
+ gap: 12px;
+ padding: 10px 0;
+ min-width: 310px;
+}
+.obs-picker label {
+ color: var(--muted);
+ font-size: 12px;
+ white-space: nowrap;
+}
+.obs-picker input {
+ width: 250px;
+ border: 1px solid var(--line);
+ border-radius: 7px;
+ padding: 9px 11px;
+ background: var(--canvas);
+ color: var(--ink);
+}
+.obs-run-menu {
+ position: absolute;
+ top: calc(100% - 2px);
+ right: 0;
+ width: 370px;
+ max-width: 90vw;
+ background: white;
+ border: 1px solid var(--line);
+ border-radius: 9px;
+ z-index: 10;
+ box-shadow: 0 10px 35px #20232815;
+ padding: 6px;
+}
+#obs-run-options {
+ max-height: 310px;
+ overflow-y: auto;
+}
+.obs-run-option {
+ display: block;
+ text-align: left;
+ width: 100%;
+ padding: 10px 11px;
+ border-radius: 5px;
+}
+.obs-run-option:hover,
+.obs-run-option[aria-selected="true"] {
+ background: var(--wash);
+}
+.obs-run-option strong {
+ display: block;
+ font-weight: 500;
+ overflow-wrap: anywhere;
+}
+.obs-run-option span,
+.obs-run-option small {
+ display: block;
+ color: var(--muted);
+ margin-top: 4px;
+ font-size: 11px;
+ overflow-wrap: anywhere;
+}
+.obs-run-menu p {
+ color: var(--muted);
+ padding: 8px 11px;
+ font-size: 11px;
+}
+.obs-more {
+ width: 100%;
+ border-top: 1px solid var(--line);
+ text-align: left;
+ padding: 10px 11px;
+}
+.obs-scope {
+ display: flex;
+ align-items: center;
+ gap: 18px;
+ margin: 0 0 22px;
+ min-height: 26px;
+}
+.obs-scope > div {
+ flex: 1;
+ min-width: 0;
+}
+.obs-scope strong {
+ display: block;
+ font-weight: 550;
+ font-size: 15px;
+ overflow-wrap: anywhere;
+}
+.obs-scope span {
+ display: block;
+ margin-top: 5px;
+ color: var(--muted);
+ font-size: 12px;
+}
+.obs-stats {
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ margin-bottom: 12px;
+}
+.obs-stats .stat-value {
+ font-size: 26px;
+ font-weight: 500;
+ letter-spacing: -0.8px;
+ overflow-wrap: anywhere;
+}
+.obs-stats .stat {
+ min-width: 0;
+}
+.obs-grid {
+ display: grid;
+ grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
+ gap: 36px;
+}
+.obs-panel {
+ padding: 26px 0;
+ border-bottom: 1px solid var(--line);
+ min-width: 0;
+}
+.obs-panel h2 {
+ margin-bottom: 8px;
+}
+.obs-hint {
+ color: var(--muted);
+ font-size: 12px;
+ line-height: 1.6;
+ margin-top: 7px;
+}
+.obs-bars {
+ margin: 20px 0 12px;
+}
+.obs-bar-row {
+ display: grid;
+ grid-template-columns: minmax(95px, 1fr) minmax(60px, 1.4fr) minmax(
+ 56px,
+ auto
+ );
+ align-items: center;
+ gap: 14px;
+ margin-top: 14px;
+ font-size: 12px;
+}
+.obs-bar-row > span:first-child {
+ overflow-wrap: anywhere;
+}
+.obs-track {
+ height: 12px;
+ background: var(--wash);
+ border-radius: 3px;
+ overflow: hidden;
+}
+.obs-fill {
+ display: block;
+ height: 100%;
+ border-radius: 3px;
+ background: var(--slate);
+}
+.obs-model {
+ background: var(--obs-model);
+}
+.obs-tool {
+ background: var(--obs-tool);
+}
+.obs-policy {
+ background: var(--obs-policy);
+}
+.obs-approval_wait,
+.obs-question_wait {
+ background: var(--obs-wait);
+}
+.obs-other_active {
+ background: #9aa0ab;
+}
+.obs-parallel {
+ background: repeating-linear-gradient(
+ 135deg,
+ #868b97 0px,
+ #868b97 3px,
+ #b2b5bd 3px,
+ #b2b5bd 6px
+ );
+}
+.obs-unobserved_gap {
+ background: #dadde3;
+}
+.obs-number {
+ font-variant-numeric: tabular-nums;
+ text-align: right;
+ white-space: nowrap;
+}
+.obs-table-wrap {
+ overflow-x: auto;
+}
+.obs-table-wrap .table {
+ margin-top: 15px;
+ min-width: 480px;
+}
+.obs-table-wrap .table td {
+ vertical-align: top;
+ font-variant-numeric: tabular-nums;
+}
+.obs-table-wrap .table th {
+ white-space: nowrap;
+}
+.obs-table-wrap .table td:first-child {
+ max-width: 320px;
+ overflow-wrap: anywhere;
+}
+.obs-text {
+ color: var(--slate);
+ font-weight: 450;
+ text-align: left;
+ padding: 0;
+}
+.obs-text:hover {
+ color: var(--ink);
+ text-decoration: underline;
+}
+.obs-run-name {
+ color: var(--ink);
+ font-weight: 500;
+}
+.obs-pagination {
+ display: flex;
+ justify-content: flex-end;
+ gap: 10px;
+ margin-top: 16px;
+}
+.obs-notice {
+ padding: 18px 0;
+ color: var(--muted);
+ line-height: 1.6;
+}
+.obs-empty-cell {
+ color: var(--muted);
+ padding: 20px 0 !important;
+}
+.obs-panel-title {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 16px;
+}
+.obs-axis {
+ display: flex;
+ justify-content: space-between;
+ margin: 20px 85px 4px 190px;
+ color: var(--muted);
+ font-size: 11px;
+}
+.obs-lanes {
+ max-height: 560px;
+ overflow: auto;
+ padding-right: 4px;
+ margin: 8px 0 16px;
+}
+.obs-lane {
+ display: grid;
+ grid-template-columns: 174px minmax(100px, 1fr) 70px;
+ gap: 16px;
+ width: 100%;
+ padding: 8px 0;
+ text-align: left;
+ align-items: center;
+ border-radius: 4px;
+}
+.obs-lane:hover,
+.obs-lane[aria-pressed="true"] {
+ background: var(--wash);
+}
+.obs-lane-name {
+ font-size: 12px;
+ padding-left: 6px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+.obs-lane-name small {
+ display: block;
+ color: var(--muted);
+ font-size: 10px;
+ margin-top: 4px;
+}
+.obs-lane-track {
+ display: block;
+ height: 26px;
+ position: relative;
+ background: repeating-linear-gradient(
+ to right,
+ var(--line) 0,
+ var(--line) 1px,
+ transparent 1px,
+ transparent 25%
+ );
+ border-bottom: 1px solid var(--line);
+}
+.obs-mark {
+ position: absolute;
+ height: 14px;
+ top: 5px;
+ border-radius: 3px;
+ min-width: 3px;
+ max-width: 100%;
+ transform: translateX(-1px);
+}
+.obs-lane[aria-pressed="true"] .obs-mark {
+ outline: 1px solid var(--ink);
+ outline-offset: 2px;
+}
+.obs-lane .obs-number {
+ font-size: 11px;
+ padding-right: 4px;
+}
+.obs-step-detail {
+ background: var(--wash);
+ padding: 16px 18px;
+ border-radius: 7px;
+ margin: 14px 0;
+}
+.obs-step-detail strong {
+ font-weight: 550;
+}
+.obs-step-detail > div > span {
+ display: block;
+ color: var(--muted);
+ font-size: 12px;
+ margin-top: 5px;
+}
+.obs-step-detail dl {
+ display: grid;
+ grid-template-columns: repeat(3, minmax(0, 1fr));
+ gap: 16px;
+ margin: 20px 0;
+}
+.obs-step-detail dt,
+.obs-health-values dt {
+ color: var(--muted);
+ font-size: 11px;
+ margin-bottom: 5px;
+}
+.obs-step-detail dd,
+.obs-health-values dd {
+ margin: 0;
+ font-variant-numeric: tabular-nums;
+}
+.obs-details {
+ border-bottom: 1px solid var(--line);
+ padding: 17px 0;
+ font-size: 12px;
+}
+.obs-details summary,
+.obs-step-detail summary {
+ cursor: pointer;
+ color: var(--slate);
+}
+.obs-details p,
+.obs-details ul {
+ max-width: 760px;
+ margin: 12px 0 0;
+ color: var(--muted);
+ line-height: 1.7;
+}
+.obs-details ul {
+ padding-left: 18px;
+}
+.obs-details pre,
+.obs-step-detail pre {
+ max-height: 300px;
+ overflow: auto;
+ font-size: 11px;
+ line-height: 1.6;
+ white-space: pre-wrap;
+ overflow-wrap: anywhere;
+ padding: 14px;
+ background: var(--wash);
+ border-radius: 6px;
+}
+.obs-health-values {
+ display: flex;
+ gap: 50px;
+ margin: 16px 0;
+}
+@media (max-width: 1100px) {
+ .obs-bar-row {
+ grid-template-columns: minmax(90px, 1fr) minmax(35px, 1fr) auto;
+ gap: 9px;
+ }
+ .obs-grid {
+ gap: 24px;
+ }
+}
+@media (max-width: 800px) {
+ .obs-grid {
+ grid-template-columns: minmax(0, 1fr);
+ gap: 0;
+ }
+ .obs-toolbar {
+ flex-wrap: wrap;
+ gap: 0;
+ }
+ .obs-picker {
+ min-width: 0;
+ }
+ .obs-stats .stat-value {
+ font-size: 22px;
+ }
+}
+@media (max-width: 560px) {
+ .obs-stats {
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ }
+ .obs-picker {
+ width: 100%;
+ padding-top: 0;
+ }
+ .obs-picker input {
+ flex: 1;
+ min-width: 0;
+ width: auto;
+ }
+ .obs-run-menu {
+ width: 100%;
+ }
+ .obs-scope {
+ flex-wrap: wrap;
+ gap: 12px;
+ }
+ .obs-scope > div {
+ flex-basis: 100%;
+ }
+ .obs-lane {
+ grid-template-columns: 108px minmax(30px, 1fr) 58px;
+ gap: 8px;
+ }
+ .obs-axis {
+ margin-left: 116px;
+ margin-right: 66px;
+ }
+ .obs-step-detail dl {
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ }
+}
+.obs-partial {
+ display: inline-block;
+ border: 1px solid var(--line);
+ border-radius: 4px;
+ padding: 3px 7px;
+ font-size: 11px;
+ color: var(--muted);
+ margin-bottom: 12px;
+}
+.obs-health-values {
+ flex-wrap: wrap;
+ gap: 20px 40px;
+}
+.ow-trace-viewer .stats {display:grid;grid-template-columns:repeat(4,minmax(0,1fr));border:1px solid var(--line);border-radius:10px;margin:0 0 24px}
+.ow-trace-viewer .stat {padding:20px 24px}
+.ow-trace-viewer .stat+.stat {border-left:1px solid var(--line)}
+.ow-trace-viewer .stat-label {font-size:12px;color:#737b88}
+.ow-trace-viewer .stat-main {margin-top:10px}
+.ow-trace-viewer .stat-value {font-size:29px;letter-spacing:-1px;font-weight:550;font-variant-numeric:tabular-nums}
+.ow-trace-viewer .obs-lane {appearance:none;border:0;border-bottom:1px solid #eef0f3;background:white;text-align:left;border-radius:4px}
+.ow-trace-viewer .obs-lane:hover {background:#f7f8fa}
+.ow-trace-viewer .obs-lane[aria-pressed=true] {background:#f0f3f8;box-shadow:inset 3px 0 #687fba}
diff --git a/harness/trace-viewer/sync.mjs b/harness/trace-viewer/sync.mjs
new file mode 100644
index 00000000..c2054ea9
--- /dev/null
+++ b/harness/trace-viewer/sync.mjs
@@ -0,0 +1,9 @@
+// One authoritative viewer implementation, copied into the standalone UI image.
+import {readFile,writeFile} from 'node:fs/promises';
+import assert from 'node:assert/strict';
+for(const file of ['observability.js','observability-data.js']) {
+ const source=await readFile(new URL(`src/${file}`,import.meta.url),'utf8');
+ const target=new URL(`../service-ui/public/${file}`,import.meta.url);
+ if(process.argv.includes('--check'))assert.equal(await readFile(target,'utf8'),source,`${file}: run npm run sync in trace-viewer`);
+ else await writeFile(target,source);
+}
diff --git a/harness/trace-viewer/test/browser.mjs b/harness/trace-viewer/test/browser.mjs
new file mode 100644
index 00000000..735bfa71
--- /dev/null
+++ b/harness/trace-viewer/test/browser.mjs
@@ -0,0 +1,80 @@
+// Run against the loopback preview with a caller-supplied Playwright installation.
+// No dependency or browser download is added to the SDK/runtime package.
+import assert from 'node:assert/strict';
+import {mkdir} from 'node:fs/promises';
+import {resolve} from 'node:path';
+import {pathToFileURL} from 'node:url';
+const [url,modulePath,output]=process.argv.slice(2);
+assert(url&&modulePath&&output,'Usage: browser.mjs LOOPBACK_URL PLAYWRIGHT_MODULE OUTPUT_DIRECTORY');
+assert(['127.0.0.1','localhost'].includes(new URL(url).hostname));
+const {chromium}=await import(pathToFileURL(resolve(modulePath)).href);
+const browser=await chromium.launch({headless:true});
+try {
+ const page=await browser.newPage({viewport:{width:1440,height:1100}});
+ const errors=[];page.on('pageerror',e=>errors.push(e.message));
+ await page.goto(url);await page.getByText('Reported tokens',{exact:true}).waitFor();
+ await mkdir(output,{recursive:true});
+ await page.screenshot({path:resolve(output,'sdk-trace-usage.png'),fullPage:true});
+ await page.getByRole('tab',{name:'Timeline',exact:true}).click();
+ await page.getByRole('heading',{name:'Turn timeline'}).waitFor();
+ await page.locator('[data-obs-step]').first().click();
+ await page.getByText('Record details',{exact:true}).waitFor();
+ await page.screenshot({path:resolve(output,'sdk-trace-timeline.png'),fullPage:true});
+ // Multiple independently mounted instances, hostile text, subtree/eval panels.
+ await page.evaluate(async()=>{
+ const {mountTurnTrace}=await import('/src/index.js');
+ const evidence=await fetch('/trace.json').then(r=>r.json());
+ const adapter={timeline:async()=>evidence.timeline,summary:async()=>evidence.summary,
+ children:async()=>({children:[{child_session_id:'child',child_turn_id:'initial',agent_id:'<img src=x onerror=alert(1)>',phase:'ready'}]}),
+ subtreeSummary:async()=>({selection:{runs:[{},{}],selection_truncated:false},summary:evidence.summary.summary}),
+ evaluations:async()=>({results:[{evaluation:{result:{name:'correctness',status:'completed',score:{value:true},evaluator:{name:'fixture',version:'1',source:'code'},explanation:'<script>alert(1)</script>'}}}],next:null})};
+ window.testViews=[];
+ for(let i=0;i<2;i++){const root=document.createElement('main');root.id=`test-${i}`;document.body.append(root);window.testViews.push(mountTurnTrace(root,{adapter,sessionId:evidence.timeline.run.session_id,turnId:evidence.timeline.run.turn_id,onChild:child=>{window.selectedChild=child;}}));}
+ });
+ await page.locator('#test-0').getByRole('heading',{name:'Linked work',exact:true}).waitFor();
+ const ids=await page.locator('[id]').evaluateAll(nodes=>nodes.map(n=>n.id));assert.equal(new Set(ids).size,ids.length,'Mounted viewers must have unique DOM IDs');
+ assert.equal(await page.locator('#test-0 img, #test-0 script').count(),0,'Content must be escaped');
+ await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).focus();await page.keyboard.press('ArrowRight');
+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Timeline',exact:true}).getAttribute('aria-selected'),'true');
+ await page.keyboard.press('Home');
+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).getAttribute('aria-selected'),'true');
+ const child=page.locator('#test-0').getByRole('button',{name:/img src/});await child.focus();await page.keyboard.press('Enter');
+ assert.equal(await page.evaluate(()=>window.selectedChild.child_turn_id),'initial');
+ await page.locator('#test-1').getByRole('tab',{name:'Timeline',exact:true}).click();
+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).getAttribute('aria-selected'),'true');
+ // The same component's service adapter follows an exact child turn without listing all runs.
+ await page.evaluate(async()=>{
+ const {mountObservability}=await import('/src/index.js');const data=await fetch('/trace.json').then(r=>r.json());
+ const root=document.createElement('main');root.id='management';document.body.append(root);window.managementRequests=[];
+ window.managementDispose=mountObservability(root,{initialRun:data.timeline.run,directory:false,openSession:false,children:true,request:async path=>{
+ window.managementRequests.push(path);const u=new URL(path,location.origin),parts=u.pathname.split('/');const run={session_id:parts[3],turn_id:parts[5]};
+ if(parts[6]==='children')return {children:run.session_id==='child'?[]:[{child_session_id:'child',child_turn_id:'initial',agent_id:'builder',phase:'ready'}]};
+ if(parts[6]==='subtree-summary')return {selection:{runs:[{},{}],selection_truncated:false},summary:data.summary.summary};
+ if(parts[6]==='timeline')return {...data.timeline,run};if(parts[6]==='summary')return {...data.summary,run};throw Error('Unexpected service request');
+ }});
+ });
+ await page.locator('#management [data-obs-child]').waitFor();await page.locator('#management [data-obs-child]').click();
+ await page.waitForFunction(()=>window.managementRequests.some(p=>p.includes('/sessions/child/observability/initial/summary')));
+ assert(!(await page.evaluate(()=>window.managementRequests)).some(p=>p.includes('/observability/runs')));
+ await page.locator('#management [data-obs="parent"]').click();
+ await page.evaluate(()=>window.managementDispose());
+ // A large trace stays paged and stops at the documented browser bound.
+ await page.evaluate(async()=>{
+ const {mountTurnTrace}=await import('/src/index.js');const data=await fetch('/trace.json').then(r=>r.json());
+ const root=document.createElement('main');root.id='large';document.body.append(root);
+ const base=data.timeline.page.steps[0].step;
+ const rows=Array.from({length:1001},(_,i)=>({cursor:i+1,step:{...base,span_id:(i+1).toString(16).padStart(16,'0')}}));
+ window.largeRequests=[];
+ window.largeView=mountTurnTrace(root,{sessionId:data.timeline.run.session_id,turnId:data.timeline.run.turn_id,adapter:{summary:async()=>data.summary,timeline:async q=>{window.largeRequests.push(q);const start=q.after??0;return {...data.timeline,page:{steps:rows.slice(start,start+100),through:1001,next:start+100<1001?start+100:null}};}}});
+ });
+ await page.locator('#large').getByText('Reported tokens',{exact:true}).waitFor();
+ await page.locator('#large').getByRole('tab',{name:'Timeline',exact:true}).click();
+ for(let i=0;i<9;i++) {await page.locator('#large').getByRole('button',{name:'Load more records',exact:true}).click();await page.waitForFunction(n=>document.querySelectorAll('#large [data-obs-step]').length===n,(i+2)*100);}
+ assert.equal(await page.locator('#large [data-obs-step]').count(),1000);
+ assert.equal(await page.locator('#large').getByRole('button',{name:'Load more records',exact:true}).count(),0);
+ assert.equal(await page.evaluate(()=>window.largeRequests.length),10);
+ await page.evaluate(()=>window.largeView.dispose());
+ await page.evaluate(()=>window.testViews.forEach(v=>v.dispose()));
+ assert.equal(await page.locator('#test-0').textContent(),'');assert.deepEqual(errors,[]);
+ console.log('Viewer passed: usage, timeline/detail, unique IDs, independent instances, child keyboard navigation, escaped evaluations, disposal.');
+}finally{await browser.close();}
diff --git a/harness/typescript-sdk/README.md b/harness/typescript-sdk/README.md
index 3b06305c..95df5efd 100644
--- a/harness/typescript-sdk/README.md
+++ b/harness/typescript-sdk/README.md
@@ -518,3 +518,14 @@ tool subset. The legacy `subagents: Agent[]` form remains unchanged. This requir
a matching feature runtime; it is not in published runtime 0.1.0-beta.2.
See [standalone sub-agents](../../docs/harness/guides/sessions.md#standalone-sub-agents)
for limits, approvals and shared-sandbox behavior.
+
+
+## Optional turn traces and evaluation evidence
+
+The source candidate includes opt-in recording, per-turn timeline/grouped queries,
+child drill-down and durable evaluation results. Recording and evaluation delivery
+are disabled until configured; browser UI is a separate optional component.
+See the [observability guide](../../docs/harness/guides/observability.md) for local
+SQLite/PostgreSQL, Langfuse export, content selection and audit references. These
+new APIs require matching local candidate builds and are not in the published
+0.1.0-beta.2 packages yet.
diff --git a/harness/typescript-sdk/package.json b/harness/typescript-sdk/package.json
index 13cde785..0e7d61d8 100644
--- a/harness/typescript-sdk/package.json
+++ b/harness/typescript-sdk/package.json
@@ -11,6 +11,10 @@
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
+ },
+ "./evaluation-export": {
+ "types": "./dist/evaluation-export.d.ts",
+ "import": "./dist/evaluation-export.js"
}
},
"files": [
@@ -37,7 +41,9 @@
"test:provider-auth": "npm run build && node --test test/provider-auth.test.mjs",
"prepack": "npm run build",
"test:service": "npm run build && node --test test/service.test.mjs",
- "test:team-types": "npm run build && node --test --test-concurrency=1 test/team-types.test.mjs"
+ "test:team-types": "npm run build && node --test --test-concurrency=1 test/team-types.test.mjs",
+ "test:observability": "npm run build && node --test test/observability.test.mjs test/evaluation-export.test.mjs",
+ "test:observability:integration": "npm run build && node --test test/observability-live.test.mjs"
},
"devDependencies": {
"@types/node": "^22.0.0",
diff --git a/harness/typescript-sdk/src/config.ts b/harness/typescript-sdk/src/config.ts
index c632d6f1..b62afdff 100644
--- a/harness/typescript-sdk/src/config.ts
+++ b/harness/typescript-sdk/src/config.ts
@@ -139,6 +139,19 @@ export function hostConfig(options: ResolvedHarnessOptions): object {
: (options.sandbox ?? "openshell"),
scripted: options.scripted ?? false,
telemetry: options.telemetry ?? null,
+ ...(options.recording ? { recording: {
+ evaluations: options.recording.evaluations ?? false,
+ max_pending_evaluations:options.recording.maxPendingEvaluations??10000,
+ evaluation_retention_days:options.recording.evaluationRetentionDays??30,
+ evaluation_sinks:(options.recording.evaluationSinks??[]).map(s=>({name:s.name,url:s.url,authorization_env:s.authorizationEnv,include_explanation:s.includeExplanation??false})),
+ evidence: options.recording.evidence ? {user_input:options.recording.evidence.userInput??false,assistant_output:options.recording.evidence.assistantOutput??false,tool_arguments:options.recording.evidence.toolArguments??false,tool_results:options.recording.evidence.toolResults??false,max_bytes:options.recording.evidence.maxBytes}:null,
+ database: options.recording.database.type === "sqlite"
+ ? {kind:"sqlite",path:resolve(options.recording.database.path)}
+ : {kind:"postgres",connection_env:options.recording.database.connectionEnv,schema:options.recording.database.schema,ca_file:options.recording.database.caFile ?? null,loopback_plaintext:options.recording.database.loopbackPlaintext ?? false},
+ deployment:options.recording.deployment ?? "sdk",
+ retention_days:options.recording.retentionDays ?? 30,
+ queue_capacity:options.recording.queueCapacity ?? 1024,
+ }} : {}),
};
}
export function agentWire(a: Agent, depth = 0): object {
@@ -186,6 +199,8 @@ export function environment(options: HarnessOptions): NodeJS.ProcessEnv {
const explicit = [...(options.inheritEnv ?? []), ...(options.complianceSinks ?? []).map(s=>s.credentialEnv)];
if (options.storage?.type === "postgres")
explicit.push(options.storage.connectionEnv);
+ if(options.recording?.database.type === "postgres") explicit.push(options.recording.database.connectionEnv);
+ for (const sink of options.recording?.evaluationSinks ?? []) explicit.push(sink.authorizationEnv);
for (const header of options.telemetry?.headers ?? [])
if ("value_env" in header) explicit.push(header.value_env);
for (const key of explicit) {
diff --git a/harness/typescript-sdk/src/evaluation-export.ts b/harness/typescript-sdk/src/evaluation-export.ts
new file mode 100644
index 00000000..a3b5b13c
--- /dev/null
+++ b/harness/typescript-sdk/src/evaluation-export.ts
@@ -0,0 +1,34 @@
+/** Optional explicit export helpers. No background worker or network at import. */
+import {createHash} from 'node:crypto';
+import type {Evaluation} from './observability.js';
+export interface ScoreExportOptions {url:string;authorizationEnv:string;includeExplanation?:boolean}
+export function langfuseScore(value:Evaluation,includeExplanation=false):Record<string,unknown> {
+ if(!value.target.trace_id)throw Error('Select a recorded span before exporting a Langfuse score');
+ if(value.result.status!=='completed'||!value.result.score)throw Error('Skipped/error evaluations are not quality scores');
+ const s=value.result.score;
+ return {id:createHash('sha256').update(`${value.deployment}\0${value.result.id}`).digest('hex'),name:value.result.name,traceId:value.target.trace_id,observationId:value.target.span_id??undefined,dataType:s.type.toUpperCase(),value:s.type==='boolean'?(s.value?1:0):s.value,timestamp:new Date(value.result.created_ms).toISOString(),...(includeExplanation&&value.result.explanation?{comment:value.result.explanation}:{}),metadata:{'openworker.evaluation.id':value.result.id,evaluator:value.result.evaluator.name,evaluatorVersion:value.result.evaluator.version,evaluatorSource:value.result.evaluator.source,evaluatorModel:value.result.evaluator.model,evaluatorUsage:value.result.evaluator.usage,...(value.result.supersedes?{supersedes:value.result.supersedes}:{})}};
+}
+/** Sends one stable score. Does not retry unknown outcomes or re-export a trace. */
+export async function exportLangfuseScore(value:Evaluation,options:ScoreExportOptions):Promise<{id:string}> {
+ const url=new URL(options.url);
+ if(url.username||url.password||url.search||url.hash||url.pathname!=='/'||(url.protocol!=='https:'&&!(url.protocol==='http:'&&['localhost','127.0.0.1','[::1]'].includes(url.hostname))))throw Error('Use a HTTPS origin or explicit loopback Langfuse origin');
+ if(!/^[A-Z_][A-Z0-9_]{0,99}$/.test(options.authorizationEnv))throw Error('Invalid export credential reference');
+ const auth=process.env[options.authorizationEnv];if(!auth)throw Error('Export credential unavailable');
+ const body=langfuseScore(value,options.includeExplanation);let response;
+ try {response=await fetch(new URL('/api/public/scores',url),{method:'POST',headers:{Authorization:auth,'Content-Type':'application/json'},body:JSON.stringify(body),redirect:'error',signal:AbortSignal.timeout(3000)});}
+ catch {throw Error('Score export outcome unknown; reconcile the stable score ID before retrying');}
+ if(response.status!==200){await response.body?.cancel();throw Error(`Score export not confirmed (HTTP ${response.status}); no automatic retry`);}
+ const reader=response.body?.getReader();if(!reader)throw Error('Score export acknowledgement missing');
+ let bytes=0;const chunks:Uint8Array[]=[];
+ for(;;){const {value,done}=await reader.read();if(done)break;bytes+=value.byteLength;if(bytes>65536){await reader.cancel();throw Error('Score acknowledgement too large; outcome unknown');}chunks.push(value);}
+ let ack;try{ack=JSON.parse(Buffer.concat(chunks).toString('utf8'));}catch{throw Error('Invalid score acknowledgement; outcome unknown');}
+ if(ack?.id!==body.id)throw Error('Score identity mismatch; outcome unknown');
+ return {id:String(body.id)};
+}
+/** Caller supplies authorized projections. Content is absent unless explicitly supplied. */
+export function evidenceJsonl(input:{selection:{sessionId:string;turnId:string};watermark:number;timeline:unknown;summary:unknown;evaluations?:Evaluation[];content?:unknown}):string {
+ const records=[{kind:'timeline',value:input.timeline},{kind:'summary',value:input.summary},...(input.evaluations??[]).map(value=>({kind:'evaluation',value})),...(input.content===undefined?[]:[{kind:'selected_content',value:input.content}])];
+ const lines=records.map(r=>JSON.stringify(r)).join('\n')+'\n';
+ if(Buffer.byteLength(lines)>8*1024*1024)throw Error('Evidence export exceeds 8 MiB; export smaller scoped pages');
+ return JSON.stringify({kind:'manifest',version:1,selection:input.selection,watermark:input.watermark,contentIncluded:input.content!==undefined,coverage:'selected_pages_only',guarantee:'best_effort_traces_not_an_audit_archive',sha256:createHash('sha256').update(lines).digest('hex')})+'\n'+lines;
+}
diff --git a/harness/typescript-sdk/src/index.ts b/harness/typescript-sdk/src/index.ts
index ed0b808f..630a9f7c 100644
--- a/harness/typescript-sdk/src/index.ts
+++ b/harness/typescript-sdk/src/index.ts
@@ -1,3 +1,5 @@
+import { Observability } from "./observability.js";
+export * from "./observability.js";
import { normalizeSafety, type SafetyOptions, type SafetyHandler, type SafetyContext, type SafetyDecision } from "./safety.js";
export * from "./safety.js";
import { Service, type ServiceOptions } from "./service.js";
@@ -101,6 +103,7 @@ function interruptionOf(value: unknown): Interruption {
/** Node.js host owner. Agent logic, storage and permission enforcement remain in Rust. */
export class Harness {
+ get observability(): Observability { return new Observability((method,args)=>this.request(method,args as Json)); }
static connect(options: ServiceOptions): Promise<Service> {
return Service.connect(options);
}
diff --git a/harness/typescript-sdk/src/observability.ts b/harness/typescript-sdk/src/observability.ts
new file mode 100644
index 00000000..7b3af046
--- /dev/null
+++ b/harness/typescript-sdk/src/observability.ts
@@ -0,0 +1,120 @@
+/** Optional trace queries. This module neither starts a server nor installs an exporter. */
+import { HarnessError, type Telemetry } from "./types.js";
+export interface TraceQuery { sessionId:string; turnId:string; after?:number; through?:number; limit?:number }
+export interface TraceStep {
+ version:number; deployment:string; role:string; trace_id:string; span_id:string;
+ parent_trace_id:string|null; parent_span_id:string|null; session_hash:string|null; turn_hash?:string|null;
+ operation:string; outcome:string; started_ms:number; ended_ms:number; duration_us:number|null;
+ first_output_us:number|null; attempt:number|null; model:string|null; tool:string|null; purpose:string|null;
+ output_bytes:number|null; usage:{input:number|null;output:number|null;cache_read:number|null;cache_write:number|null;reasoning:number|null}|null;
+ links?:{trace_id:string;span_id:string}[];
+ sandbox?:string|null;enforcement?:string|null;policy_digest?:string|null;
+}
+export interface Timeline {
+ run:{session_id:string;turn_id:string};
+ page:{steps:{cursor:number;step:TraceStep}[];through:number;next:number|null};
+ coverage:{guarantee:"best_effort";order:"recording_cursor";snapshot_across_requests:false};
+}
+export interface Measure {total:number|null;known:number;unknown:number;overflow:boolean}
+export interface Usage {input:Measure;output:Measure;cache_read:Measure;cache_write:Measure;reasoning:Measure}
+export interface TraceGroup {
+ name:string|null;purpose:string|null;calls:number;outcomes:Record<string,number>;
+ accumulated_us:Measure;active_union_us:number|null;first_output_us:Measure;returned_bytes:Measure;usage:Usage|null;
+}
+export interface TraceCoverage {
+ guarantee:string;scanned_steps:number;scan_truncated:boolean;group_truncated:boolean;
+ expected_segments:number|null;recorded_segments:number;all_visible_segments_closed:boolean;pending_waits:number|null;
+ requests_without_completion:number;completions_without_start:number;unknown_timing_steps:number;unknown_outcome_steps:number;
+ clock_or_range_anomaly:boolean;hierarchy_incomplete:boolean;
+}
+export interface Summary {
+ run:{session_id:string;turn_id:string};
+ summary:{projection_version:number;through:number;coverage:TraceCoverage;wall_elapsed_us:number|null;observed_window_us:number|null;active_segments_accumulated_us:Measure;model_accumulated_us:Measure;tool_accumulated_us:Measure;exclusive_observed_us:Record<string,number>;usage:Usage;models:TraceGroup[];tools:TraceGroup[]};
+}
+export interface TurnPage {turns:{session_id:string;turn_id:string;status:string|null;agent:string;name:string}[];next_cursor:number|null;through:number}
+export interface ChildLink {
+ origin:{parent_session_id:string;parent_turn_id:string;parent_call_id:string|null};
+ child_session_id:string;child_turn_id:string|null;agent_id:string;phase:string;
+}
+export interface SubtreeSummary {selection:{runs:{session_id:string;turn_id:string}[];selection_truncated:boolean;scope:"root_and_visible_spawned_initial_turns"};summary:Summary["summary"]}
+export interface Children {children:ChildLink[];coverage:string}
+export interface ObservabilityAdapter {
+ subtreeSummary?(query:TraceQuery):Promise<SubtreeSummary>;
+ children?(query:TraceQuery):Promise<Children>;
+
+ timeline(query:TraceQuery):Promise<Timeline>;
+ summary(query:TraceQuery):Promise<Summary>;
+}
+function checked(q:TraceQuery):void {
+ if (![q.sessionId,q.turnId].every(s=>typeof s==="string"&&/^[a-zA-Z0-9_-]{1,80}$/.test(s)) ||
+ ![q.after??0,q.through??0,q.limit??100].every(n=>Number.isSafeInteger(n)&&n>=0) ||
+ (q.limit??100)<1 || (q.limit??100)>200 || (q.through!==undefined&&q.through<(q.after??0)))
+ throw new HarnessError("InvalidInput","Invalid trace scope or page bounds");
+}
+function wire(q:TraceQuery):Record<string,unknown> {
+ checked(q);
+ return {session_id:q.sessionId,turn_id:q.turnId,after:q.after??0,through:q.through??null,limit:q.limit??100};
+}
+export class Observability implements ObservabilityAdapter {
+ constructor(private readonly request:(method:string,args:Record<string,unknown>)=>Promise<unknown>){}
+ async flush(timeoutMs=5000):Promise<Record<string,unknown>> {if(!Number.isInteger(timeoutMs)||timeoutMs<1||timeoutMs>5000)throw new HarnessError("InvalidInput","Flush budget must be 1..5000ms");return await this.request("observability/flush",{timeout_ms:timeoutMs}) as Record<string,unknown>;}
+ async health():Promise<Record<string,unknown>> {return await this.request("observability/health",{}) as Record<string,unknown>;}
+ async evidence(q:{sessionId:string;turnId:string;afterEventId?:number}):Promise<Record<string,unknown>> {
+ checked({sessionId:q.sessionId,turnId:q.turnId});
+ return await this.request("observability/evidence",{session_id:q.sessionId,turn_id:q.turnId,after_event_id:q.afterEventId??0}) as Record<string,unknown>;
+ }
+ async recordEvaluation(q:TraceQuery,input:EvaluationInput,spanId?:string):Promise<{result:Evaluation;duplicate:boolean}> {
+ checked(q);
+ return await this.request("observability/evaluation-record",{session_id:q.sessionId,turn_id:q.turnId,span_id:spanId??null,result:evaluationWire(input)}) as {result:Evaluation;duplicate:boolean};
+ }
+ async evaluations(q:TraceQuery):Promise<EvaluationPage> {
+ return await this.request("observability/evaluation-list",wire({...q,limit:q.limit??25})) as EvaluationPage;
+ }
+ async listTurns(q:{sessionId:string;after?:number;limit?:number}):Promise<TurnPage> {
+ if(!Number.isSafeInteger(q.after??0)||(q.after??0)<0||!Number.isSafeInteger(q.limit??25)||(q.limit??25)<1||(q.limit??25)>100)throw new HarnessError("InvalidInput","Invalid turn page");
+ return await this.request("observability/turns",{session_id:q.sessionId,after:q.after??0,limit:q.limit??25}) as TurnPage;
+ }
+ async subtreeSummary(q:TraceQuery):Promise<SubtreeSummary>{return await this.request("observability/subtree-summary",wire(q)) as SubtreeSummary;}
+ async children(q:TraceQuery):Promise<Children>{return await this.request("observability/children",wire(q)) as Children;}
+ async timeline(q:TraceQuery):Promise<Timeline>{return await this.request("observability/timeline",wire(q)) as Timeline;}
+ async summary(q:TraceQuery):Promise<Summary>{return await this.request("observability/summary",wire(q)) as Summary;}
+}
+/** Authenticated service APIs enforce access on every read, including descendants. */
+export class ServiceObservability implements ObservabilityAdapter {
+ constructor(private readonly get:(path:string)=>Promise<unknown>){}
+ async health():Promise<Record<string,unknown>> {return await this.get("/v1/observability/health") as Record<string,unknown>;}
+ async search(q:{q?:string;agent?:string;limit?:number;after?:string}={}):Promise<{runs:Record<string,unknown>[];next_cursor:string|null;as_of_us:number}> {
+ const query=new URLSearchParams();for(const [k,v] of Object.entries(q))if(v!==undefined)query.set(k,String(v));
+ return await this.get(`/v1/observability/runs?${query}`) as {runs:Record<string,unknown>[];next_cursor:string|null;as_of_us:number};
+ }
+ private path(q:TraceQuery,kind:string):string {
+ checked(q);const args=new URLSearchParams();
+ if(kind!=="children"&&q.through!==undefined)args.set("through",String(q.through));
+ if(kind==="timeline"){if(q.after!==undefined)args.set("after",String(q.after));if(q.limit!==undefined)args.set("limit",String(q.limit));}
+ return `/v1/sessions/${encodeURIComponent(q.sessionId)}/observability/${encodeURIComponent(q.turnId)}/${kind}${args.size ? `?${args}` : ""}`;
+ }
+ async subtreeSummary(q:TraceQuery):Promise<SubtreeSummary>{return await this.get(this.path(q,"subtree-summary")) as SubtreeSummary;}
+ async children(q:TraceQuery):Promise<Children>{return await this.get(this.path(q,"children")) as Children;}
+ async timeline(q:TraceQuery):Promise<Timeline>{return await this.get(this.path(q,"timeline")) as Timeline;}
+ async summary(q:TraceQuery):Promise<Summary>{return await this.get(this.path(q,"summary")) as Summary;}
+}
+/** Direct Langfuse traces; auth is read in the trusted harness process. */
+export function langfuseTelemetry(endpoint:string, authorizationEnv:string):Telemetry {
+ return {endpoint,signals:["traces"],acknowledgement:"langfuse_v4",headers:[{name:"Authorization",value_env:authorizationEnv},{name:"x-langfuse-ingestion-version",value:"4"}],content:{capture:"metadata"}};
+}
+
+export interface EvaluationInput {
+ id:string;name:string;evaluator:{name:string;version:string;source:"human"|"code"|"model";model?:string;usage?:TraceStep["usage"]};
+ status:"completed"|"skipped"|"error";
+ score?:{type:"numeric";value:number}|{type:"boolean";value:boolean}|{type:"categorical"|"text";value:string};
+ createdMs:number;explanation?:string;datasetCase?:string;experiment?:string;supersedes?:string;
+}
+export interface Evaluation {
+ version:1;deployment:string;
+ target:{session_hash:string;turn_hash:string;trace_id:string|null;span_id:string|null};
+ result:{id:string;name:string;evaluator:EvaluationInput["evaluator"];status:EvaluationInput["status"];score:EvaluationInput["score"]|null;created_ms:number;explanation:string|null;dataset_case:string|null;experiment:string|null;supersedes:string|null};
+}
+export interface EvaluationPage {results:{cursor:number;evaluation:Evaluation}[];next:number|null}
+export function evaluationWire(input:EvaluationInput):Record<string,unknown> {
+ return {id:input.id,name:input.name,evaluator:input.evaluator,status:input.status,score:input.score??null,created_ms:input.createdMs,explanation:input.explanation??null,dataset_case:input.datasetCase??null,experiment:input.experiment??null,supersedes:input.supersedes??null};
+}
diff --git a/harness/typescript-sdk/src/service.ts b/harness/typescript-sdk/src/service.ts
index 4ff91b74..0ed6fc6f 100644
--- a/harness/typescript-sdk/src/service.ts
+++ b/harness/typescript-sdk/src/service.ts
@@ -1,3 +1,4 @@
+import { ServiceObservability } from "./observability.js";
/** Authenticated remote service handles. The service owns all execution and policy. */
import { randomUUID } from "node:crypto";
import {sandboxWire, serviceDelegationWire, type ServiceTeamDelegation} from "./team-placement.js";
@@ -222,6 +223,7 @@ export async function* serviceFrames(
}
}
export class Service {
+ get observability():ServiceObservability {return new ServiceObservability(path=>this.json("GET",path));}
private closed = false;
private requests = new Set<AbortController>();
private endpoint = 0;
diff --git a/harness/typescript-sdk/src/types.ts b/harness/typescript-sdk/src/types.ts
index 8b34028d..2295394a 100644
--- a/harness/typescript-sdk/src/types.ts
+++ b/harness/typescript-sdk/src/types.ts
@@ -167,6 +167,8 @@ export type Storage =
loopbackPlaintext?: boolean;
};
export interface Telemetry {
+ /** Receiver protocol; direct Langfuse v4 requires its explicit receipt dialect. */
+ acknowledgement?: "otlp" | "langfuse_v4";
endpoint: string;
deployment?: string;
headers?: (
@@ -205,6 +207,7 @@ export interface HarnessOptions extends SafetyOptions {
sandbox?: "openshell" | "off" | { provider: "openshell" };
mcpServers?: McpServer[];
telemetry?: Telemetry;
+ recording?: RecordingOptions;
restart?: { maxRestarts?: number; delayMs?: number };
requestTimeoutMs?: number;
shutdownTimeoutMs?: number;
@@ -452,3 +455,17 @@ export interface ComplianceActivity {
}
export interface CompliancePage { activities:ComplianceActivity[]; nextAfter:number; hasMore:boolean }
export interface ComplianceDeliveryHealth { sinks:{sink:string;cursor:number;pending:number;failures:number;nextAtMs:number;lastSuccessMs:number|null}[] }
+
+/** Optional diagnostics storage, independent from execution storage. */
+export interface RecordingOptions {
+ maxPendingEvaluations?:number;
+ evaluationRetentionDays?:number;
+ evaluationSinks?: {name:string;url:string;authorizationEnv:string;includeExplanation?:boolean}[];
+ evidence?: {userInput?:boolean;assistantOutput?:boolean;toolArguments?:boolean;toolResults?:boolean;maxBytes:number};
+ /** Enable acknowledged evaluation-result storage (requires runtime capability). */
+ evaluations?: boolean;
+ database: { type: "sqlite"; path: string } | { type: "postgres"; connectionEnv: string; schema: string; caFile?: string; loopbackPlaintext?: boolean };
+ deployment?: string;
+ retentionDays?: number;
+ queueCapacity?: number;
+}
diff --git a/harness/typescript-sdk/test/evaluation-delivery-live.test.mjs b/harness/typescript-sdk/test/evaluation-delivery-live.test.mjs
new file mode 100644
index 00000000..f7398f18
--- /dev/null
+++ b/harness/typescript-sdk/test/evaluation-delivery-live.test.mjs
@@ -0,0 +1,57 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {createServer} from 'node:http';
+import {mkdtemp,mkdir,realpath,rm} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {setTimeout as delay} from 'node:timers/promises';
+import {Harness,guard} from '../dist/index.js';
+const executable=process.env.OW_TS_TEST_HOST;
+assert(executable,'Set OW_TS_TEST_HOST to sdk-test-host built with evaluation-export');
+async function until(check) {const end=Date.now()+15000;do {if(await check())return;await delay(50);}while(Date.now()<end);assert.fail('Timed out waiting for delivery');}
+test('pending score survives host restart and uncertain ACK without replaying a turn',{timeout:60000},async t=>{
+ const root=await realpath(await mkdtemp(join(tmpdir(),'ow-score-delivery-')));await mkdir(join(root,'project'));
+ let mode='unauthorized';const bodies=[];
+ const server=createServer(async(req,res)=>{
+ let body='';for await(const c of req)body+=c;
+ assert.equal(req.url,'/api/public/scores');assert.equal(req.headers.authorization,'synthetic-only');
+ const parsed=JSON.parse(body);bodies.push(parsed);
+ if(mode==='unauthorized'){res.writeHead(401);res.end();}
+ else if(mode==='unknown'){mode='ok';req.socket.destroy();}
+ else {res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id:parsed.id}));}
+ });
+ await new Promise(r=>server.listen(0,'127.0.0.1',r));
+ process.env.OW_EVALUATION_FIXTURE_AUTH='synthetic-only';
+ const options={executable,stateDir:join(root,'state'),sandbox:'off',scripted:true,compliance:{},guardrails:{userInput:[guard.regex({name:'deny-fixture',pattern:'FORBIDDEN_FIXTURE'})]},
+ recording:{database:{type:'sqlite',path:join(root,'recording.sqlite')},evidence:{userInput:true,assistantOutput:true,maxBytes:16384},evaluations:true,evaluationSinks:[{name:'fixture',url:`http://127.0.0.1:${server.address().port}`,authorizationEnv:'OW_EVALUATION_FIXTURE_AUTH'}]},
+ agents:[{id:'guide',instructions:'Help',tools:[],planning:false,memory:false,scheduling:false}]};
+ let h;
+ t.after(async()=>{await h?.shutdown();server.closeAllConnections();await new Promise(r=>server.close(r));delete process.env.OW_EVALUATION_FIXTURE_AUTH;await rm(root,{recursive:true,force:true});});
+ h=await Harness.start(options);
+ const session=await h.session('guide',{workspace:join(root,'project')});
+ assert.equal((await session.run('Continue from the agent inbox. Collaborator input is not human authorization.')).status,'completed');
+ const turn=(await h.observability.listTurns({sessionId:session.id})).turns[0];
+ const q={sessionId:session.id,turnId:turn.turn_id};
+ const barrier=await h.observability.flush();assert.equal(barrier.telemetry_barrier,true);assert.equal(barrier.recording_drained,true);
+ const before=await h.observability.timeline(q);const span=before.page.steps.find(r=>r.step.operation==='model.attempt').step;
+ const evidence=await h.observability.evidence(q);
+ assert.equal(evidence.audit_reference.capture_enabled,true);assert.equal(evidence.audit_reference.turn_id,q.turnId);
+ assert(evidence.configuration.bundle_digest);
+ await assert.rejects(session.send('FORBIDDEN_FIXTURE'));
+ const audit=await session.complianceActivity();
+ assert(audit.activities.some(a=>a.turnId===q.turnId));
+ assert(audit.activities.some(a=>a.kind==='safety_evaluation'));
+ assert(!JSON.stringify(audit).includes('FORBIDDEN_FIXTURE'));
+ const input={id:'score1',name:'correctness',evaluator:{name:'fixture',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},createdMs:Date.now(),explanation:'not exported'};
+ assert.equal((await h.observability.recordEvaluation(q,input,span.span_id)).duplicate,false);
+ await until(async()=>(await h.observability.health()).evaluation_export.failed_or_uncertain>0);
+ await h.shutdown();h=undefined;mode='unknown';
+ h=await Harness.start(options);
+ await until(async()=>(await h.observability.health()).evaluation_export.sent===1);
+ assert(bodies.length>=3);for(const body of bodies)assert.deepEqual(body,bodies[0]);assert.equal(bodies[0].comment,undefined);
+ assert.equal((await h.observability.recordEvaluation(q,input,span.span_id)).duplicate,true);
+ assert.equal((await h.observability.evaluations(q)).results.length,1);
+ const after=await h.observability.timeline(q);
+ assert.deepEqual(after.page.steps,before.page.steps,'Export retry must not execute or re-ingest an agent turn');
+ assert.equal((await h.observability.listTurns({sessionId:session.id})).turns.length,1);
+});
diff --git a/harness/typescript-sdk/test/evaluation-export.test.mjs b/harness/typescript-sdk/test/evaluation-export.test.mjs
new file mode 100644
index 00000000..fea71ebb
--- /dev/null
+++ b/harness/typescript-sdk/test/evaluation-export.test.mjs
@@ -0,0 +1,18 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {createServer} from 'node:http';
+import {langfuseScore,exportLangfuseScore,evidenceJsonl} from '../dist/evaluation-export.js';
+const evaluation={version:1,deployment:'fixture',target:{session_hash:'1'.repeat(64),turn_hash:'2'.repeat(64),trace_id:'3'.repeat(32),span_id:'4'.repeat(16)},result:{id:'quality-1',name:'correctness',evaluator:{name:'code',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},created_ms:1000,explanation:'private explanation'}};
+test('score identity and timestamp are stable; explanation is opt-in',()=>{
+ const a=langfuseScore(evaluation);assert.deepEqual(a,langfuseScore(evaluation));assert.equal(a.timestamp,'1970-01-01T00:00:01.000Z');assert.equal(a.comment,undefined);assert.equal(a.value,1);assert.equal(langfuseScore(evaluation,true).comment,'private explanation');assert.throws(()=>langfuseScore({...evaluation,target:{...evaluation.target,trace_id:null}}));
+});
+test('export sends only score endpoint and never follows redirects or retries',async t=>{
+ let calls=0;let mode='ok';const server=createServer(async(req,res)=>{calls++;let text='';for await(const c of req)text+=c;assert.equal(req.url,'/api/public/scores');const body=JSON.parse(text);if(mode==='redirect'){res.writeHead(302,{location:'/unwanted'});res.end();}else {res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id:body.id}));}});
+ await new Promise(r=>server.listen(0,'127.0.0.1',r));t.after(()=>{server.closeAllConnections();server.close();delete process.env.OW_TEST_EVAL_AUTH;});process.env.OW_TEST_EVAL_AUTH='synthetic-fixture-only';
+ const options={url:`http://127.0.0.1:${server.address().port}`,authorizationEnv:'OW_TEST_EVAL_AUTH'};
+ assert.equal((await exportLangfuseScore(evaluation,options)).id,langfuseScore(evaluation).id);mode='redirect';await assert.rejects(exportLangfuseScore(evaluation,options),/unknown/);assert.equal(calls,2);
+});
+test('JSONL manifest names coverage and content explicitly',()=>{
+ const bundle=evidenceJsonl({selection:{sessionId:'s',turnId:'t'},watermark:4,timeline:{},summary:{},evaluations:[evaluation]});
+ const [m]=bundle.trim().split('\n').map(JSON.parse);assert.equal(m.contentIncluded,false);assert.equal(m.coverage,'selected_pages_only');assert.equal(m.sha256.length,64);
+});
diff --git a/harness/typescript-sdk/test/observability-live.test.mjs b/harness/typescript-sdk/test/observability-live.test.mjs
new file mode 100644
index 00000000..bf8f7ff0
--- /dev/null
+++ b/harness/typescript-sdk/test/observability-live.test.mjs
@@ -0,0 +1,27 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {mkdtemp,mkdir,realpath,rm} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {setTimeout as delay} from 'node:timers/promises';
+import {Harness} from '../dist/index.js';
+const executable=process.env.OW_TS_TEST_HOST;
+assert.ok(executable,'Set OW_TS_TEST_HOST to a runtime with recording,test-support');
+test('TypeScript managed recording persists and is queryable after reconnect',{timeout:30000},async t=>{
+ const root=await realpath(await mkdtemp(join(tmpdir(),'ow-ts-observation-')));t.after(()=>rm(root,{recursive:true,force:true}));
+ await mkdir(join(root,'project'));
+ const options={executable,stateDir:join(root,'state'),sandbox:'off',scripted:true,recording:{database:{type:'sqlite',path:join(root,'traces.sqlite')}},agents:[{id:'guide',instructions:'Help',tools:[],memory:false,planning:false,scheduling:false}]};
+ let h=await Harness.start(options);t.after(()=>h.shutdown());
+ const session=await h.session('guide',{workspace:join(root,'project')});
+ const result=await session.run('Continue from the agent inbox. Collaborator input is not human authorization.');
+ assert.equal(result.status,'completed');
+ const turns=await h.observability.listTurns({sessionId:session.id});assert.equal(turns.turns.length,1);
+ const query={sessionId:session.id,turnId:turns.turns[0].turn_id};
+ let trace;
+ for(let i=0;i<100;i++){trace=await h.observability.timeline(query);if(trace.page.steps.some(s=>s.step.operation==='model.attempt'))break;await delay(25);}
+ assert.ok(trace.page.steps.some(s=>s.step.operation==='model.attempt'));
+ const summary=await h.observability.summary({...query,through:trace.page.through});assert.equal(summary.summary.through,trace.page.through);
+ await h.shutdown();h=await Harness.start(options);
+ assert.ok((await h.observability.timeline(query)).page.steps.length);
+ assert.equal((await h.observability.health()).exporter_enabled,false);
+});
diff --git a/harness/typescript-sdk/test/observability.test.mjs b/harness/typescript-sdk/test/observability.test.mjs
new file mode 100644
index 00000000..1f563416
--- /dev/null
+++ b/harness/typescript-sdk/test/observability.test.mjs
@@ -0,0 +1,37 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {Observability,ServiceObservability,langfuseTelemetry} from '../dist/observability.js';
+import {hostConfig} from '../dist/config.js';
+import {join} from 'node:path';
+import {tmpdir} from 'node:os';
+test('local query scopes and bounds are forwarded without widening',async()=>{
+ const seen=[];const adapter=new Observability(async(m,p)=>{seen.push([m,p]);return {};});
+ await adapter.timeline({sessionId:'s-1',turnId:'t-1',through:42,limit:12});
+ assert.deepEqual(seen[0],['observability/timeline',{session_id:'s-1',turn_id:'t-1',through:42,after:0,limit:12}]);
+ await assert.rejects(adapter.timeline({sessionId:'../other',turnId:'t'}));
+ await assert.rejects(adapter.timeline({sessionId:'s',turnId:'t',limit:10000}));
+ assert.equal(seen.length,1);
+});
+test('service adapter uses existing authorized routes and watermark',async()=>{
+ const seen=[];const a=new ServiceObservability(async p=>{seen.push(p);return {};});
+ await a.summary({sessionId:'s',turnId:'t',through:9});
+ await a.search({q:'a/b ?'});
+ assert.equal(seen[0],'/v1/sessions/s/observability/t/summary?through=9');
+ assert.equal(seen[1],'/v1/observability/runs?q=a%2Fb+%3F');
+});
+test('recording is off by default; direct Langfuse uses typed receipts and secret reference',()=>{
+ const base={executable:join(tmpdir(),'harness'),stateDir:join(tmpdir(),'state')};
+ assert.equal(hostConfig(base).recording,undefined);
+ const telemetry=langfuseTelemetry('https://cloud.langfuse.com/api/public/otel','MY_EXPORT_AUTH');
+ const wire=hostConfig({...base,telemetry,recording:{database:{type:'sqlite',path:join(tmpdir(),'traces.sqlite')},retentionDays:7}});
+ assert.equal(wire.telemetry.acknowledgement,'langfuse_v4');
+ assert.equal(wire.telemetry.content.capture,'metadata');
+ assert.deepEqual(wire.telemetry.signals,['traces']);
+ assert.equal(wire.recording.retention_days,7);
+ assert.equal(wire.telemetry.headers[0].value,undefined);
+});
+test('PostgreSQL recording uses a distinct explicit credential reference',()=>{
+ const c=hostConfig({executable:join(tmpdir(),'harness'),stateDir:join(tmpdir(),'state'),recording:{database:{type:'postgres',connectionEnv:'OBS_DB',schema:'ow_obs',loopbackPlaintext:true}}});
+ assert.deepEqual(c.recording.database,{kind:'postgres',connection_env:'OBS_DB',schema:'ow_obs',ca_file:null,loopback_plaintext:true});
+ assert.equal(c.storage.kind,'sqlite');
+});
diff --git a/skills/install.mjs b/skills/install.mjs
index 022f1394..b96aa646 100644
--- a/skills/install.mjs
+++ b/skills/install.mjs
@@ -7,7 +7,7 @@ import { homedir } from 'node:os';
import { parseArgs } from 'node:util';
const source = dirname(fileURLToPath(import.meta.url));
const usage = 'build-with-sdk-skills --agent codex|claude (--project /path/to/your-app | --global)';
-const files = ['SKILL.md','references/desktop.md','references/web.md','references/sdk.md','references/runtime-options.md','references/subagents.md','references/railway.md','assets/railway/Dockerfile','assets/railway/.dockerignore','agents/openai.yaml'];
+const files = ['SKILL.md','references/desktop.md','references/web.md','references/sdk.md','references/runtime-options.md','references/subagents.md','references/observability.md','references/railway.md','assets/railway/Dockerfile','assets/railway/.dockerignore','agents/openai.yaml'];
function validate({agent, project, global: userWide = false}) {
if (!['codex','claude'].includes(agent) || typeof userWide !== 'boolean' ||
(project !== undefined && (typeof project !== 'string' || !project.trim())) ||
diff --git a/skills/openworker-apps/SKILL.md b/skills/openworker-apps/SKILL.md
index 20bcfb1c..d4ca5059 100644
--- a/skills/openworker-apps/SKILL.md
+++ b/skills/openworker-apps/SKILL.md
@@ -15,6 +15,8 @@ Preserve their chosen UI framework, model and deployment target.
history or changing turn preferences, read [Runtime options](references/runtime-options.md).
- For model-created child agents, read [Standalone sub-agents](references/subagents.md)
and check its runtime-version gate before using the option.
+- For optional turn traces, Langfuse export or evaluation evidence, read
+ [Observability](references/observability.md) and check its release gate.
- For desktop apps, read [Desktop](references/desktop.md).
- For browser/server apps, read [Web](references/web.md).
- For a Railway deployment, read [Railway](references/railway.md). Default to one
diff --git a/skills/openworker-apps/references/observability.md b/skills/openworker-apps/references/observability.md
new file mode 100644
index 00000000..06df829b
--- /dev/null
+++ b/skills/openworker-apps/references/observability.md
@@ -0,0 +1,36 @@
+# Optional turn traces and evaluations
+
+The high-level recording, query and evaluation APIs are an unpublished source
+candidate, not available in the currently published 0.1.0-beta.2 runtime. Use them
+only when the supplied SDK declarations and runtime capabilities both support
+`recording` (and `evaluations`/`evaluation_export` as needed). Otherwise use existing
+`model_usage` events or configured OTLP; do not invent a compatibility fallback.
+
+For a supported candidate:
+
+- Configure `Harness.start({recording:{database:{type:'sqlite',path:absolutePrivatePath}}})`
+ only if local trace retention is wanted. Keep it outside model-accessible workspaces.
+ SQLite execution and recording are different files. PostgreSQL uses a separate
+ `connectionEnv`/schema and explicit trusted migration.
+- Read `harness.observability.listTurns`, `timeline`, `summary`, `children` or
+ `subtreeSummary`. Follow bounded cursors; unknown usage is not zero. Cache and
+ reasoning tokens are subsets. Subtree includes spawned initial tasks, not every
+ later message to a child. Never sum overlapping durations as elapsed time.
+- `langfuseTelemetry(endpoint, authorizationEnv)` configures metadata-only v4 OTLP
+ traces. Keep credentials in the trusted process. Do not double-export one stream.
+- `recording.evaluations:true` enables durable `recordEvaluation` results. Same ID
+ plus same payload is idempotent; corrections use a new ID and `supersedes`.
+ `skipped`/`error` results carry no quality score. Evaluators are app-owned.
+- `recording.evaluationSinks` enables a durable score-only queue. It never retries
+ agent actions. Reuse original result timestamps and stable IDs across retries.
+ Explanations are not exported unless selected. Do not combine this queue with
+ a second automatic score exporter to the same destination.
+- `recording.evidence` is a separate content opt-in. The authorized app can request
+ bounded committed output/tool projections. Never expose the trusted host query
+ API to browsers without session ownership and separate content authorization.
+- The optional trace-viewer source package accepts an app adapter; do not add a
+ UI framework or database to a lightweight app just for instrumentation.
+
+On export failure, report diagnostics and retain durable results as configured.
+Never replay a tool or turn to recreate missing telemetry. Preserve required
+compliance policy; best-effort traces do not replace the audit journal.