Python
Add optional SDK observability and evaluation exports
Rohit P88c353516659
diff --git a/.github/workflows/harness-packages.yml b/.github/workflows/harness-packages.ymlindex 16f20b2e..1c00d8fd 100644--- a/.github/workflows/harness-packages.yml+++ b/.github/workflows/harness-packages.yml@@ -54,7 +54,7 @@ jobs:run: cargo test --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --features test-support --test desktop_api --test tool_free -- --test-threads=1# Build after qualification so test-support cannot affect the shipped host.- name: Build native host- run: cargo build --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export+ run: cargo build --manifest-path harness/Cargo.toml --locked --release -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export,recording-postgres,evaluation-export- name: Prepare immutable candidate artifactsrun: node harness/release/prepare.mjs "$GITHUB_WORKSPACE/harness/target/release/${{ matrix.exe }}" '${{ matrix.target }}' .sdk-release- name: Install tarballs and run the exact TypeScript guidediff --git a/.github/workflows/rust-sdk.yml b/.github/workflows/rust-sdk.ymlindex 88e74531..c3bbea90 100644--- a/.github/workflows/rust-sdk.yml+++ b/.github/workflows/rust-sdk.yml@@ -21,7 +21,7 @@ jobs:with:node-version: 24- name: Public layout and retained transport tests- run: node --test harness/tests/release-layout.test.mjs harness/tests/stdio-client.test.mjs harness/desktop-starters/test.mjs+ run: node --test harness/tests/release-layout.test.mjs harness/tests/stdio-client.test.mjs harness/desktop-starters/test.mjs harness/service-ui/test/observability.test.mjs harness/service-ui/test/server.test.mjs- name: Build SDK and flagship app without launching Electronrun: |npm --prefix harness/typescript-sdk ci@@ -71,6 +71,7 @@ jobs:run: |npm --prefix typescript-sdk cinpm --prefix typescript-sdk test+ npm --prefix typescript-sdk run test:observabilityif [ "$RUNNER_OS" = Windows ]; thenecho "OW_TS_TEST_HOST=$(pwd -W)/target/debug/sdk-test-host.exe" >> "$GITHUB_ENV"else@@ -219,12 +220,14 @@ jobs:CARGO_PROFILE_RELEASE_LTO: "false"CARGO_PROFILE_RELEASE_OPT_LEVEL: "1"CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"- run: cargo build --locked --release -p openworker-sdk --features test-support --bin sdk-test-host+ run: cargo build --locked --release -p openworker-sdk --features test-support,recording,evaluation-export --bin sdk-test-host- name: TypeScript SDK transport and real-host storage parityrun: |export OW_TS_TEST_HOST="$PWD/target/release/sdk-test-host"npm --prefix typescript-sdk cinpm --prefix typescript-sdk test+ node --test typescript-sdk/test/observability-live.test.mjs typescript-sdk/test/evaluation-delivery-live.test.mjs+ node ../harness/trace-viewer/sync.mjs --checknpm --prefix typescript-sdk run test:integrationnpm --prefix typescript-sdk run test:dynamic-agentsnode --test --test-concurrency=2 --test-reporter=tap typescript-sdk/test/chat-usage.test.mjs typescript-sdk/test/openrouter-streaming.test.mjs typescript-sdk/test/anthropic-caching.test.mjs typescript-sdk/test/gemini-compat.test.mjs typescript-sdk/test/session-options.test.mjs typescript-sdk/test/stream-events.test.mjs@@ -284,6 +287,8 @@ jobs:run: |cargo test --locked -p openworker-runtime -- --test-threads=1cargo test --locked -p openworker-telemetry --features otlp+ OW_OBSERVABILITY_TEST_URL="$OW_PG_TEST_URL" cargo test --locked -p openworker-observability --features postgres,evaluations -- --include-ignored --test-threads=1+ cargo test --locked -p openworker-store-postgres --features test-support --test environments terminal_team_save_ -- --ignored --test-threads=1cargo test --locked -p openworker-store-postgres --test store imported_history_and_turn_context_ -- --ignored --test-threads=1- name: Shared PostgreSQL team contractrun: cargo test --locked -p openworker-store-postgres --features test-support --test team_contract --test agent_team_contract --test agent_team_worker --test standalone_subagents -- --ignored --test-threads=1diff --git a/deploy/reference/check-recorded-langfuse.mjs b/deploy/reference/check-recorded-langfuse.mjsnew file mode 100644index 00000000..89d6b36a--- /dev/null+++ b/deploy/reference/check-recorded-langfuse.mjs@@ -0,0 +1,30 @@+// Compare an exported OpenShell service qualification with local Langfuse.+import assert from 'node:assert/strict';+import {readFile,writeFile} from 'node:fs/promises';+import {setTimeout as delay} from 'node:timers/promises';+const [file,url,credentialFile,output]=process.argv.slice(2);+const endpoint=new URL(url);assert(['127.0.0.1','localhost'].includes(endpoint.hostname));+const fixture=JSON.parse(await readFile(file,'utf8'));+const credentials=JSON.parse(await readFile(credentialFile,'utf8'));+const auth='Basic '+Buffer.from(credentials.publicKey+':'+credentials.secretKey).toString('base64');+const records=[fixture,...fixture.childTraces].flatMap(f=>f.timeline.page.steps.map(r=>r.step));+assert.equal(new Set(records.map(r=>r.span_id)).size,records.length,'Duplicate local span');+const ids=[...new Set(records.map(r=>r.trace_id))];+const start=new Date(Math.min(...records.map(r=>r.started_ms))-60000).toISOString();+let observations=[];const deadline=Date.now()+90000;+do {+ observations=[];+ for(const traceId of ids){const qs=new URLSearchParams({traceId,limit:'1000',fields:'core,basic,usage,io,model,metadata',fromStartTime:start,toStartTime:new Date().toISOString()});+ const response=await fetch(`${endpoint.origin}/api/public/v2/observations?${qs}`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});assert.equal(response.status,200);observations.push(...(await response.json()).data);}+ if(records.every(r=>observations.some(o=>o.id===r.span_id)))break;await delay(1000);+}while(Date.now()<deadline);+const observed=new Map(observations.map(o=>[o.id,o]));+for(const r of records){const o=observed.get(r.span_id);assert(o,`Missing span ${r.span_id}`);assert.equal(o.traceId,r.trace_id);assert.equal(o.parentObservationId??null,r.parent_span_id??null);assert(!o.input&&!o.output,'Content leaked');+ if(r.operation==='model.attempt'){+ assert.equal(observations.filter(o=>o.id===r.span_id).length,1,'Duplicate model span');+ if(r.usage?.input!=null)assert.equal((o.usageDetails.input??0)+(o.usageDetails.input_cached_tokens??0),r.usage.input);+ if(r.usage?.output!=null)assert.equal(o.usageDetails.output,r.usage.output);+ }+}+const result={ok:true,qualifiedAt:new Date().toISOString(),matchedSpans:records.length,modelAttempts:records.filter(r=>r.operation==='model.attempt').length,toolCalls:records.filter(r=>r.operation==='tool.execute').length,visibleTurns:fixture.subtree.selection.runs.length,traceIds:ids,contentExported:false};+await writeFile(output,JSON.stringify(result,null,2)+'\n',{mode:0o600});console.log(JSON.stringify(result));diff --git a/deploy/reference/qualify-sdk-observability.mjs b/deploy/reference/qualify-sdk-observability.mjsnew file mode 100644index 00000000..dab752c0--- /dev/null+++ b/deploy/reference/qualify-sdk-observability.mjs@@ -0,0 +1,89 @@+// Actual managed SDK, deterministic model, persistent recording and local Langfuse.+// Credentials are supplied explicitly and never written into evidence output.+import assert from 'node:assert/strict';+import {createHash} from 'node:crypto';+import {createServer} from 'node:http';+import {mkdtemp,readFile,writeFile,rm} from 'node:fs/promises';+import {tmpdir} from 'node:os';+import {join} from 'node:path';+import {setTimeout as delay} from 'node:timers/promises';+import {Harness,langfuseTelemetry} from '../../harness/typescript-sdk/dist/index.js';+const [executable,url,credentialFile,outputFile]=process.argv.slice(2);+assert(executable&&url&&credentialFile&&outputFile,'Usage: qualify-sdk-observability.mjs HOST LOCAL_LANGFUSE_URL CREDENTIAL_JSON OUTPUT_JSON');+const endpoint=new URL(url);assert(['127.0.0.1','localhost'].includes(endpoint.hostname)&&endpoint.protocol==='http:','Qualification requires local loopback Langfuse');+const credentials=JSON.parse(await readFile(credentialFile,'utf8'));+const auth='Basic '+Buffer.from(credentials.publicKey+':'+credentials.secretKey).toString('base64');+const previous=process.env.OW_SDK_OBSERVABILITY_AUTH;+process.env.OW_SDK_OBSERVABILITY_AUTH=auth;+const root=await mkdtemp(join(tmpdir(),'ow-sdk-observability-'));+let calls=0;+const model=createServer(async(req,res)=>{+ for await(const _chunk of req){};calls++;+ res.writeHead(200,{'content-type':'text/event-stream'});+ for(const frame of [+ {choices:[{index:0,delta:{role:'assistant',content:'Fixture answer.'},finish_reason:null}]},+ {choices:[{index:0,delta:{},finish_reason:'stop'}]},+ {choices:[],usage:{prompt_tokens:42,completion_tokens:7,total_tokens:49,prompt_tokens_details:{cached_tokens:12}}},+ ])res.write(`data: ${JSON.stringify(frame)}\n\n`);+ res.end('data: [DONE]\n\n');+});+await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));+let harness;+try {+ harness=await Harness.start({executable,stateDir:join(root,'state'),sandbox:'off',recording:{database:{type:'sqlite',path:join(root,'traces.sqlite')},evaluations:true,evaluationSinks:[{name:'local-langfuse',url:endpoint.origin,authorizationEnv:'OW_SDK_OBSERVABILITY_AUTH'}]},telemetry:langfuseTelemetry(`${endpoint.origin}/api/public/otel`,'OW_SDK_OBSERVABILITY_AUTH')});+ await harness.setModelProvider({name:'ObservabilityFixture',api:'chat-completions',baseUrl:`http://127.0.0.1:${model.address().port}/v1`});+ const session=await harness.createAgent({name:'recording-demo',instructions:'Reply briefly.',tools:[],planning:false,memory:false,scheduling:false,defaultModel:{provider:'ObservabilityFixture',name:'fixture'}}).newSession();+ const result=await session.run('Private synthetic prompt for recording qualification.');assert.equal(result.status,'completed');assert.equal(calls,1);+ const turns=await harness.observability.listTurns({sessionId:session.id});+ const query={sessionId:session.id,turnId:turns.turns[0].turn_id};+ let timeline,summary;+ const deadline=Date.now()+15000;+ do {timeline=await harness.observability.timeline(query);summary=await harness.observability.summary({...query,through:timeline.page.through});if(summary.summary.usage.input.total===42&&timeline.page.steps.some(s=>s.step.operation==='agent.execute'))break;await delay(100);}while(Date.now()<deadline);+ assert.equal(summary.summary.usage.input.total,42);assert.equal(summary.summary.usage.output.total,7);assert.equal(summary.summary.usage.cache_read.total,12);+ const modelSpan=timeline.page.steps.find(r=>r.step.operation==='model.attempt').step;+ const scoreInput={id:`qualification-${query.turnId}`,name:'fixture-correctness',evaluator:{name:'fixture',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},createdMs:Date.now()};+ const score=await harness.observability.recordEvaluation(query,scoreInput,modelSpan.span_id);+ assert.equal(score.duplicate,false);+ assert.equal((await harness.observability.recordEvaluation(query,scoreInput,modelSpan.span_id)).duplicate,true);+ const scoreDeadline=Date.now()+15000;+ while((await harness.observability.health()).evaluation_export.sent<1&&Date.now()<scoreDeadline)await delay(100);+ assert.equal((await harness.observability.health()).evaluation_export.sent,1);+ assert.equal(calls,1,'Evaluation export must not rerun the model');+ await harness.shutdown();harness=undefined;+ const scoreId=createHash('sha256').update(`sdk\0${scoreInput.id}`).digest('hex');+ const source=timeline.page.steps.map(s=>s.step);const traceIds=[...new Set(source.map(s=>s.trace_id))];+ const until=Date.now()+90000;let observations=[];+ do {+ observations=[];+ for(const traceId of traceIds){+ const qs=new URLSearchParams({traceId,limit:'1000',fields:'core,basic,usage,io,model,metadata',fromStartTime:new Date(Date.now()-120000).toISOString(),toStartTime:new Date().toISOString()});+ const response=await fetch(`${endpoint.origin}/api/public/v2/observations?${qs}`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});+ assert.equal(response.status,200,'Langfuse observation query failed');observations.push(...(await response.json()).data);+ }+ if(source.every(s=>observations.some(o=>o.id===s.span_id)))break;+ await delay(1000);+ }while(Date.now()<until);+ const byId=new Map(observations.map(o=>[o.id,o]));+ for(const s of source){const o=byId.get(s.span_id);assert(o,`Missing exported span ${s.span_id}`);assert.equal(o.traceId,s.trace_id);assert.equal(o.parentObservationId??null,s.parent_span_id??null);assert(!o.input&&!o.output,'Metadata-only export leaked content');}+ const generations=source.filter(s=>s.operation==='model.attempt');assert.equal(generations.length,1);+ const usage=byId.get(generations[0].span_id).usageDetails;+ assert.equal((usage.input??0)+(usage.input_cached_tokens??0),42);assert.equal(usage.output,7);assert.equal(usage.total,49);+ assert.equal(observations.filter(o=>o.id===generations[0].span_id).length,1,'Duplicate model observation');+ let vendorScore;+ const scoreUntil=Date.now()+90000;+ do {+ const response=await fetch(`${endpoint.origin}/api/public/v3/scores?id=${scoreId}&fields=subject,details&limit=100`,{headers:{authorization:auth},signal:AbortSignal.timeout(10000)});+ assert.equal(response.status,200,'Langfuse scores query failed');+ vendorScore=(await response.json()).data.find(s=>s.id===scoreId);+ if(vendorScore)break;+ await delay(1000);+ }while(Date.now()<scoreUntil);+ assert(vendorScore,'Exported evaluation missing from Langfuse');+ assert.equal(vendorScore.subject.traceId,modelSpan.trace_id);assert.equal(vendorScore.subject.id,modelSpan.span_id);assert.equal(vendorScore.value,true);+ assert.equal(vendorScore.name,scoreInput.name);+ await writeFile(outputFile,JSON.stringify({version:1,qualifiedAt:new Date().toISOString(),calls,matchedSpans:source.length,evaluationRecord:score.result,evaluation:{id:scoreId,traceId:modelSpan.trace_id,spanId:modelSpan.span_id,name:scoreInput.name,value:1},usage:{input:42,output:7,cacheRead:12},timeline,summary,langfuse:{url:endpoint.origin,traceIds}},null,2)+'\n',{mode:0o600});+ console.log(JSON.stringify({ok:true,matchedSpans:source.length,input:42,output:7,cacheRead:12}));+} finally {+ await harness?.shutdown();model.closeAllConnections();await new Promise(resolve=>model.close(resolve));await rm(root,{recursive:true,force:true});+ if(previous===undefined)delete process.env.OW_SDK_OBSERVABILITY_AUTH;else process.env.OW_SDK_OBSERVABILITY_AUTH=previous;+}diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.mdindex 5aa282cf..fb193e2b 100644--- a/docs/harness/guides/observability.md+++ b/docs/harness/guides/observability.md@@ -1,14 +1,167 @@# Observe agent work-Use the management UI to search for a run, inspect its timeline and compare grouped-model/tool activity. Export OpenTelemetry to your collector or supported analysis-service when you need organization-wide dashboards.--Observability and compliance have different jobs. Traces and metrics help explain-latency, usage and failures. The compliance journal records committed policy and-execution metadata with scoped queries and durable export. Do not turn on prompt-or tool-content capture just to obtain usage statistics.--See the [observability setup](../../../harness/observability/README.md) for-storage and export configuration, and [compliance](compliance.md) for audit-retention and SIEM delivery. A missing exporter must not trigger tool replay.+Inspect each turn's timeline, model usage, tools and spawned tasks. Keep recordings+locally, forward traces to a collector or Langfuse, or use both. These are separate+options; neither is needed to run an agent.++The high-level recording and evaluation APIs below are an **unpublished source+candidate**. They are not in the current `0.1.0-beta.2` npm release. Use matching+local SDK/runtime builds until the next release is reviewed.++## Record locally++Use a recording database separate from execution storage. Content capture is off;+timing, outcome, usage and identifiers are enough for most product dashboards.++```typescript+import {Harness} from '@openworkerai/harness-sdk';++const harness = await Harness.start({+ stateDir: '/absolute/private/app/state',+ recording: {+ database: {type: 'sqlite', path: '/absolute/private/app/observations.sqlite'},+ retentionDays: 30,+ queueCapacity: 1024,+ },+});+// Create/run your agent as usual; then query its accepted turn.+const turns = await harness.observability.listTurns({sessionId});+const query = {sessionId, turnId: turns.turns[0].turn_id};+await harness.observability.flush(5000);+const timeline = await harness.observability.timeline(query);+const summary = await harness.observability.summary({...query, through: timeline.page.through});+```++Rust uses `Harness::builder().recording(RecordingConfig::sqlite(path))` and+`harness.observability().timeline(TraceQuery::turn(session_id, turn_id))`.+Enable Cargo feature `recording`; add `recording-postgres` for PostgreSQL,+`evaluations` for durable results, and `evaluation-export` for background score+delivery. `otlp` is a separate default feature; `default-features = false` removes+it alongside default guardrail/compliance implementations. No UI dependency is+added to the runtime. Standard packaged binaries include optional capabilities,+but recording, evidence and evaluation delivery remain disabled until configured.++For PostgreSQL use a second `connectionEnv` and schema under+`recording.database: {type: 'postgres', connectionEnv: 'OBSERVABILITY_DATABASE_URL',+schema: 'ow_observability'}`. The execution and recording connections may point to+the same server or different servers. Run the explicit Rust+`RecordingConfig::migrate_postgres().await` setup operation before startup.+Production connections verify TLS; `loopbackPlaintext` is only for local fixtures.++Turn-list cursors refer to execution events; timeline cursors refer to recording+rows. Follow `next_cursor` even if a bounded turn-list page contains no turns.+Timeline `through` aligns reads to a watermark, not a cross-request transaction.+A flush is a bounded barrier, not a guarantee that previous dropped records were+recovered. Check `health()` counters for drops, write failures and export failures.++## Export traces++```typescript+import {langfuseTelemetry} from '@openworkerai/harness-sdk';++const telemetry = langfuseTelemetry(+ 'https://your-langfuse.example/api/public/otel',+ 'LANGFUSE_AUTHORIZATION',+);+// Pass telemetry to Harness.start. The environment value is+// "Basic " + base64(publicKey + ":" + secretKey), stored only in the backend.+```++The helper selects Langfuse v4 trace acknowledgements and metadata-only capture.+For a standard collector, set `telemetry.endpoint` to its OTLP HTTP base URL and+use the default `otlp` acknowledgement mode. Do not install a second exporter over+the same event stream. Rust has the matching `TelemetryConfig::langfuse` helper.+See [storage and collector setup](../../../harness/observability/README.md).++Use [Langfuse's public API](https://langfuse.com/docs/api-and-data-platform/features/public-api)+for downstream analytics. Traces are best effort, not a billing ledger. Cached+input and reasoning tokens are subsets; never add them again to input/output.+Missing usage is unknown. Tool duration and result bytes do not imply exact token+cost per tool. Export failure must never trigger replay of a tool or agent turn.++## Inspect a turn and its children++Local SDK queries and `service.observability` share timeline/summary shapes. The+service checks access against execution storage before and after recording reads.+It authorizes every child independently; parent visibility grants no child access.++`children(query)` returns durable spawn links, including `child_turn_id` when the+first task was accepted. `subtreeSummary(query)` counts the root plus visible+spawned initial turns, bounded to 25 scopes. Cycles are deduplicated. Later inbox+turns remain separate. The response names the selected runs and any truncation;+concurrent tasks have no single wall elapsed duration.++The optional [trace viewer](../../../harness/trace-viewer/README.md) provides+Usage/Timeline views, model/tool groups, child navigation and basic score display.+Your app supplies an authorized browser adapter. Do not put SDK, database or+exporter credentials in the browser.++## Record evaluations++Enable `recording.evaluations: true`, then submit results from your app or evaluator:++```typescript+const recorded = await harness.observability.recordEvaluation(query, {+ id: 'quality-review-1', name: 'correctness',+ evaluator: {name: 'answer-check', version: '1', source: 'code'},+ status: 'completed', score: {type: 'boolean', value: true},+ createdMs: Date.now(), datasetCase: 'case-42', experiment: 'candidate-a',+}, modelSpanId);+```++Use a stable ID and original timestamp when retrying the same result. Identical+writes return the existing record; conflicts fail. Corrections append a new ID+with `supersedes` referencing the old result. `skipped` and `error` have no score.+Model evaluators may attach their own model/usage; it is not added to agent usage.+This feature records results; it does not run judges or incur model charges.++To deliver results automatically, configure `recording.evaluationSinks`:++```typescript+const evaluationSinks = [{+ name: 'langfuse', url: 'https://your-langfuse.example',+ authorizationEnv: 'LANGFUSE_AUTHORIZATION',+ includeExplanation: false,+}];+```++The separate durable queue retries scores with the same ID/name/timestamp, never+traces or original actions. Delivery depends on destination idempotency semantics;+unknown acknowledgements are not proof of failure. Lease/backoff survives restart.+A sink name binds to its configuration; use a new name for a different destination.+`maxPendingEvaluations` defaults to 10,000; delivered retained results do not use+pending capacity. `evaluationRetentionDays` defaults to 30 and pruning protects+undelivered rows. Keep the recording database to resume pending delivery. Shutdown+is bounded and may leave pending scores for the next host start. Scores without a+recorded trace target and skipped/error results remain local and are explicitly+counted as not exportable. Explanations are excluded remotely unless opted in.++For caller-controlled one-shot export, import `exportLangfuseScore` from+`@openworkerai/harness-sdk/evaluation-export`. It does not retry uncertain responses.+Choose automatic or caller-controlled delivery for each destination.++## Evidence and audits++Set `recording.evidence` only when content is needed: `userInput`, `assistantOutput`,+`toolArguments`, `toolResults`, and `maxBytes` (256–65,536). `evidence(query)` projects+selected committed events, applies best-effort secret redaction, and reports fields+omitted for size. It excludes failed-attempt deltas, hidden reasoning and blocked+output. Current-turn input is available; historical input is not reconstructed.+Execution event IDs are separate from recording cursors.++`evidenceJsonl` in the optional export subpath combines authorized timeline,+summary, evaluations and explicitly supplied content with a versioned manifest,+selection, watermark and checksum. It covers only supplied pages, not all history.+It includes no content unless the caller supplies it. Keep expected answers and+rubrics in your dataset keyed by `datasetCase`; never inject them into the tested+agent. Artifact revisions can be included through selected tool results or your+application's evidence records. Checksums do not establish completeness or+administrator-resistant integrity.++Use the included session/turn/event references and configuration digests to join+existing [compliance records and SIEM delivery](compliance.md). Compliance records+remain the authority for approvals and effects; traces and evaluations are not+proof of compliance. A metadata reader is not a content auditor. In server apps,+authorize evidence access separately before returning it, and keep database/host+APIs in the trusted backend. The service's existing compliance content permission+continues to control audit reads. Cluster experiment/review tooling comes later.diff --git a/harness/Cargo.lock b/harness/Cargo.lockindex 5a21cb04..355f3eee 100644--- a/harness/Cargo.lock+++ b/harness/Cargo.lock@@ -1882,10 +1882,11 @@ dependencies = [[[package]]name = "openworker-observability"-version = "0.1.0"+version = "0.1.0-beta.2"dependencies = ["anyhow","native-tls",+ "openworker-compliance","openworker-telemetry","postgres","postgres-native-tls",@@ -1940,8 +1941,10 @@ version = "0.1.0-beta.2"dependencies = ["anyhow","async-stream",+ "chrono","fs2","futures-util",+ "openworker-observability","openworker-runtime","openworker-security-adapters","openworker-store-postgres",diff --git a/harness/harness-sdk/Cargo.toml b/harness/harness-sdk/Cargo.tomlindex 252d857a..1c27fd64 100644--- a/harness/harness-sdk/Cargo.toml+++ b/harness/harness-sdk/Cargo.toml@@ -10,7 +10,12 @@ license.workspace = truepublish = ["crates-io"][features]-default = ["guardrails", "compliance"]+default = ["guardrails", "compliance", "otlp"]+otlp = ["openworker-telemetry/otlp"]+recording = ["dep:openworker-observability"]+evaluations = ["recording", "openworker-observability/evaluations"]+evaluation-export = ["evaluations", "reqwest/blocking", "dep:chrono"]+recording-postgres = ["recording", "openworker-observability/postgres"]guardrails = ["openworker-runtime/guardrails", "openworker-store-postgres/guardrails"]compliance = ["openworker-runtime/compliance", "openworker-store-postgres/compliance"]test-support = []@@ -22,6 +27,8 @@ name = "openworker-harness"path = "src/main.rs"[dependencies]+chrono = { version = "0.4", default-features = false, features = ["std"], optional = true }+openworker-observability = { path = "../observability", version = "=0.1.0-beta.2", optional = true }openworker-security-adapters = { version = "=0.1.0-beta.2", path = "../security-adapters", optional = true }reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }anyhow = "1"@@ -29,7 +36,7 @@ fs2 = "0.4"rusqlite = { version = "0.32", features = ["bundled"] }openworker-store-postgres = { version = "=0.1.0-beta.2", path = "../store-postgres" }openworker-runtime = { version = "=0.1.0-beta.2", path = "../runtime", default-features = false }-openworker-telemetry = { version = "=0.1.0-beta.2", path = "../telemetry", features = ["otlp"] }+openworker-telemetry = { version = "=0.1.0-beta.2", path = "../telemetry" }openworker-tools = { version = "=0.1.0-beta.2", path = "../tools-sdk" }serde = { version = "1", features = ["derive"] }serde_json = "1"diff --git a/harness/harness-sdk/README.md b/harness/harness-sdk/README.mdindex b4b73eaa..1e946cd9 100644--- a/harness/harness-sdk/README.md+++ b/harness/harness-sdk/README.md@@ -343,3 +343,14 @@ coordination engine without application-managed team setup. This requires amatching feature runtime; it is not in published runtime 0.1.0-beta.2.See [standalone sub-agents](../../docs/harness/guides/sessions.md#standalone-sub-agents)for limits, approvals and shared-sandbox behavior.+++## Optional turn traces and evaluation evidence++The source candidate includes opt-in recording, per-turn timeline/grouped queries,+child drill-down and durable evaluation results. Recording and evaluation delivery+are disabled until configured; browser UI is a separate optional component.+See the [observability guide](../../docs/harness/guides/observability.md) for local+SQLite/PostgreSQL, Langfuse export, content selection and audit references. These+new APIs require matching local candidate builds and are not in the published+0.1.0-beta.2 packages yet.diff --git a/harness/harness-sdk/src/agents/harness.rs b/harness/harness-sdk/src/agents/harness.rsindex 6a9e1b7f..3ddf2b43 100644--- a/harness/harness-sdk/src/agents/harness.rs+++ b/harness/harness-sdk/src/agents/harness.rs@@ -20,6 +20,7 @@ pub struct HarnessBuilder {sandbox: Sandbox,scripted: bool,telemetry: Option<TelemetryConfig>,+ recording: Option<crate::RecordingConfig>,options: ClientOptions,}@@ -129,7 +130,8 @@ impl HarnessBuilder {/// TelemetryHeader::literal("x-langfuse-ingestion-version", "4"),/// ],/// signals: vec![TelemetrySignal::Traces],- /// content: ContentPolicy { capture: Capture::Full, ..ContentPolicy::default() },+ /// acknowledgement: TelemetryAcknowledgement::LangfuseV4,+ /// content: ContentPolicy { capture: Capture::Metadata, ..ContentPolicy::default() },/// ..TelemetryConfig::default()/// });/// # }@@ -138,6 +140,11 @@ impl HarnessBuilder {self.telemetry = Some(config);self}+ /// Record diagnostics independently of execution storage and external export.+ pub fn recording(mut self, config: crate::RecordingConfig) -> Self {+ self.recording = Some(config);+ self+ }pub fn options(mut self, options: ClientOptions) -> Self {self.options = options;self@@ -175,6 +182,7 @@ impl HarnessBuilder {sandbox: self.sandbox,scripted: self.scripted,telemetry: self.telemetry.clone(),+ recording: self.recording.clone(),agents: self.agents.clone(),})}@@ -310,6 +318,7 @@ impl Harness {sandbox: Sandbox::default(),scripted: false,telemetry: None,+ recording: None,options: ClientOptions::default(),}}@@ -338,6 +347,9 @@ impl Harness {})}/// The underlying protocol client, for anything this layer does not wrap.+ pub fn observability(&self) -> crate::recording::Observability {+ crate::recording::Observability::new(self.client.clone())+ }pub fn client(&self) -> &HarnessClient {&self.client}diff --git a/harness/harness-sdk/src/agents/mod.rs b/harness/harness-sdk/src/agents/mod.rsindex 37709d7a..9557f942 100644--- a/harness/harness-sdk/src/agents/mod.rs+++ b/harness/harness-sdk/src/agents/mod.rs@@ -51,8 +51,8 @@ mod team;pub use crate::{AccessToken, ApprovalDecision, Capture, ContentPolicy, Credentials, Model, ModelChoice,- ModelProvider, ProviderKind, Sandbox, Storage, TelemetryConfig, TelemetryHeader,- TelemetrySignal, TokenRefreshReason, ToolRegistry,+ ModelProvider, ProviderKind, Sandbox, Storage, TelemetryAcknowledgement, TelemetryConfig,+ TelemetryHeader, TelemetrySignal, TokenRefreshReason, ToolRegistry,};pub use agent_team::{AgentTeam, AgentTeamRef, ChildContext, ChildOptions, ChildWorkspace, Communication,diff --git a/harness/harness-sdk/src/evaluation_export.rs b/harness/harness-sdk/src/evaluation_export.rsnew file mode 100644index 00000000..17596327--- /dev/null+++ b/harness/harness-sdk/src/evaluation_export.rs@@ -0,0 +1,197 @@+//! Optional durable score delivery using the compliance checkpoint contract.+//! An export retry never submits a turn or executes a tool.+use crate::recording::{EvaluationSink, RecordingConfig};+use anyhow::{ensure, Context, Result};+use openworker_observability::evaluations::{Evaluation, Score, Status};+use serde_json::{json, Value};+use std::{+ io::Read,+ sync::{+ atomic::{AtomicBool, AtomicU64, Ordering},+ mpsc, Arc, Mutex,+ },+ time::Duration,+};+#[derive(Default)]+struct Counters {+ sent: AtomicU64,+ skipped: AtomicU64,+ failed: AtomicU64,+}+pub(crate) struct Controller {+ stop: Arc<AtomicBool>,+ wake: mpsc::SyncSender<()>,+ done: Mutex<mpsc::Receiver<()>>,+ counters: Arc<Counters>,+}+impl Controller {+ pub fn notify(&self) {+ let _ = self.wake.try_send(());+ }+ pub fn health(&self) -> Value {+ json!({"scope":"sdk_host","sent":self.counters.sent.load(Ordering::Relaxed),"skipped_not_exportable":self.counters.skipped.load(Ordering::Relaxed),"failed_or_uncertain":self.counters.failed.load(Ordering::Relaxed)})+ }+ pub fn shutdown(&self) {+ self.stop.store(true, Ordering::Release);+ self.notify();+ let _ = self+ .done+ .lock()+ .unwrap()+ .recv_timeout(Duration::from_secs(5));+ }+}+impl Drop for Controller {+ fn drop(&mut self) {+ self.stop.store(true, Ordering::Release);+ self.notify();+ }+}+fn now() -> u64 {+ openworker_runtime::store::now_ms()+}+pub(crate) fn start(options: RecordingConfig) -> Result<Option<Arc<Controller>>> {+ if options.evaluation_sinks.is_empty() {+ return Ok(None);+ }+ for sink in &options.evaluation_sinks {+ sink.validate()?;+ ensure!(+ std::env::var(&sink.authorization_env).is_ok(),+ "Evaluation export credential unavailable"+ );+ }+ let mut store = options.open()?;+ for sink in &options.evaluation_sinks {+ let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());+ store.register_evaluation_sink(&options.deployment, &sink.name, &binding)?;+ }+ drop(store);+ let (wake, rx) = mpsc::sync_channel(1);+ let (done, wait) = mpsc::sync_channel(1);+ let stop = Arc::new(AtomicBool::new(false));+ let stopped = stop.clone();+ let counters = Arc::new(Counters::default());+ let stats = counters.clone();+ std::thread::Builder::new()+ .name("ow-evaluation-export".into())+ .spawn(move || {+ let client = reqwest::blocking::Client::builder()+ .no_proxy()+ .redirect(reqwest::redirect::Policy::none())+ .timeout(Duration::from_secs(3))+ .build();+ if let Ok(client) = client {+ while !stopped.load(Ordering::Acquire) {+ for sink in &options.evaluation_sinks {+ if stopped.load(Ordering::Acquire) {+ break;+ }+ if deliver_next(&options, sink, &client, &stats).is_err() {+ stats.failed.fetch_add(1, Ordering::Relaxed);+ }+ }+ let _ = rx.recv_timeout(Duration::from_secs(1));+ }+ } else {+ stats.failed.fetch_add(1, Ordering::Relaxed);+ }+ let _ = done.send(());+ })?;+ Ok(Some(Arc::new(Controller {+ stop,+ wake,+ done: Mutex::new(wait),+ counters,+ })))+}+fn deliver_next(+ options: &RecordingConfig,+ sink: &EvaluationSink,+ client: &reqwest::blocking::Client,+ stats: &Counters,+) -> Result<()> {+ let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());+ let token = openworker_runtime::id("eval");+ let mut store = options.open()?;+ let Some(delivery) =+ store.claim_evaluation_export(&options.deployment, &sink.name, &binding, &token, now())?+ else {+ return Ok(());+ };+ let value = &delivery.row.evaluation;+ // Non-quality outcomes stay available locally without fabricating a score.+ let result = if value.result.status != Status::Completed || value.target.trace_id.is_none() {+ stats.skipped.fetch_add(1, Ordering::Relaxed);+ Ok(())+ } else {+ send(client, sink, value).map(|_| {+ stats.sent.fetch_add(1, Ordering::Relaxed);+ })+ };+ // Stable score ID/name/timestamp survive lease expiry, restart and lost ACK.+ // This checkpoint advances only for an acknowledged score or explicit skip.+ store.settle_evaluation_export(+ &options.deployment,+ &sink.name,+ &token,+ now(),+ result.is_ok(),+ )?;+ result+}+pub(crate) fn payload(e: &Evaluation, explanation: bool) -> Result<Value> {+ e.validate()?;+ ensure!(e.result.status == Status::Completed, "Not a quality score");+ let trace = e.target.trace_id.as_ref().context("Score trace missing")?;+ let (kind, value) = match e.result.score.as_ref().context("Score missing")? {+ Score::Numeric(v) => ("NUMERIC", json!(v)),+ Score::Boolean(v) => ("BOOLEAN", json!(u8::from(*v))),+ Score::Categorical(v) => ("CATEGORICAL", json!(v)),+ Score::Text(v) => ("TEXT", json!(v)),+ };+ let id =+ openworker_runtime::bundle::digest(format!("{}\0{}", e.deployment, e.result.id).as_bytes());+ let timestamp = chrono::DateTime::from_timestamp_millis(e.result.created_ms as i64)+ .context("Invalid score time")?+ .to_rfc3339_opts(chrono::SecondsFormat::Millis, true);+ let mut body = json!({"id":id,"name":e.result.name,"traceId":trace,"dataType":kind,"value":value,"timestamp":timestamp,"metadata":{"openworker.evaluation.id":e.result.id,"evaluator":e.result.evaluator.name,"evaluatorVersion":e.result.evaluator.version,"evaluatorSource":e.result.evaluator.source,"evaluatorModel":e.result.evaluator.model,"evaluatorUsage":e.result.evaluator.usage,"supersedes":e.result.supersedes}});+ if let Some(span) = &e.target.span_id {+ body["observationId"] = json!(span);+ }+ if explanation {+ if let Some(comment) = &e.result.explanation {+ body["comment"] = json!(comment);+ }+ }+ Ok(body)+}+fn send(client: &reqwest::blocking::Client, sink: &EvaluationSink, e: &Evaluation) -> Result<()> {+ let body = payload(e, sink.include_explanation)?;+ let authorization = zeroize::Zeroizing::new(+ std::env::var(&sink.authorization_env).context("Evaluation credential unavailable")?,+ );+ let response = client+ .post(format!(+ "{}/api/public/scores",+ sink.url.trim_end_matches('/')+ ))+ .header("Authorization", authorization.as_str())+ .json(&body)+ .send()+ .map_err(|_| anyhow::anyhow!("Evaluation delivery outcome unknown"))?;+ ensure!(+ response.status().as_u16() == 200,+ "Evaluation delivery not acknowledged"+ );+ let mut bytes = Vec::new();+ response.take(65537).read_to_end(&mut bytes)?;+ ensure!(bytes.len() <= 65536, "Evaluation acknowledgement too large");+ let ack: Value = serde_json::from_slice(&bytes)+ .map_err(|_| anyhow::anyhow!("Invalid evaluation acknowledgement"))?;+ ensure!(+ ack["id"] == body["id"],+ "Evaluation acknowledgement identity mismatch"+ );+ Ok(())+}diff --git a/harness/harness-sdk/src/host.rs b/harness/harness-sdk/src/host.rsindex 3a7a1f37..e2f5283e 100644--- a/harness/harness-sdk/src/host.rs+++ b/harness/harness-sdk/src/host.rs@@ -20,10 +20,18 @@ use tokio::sync::broadcast;use tokio_util::sync::CancellationToken;#[path = "host_catalog.rs"]mod catalog;+#[cfg(feature = "evaluations")]+#[path = "host_evaluations.rs"]+mod evaluations;+#[cfg(feature = "recording")]+#[path = "host_evidence.rs"]+mod evidence;#[path = "host_named_lifecycle.rs"]mod named_lifecycle;#[path = "host_named_types.rs"]mod named_types;+#[path = "host_observability.rs"]+mod observability;#[path = "host_runs.rs"]mod runs;@@ -106,6 +114,8 @@ pub struct CoworkerConfig {#[derive(Clone, Serialize, Deserialize)]#[serde(deny_unknown_fields)]pub struct HostConfig {+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub recording: Option<crate::RecordingConfig>,#[serde(default)]pub desktop: crate::DesktopOptions,pub state_dir: PathBuf,@@ -119,7 +129,7 @@ pub struct HostConfig {/// Opt-in OTLP/HTTP export (content-free spans, metrics and logs). Point/// it at an OpenTelemetry Collector that forwards to your backend.#[serde(default, skip_serializing_if = "Option::is_none")]- pub telemetry: Option<openworker_telemetry::otlp::Config>,+ pub telemetry: Option<openworker_telemetry::config::Config>,/// Agent definitions compiled by the host into coworker bundles under/// `state_dir/agents` (see `agents::Agent` for the JSON shape). May be/// used instead of, or alongside, hand-written `coworkers`.@@ -133,6 +143,12 @@ pub struct HostProcessConfig {pub restart: RestartPolicy,}pub(crate) struct Host {+ #[cfg(feature = "evaluation-export")]+ evaluation_exporter: Option<Arc<crate::evaluation_export::Controller>>,+ #[cfg(feature = "recording")]+ recording: Option<Arc<crate::instrumentation::Recording>>,+ #[cfg(feature = "recording")]+ recording_readers: Arc<tokio::sync::Semaphore>,pub events: broadcast::Sender<openworker_runtime::store::Event>,pub gaps: broadcast::Sender<()>,pub failed: CancellationToken,@@ -164,6 +180,50 @@ impl HostConfig {!self.scripted || cfg!(feature = "test-support"),"Scripted models require a development SDK build with test-support");+ ensure!(+ self.telemetry.is_none() || cfg!(feature = "otlp"),+ "Telemetry exporter not included; enable the otlp feature"+ );+ if let Some(recording) = &self.recording {+ ensure!(+ cfg!(feature = "recording"),+ "Recording not included; enable the recording feature"+ );+ recording.validate()?;+ ensure!(+ recording.evaluation_sinks.is_empty()+ || (recording.evaluations && cfg!(feature = "evaluation-export")),+ "Evaluation export requires evaluations and the evaluation-export feature"+ );+ ensure!(+ !recording.evaluations || cfg!(feature = "evaluations"),+ "Evaluation support not included"+ );+ if let crate::RecordingDatabase::Postgres { schema, .. } = &recording.database {+ ensure!(+ cfg!(feature = "recording-postgres"),+ "PostgreSQL recording not included"+ );+ if let Storage::Postgres {+ schema: execution, ..+ } = &self.storage+ {+ ensure!(+ schema != execution,+ "Recording schema must differ from execution schema"+ );+ }+ }+ if let crate::RecordingDatabase::Sqlite { path } = &recording.database {+ let execution = self+ .desktop+ .session_dir+ .as_ref()+ .unwrap_or(&self.state_dir)+ .join("runtime.sqlite3");+ ensure!(path != &execution, "Recording must use a separate database");+ }+ }self.desktop.compaction.validate()?;self.desktop.safety.validate()?;ensure!(@@ -410,16 +470,13 @@ impl Host {let store = Arc::new(Mutex::new(storage));// The exporter builds a blocking HTTP client, which must not happen on// a Tokio worker; a short-lived plain thread works on any runtime flavor.- let (telemetry, controller) = match config.telemetry.clone() {- Some(c) => {- let (t, ctl) = std::thread::spawn(move || c.start("sdk-host"))- .join()- .map_err(|_| anyhow::anyhow!("Telemetry startup panicked"))?- .context("Telemetry configuration failed; check the explicit endpoint")?;- (t, Some(ctl))- }- None => (openworker_telemetry::Telemetry::default(), None),- };+ let instrumentation_config = config.clone();+ let started =+ std::thread::spawn(move || crate::instrumentation::start(&instrumentation_config))+ .join()+ .map_err(|_| anyhow::anyhow!("Instrumentation startup panicked"))??;+ let telemetry = started.telemetry;+ let controller = started.controller;let mut profiles = BTreeMap::new();for (c, registry, bundle) in prepared {let mut rt = Runtime::open_instrumented(@@ -461,6 +518,12 @@ impl Host {let (events, _) = broadcast::channel(256);let (gaps, _) = broadcast::channel(16);let host = Arc::new(Self {+ #[cfg(feature = "evaluation-export")]+ evaluation_exporter: started.evaluation_exporter,+ #[cfg(feature = "recording")]+ recording: started.recording,+ #[cfg(feature = "recording")]+ recording_readers: Arc::new(tokio::sync::Semaphore::new(2)),events,gaps,failed: CancellationToken::new(),@@ -1028,6 +1091,9 @@ impl Host {}return Ok(json!({"registered":true}));}+ if method.starts_with("observability/") {+ return self.observation_dispatch(method, p).await;+ }if method == "agent/register" {return self.register_agent(p).await;}@@ -1052,6 +1118,17 @@ impl Host {json!(cfg!(feature = "security-adapters")),);c.insert("compliance".into(), json!(cfg!(feature = "compliance")));+ c.insert(+ "evaluation_export".into(),+ json!(cfg!(feature = "evaluation-export")),+ );+ c.insert("evaluations".into(), json!(cfg!(feature = "evaluations")));+ c.insert("recording".into(), json!(cfg!(feature = "recording")));+ c.insert(+ "recording_postgres".into(),+ json!(cfg!(feature = "recording-postgres")),+ );+ c.insert("telemetry_export".into(), json!(cfg!(feature = "otlp")));c.insert("turn_limits".into(), json!(true));c.insert("turn_context".into(), json!(true));c.insert("initial_history".into(), json!(true));@@ -1302,6 +1379,19 @@ impl Host {);}}+ if let Some(crate::RecordingConfig {+ database: crate::RecordingDatabase::Sqlite { path },+ ..+ }) = &self.config.recording+ {+ let recording = path+ .canonicalize()+ .context("Recording database unavailable")?;+ ensure!(+ !recording.starts_with(&workspace),+ "Workspace must not contain the private recording database"+ );+ }if let Some(required) = &self.config.desktop.compliance {// Application requirements cannot be disabled or weakened by// a session option. Retain the application retention policy.@@ -1353,6 +1443,10 @@ impl Host {db.release(&s.id)?;}}+ #[cfg(feature = "evaluation-export")]+ if let Some(exporter) = self.evaluation_exporter.clone() {+ let _ = tokio::task::spawn_blocking(move || exporter.shutdown()).await;+ }// Bounded flush (at most 5 s); export problems never fail shutdown.let controller = self.telemetry.lock().unwrap().take();if let Some(controller) = controller {@@ -1360,6 +1454,16 @@ impl Host {tokio::task::spawn_blocking(move || controller.shutdown(Duration::from_secs(5))).await;}+ #[cfg(feature = "recording")]+ if let Some(recording) = &self.recording {+ let controller = recording.controller.lock().unwrap().take();+ if let Some(controller) = controller {+ let _ = tokio::task::spawn_blocking(move || {+ controller.shutdown(Duration::from_secs(5))+ })+ .await;+ }+ }Ok(())}}diff --git a/harness/harness-sdk/src/host_evaluations.rs b/harness/harness-sdk/src/host_evaluations.rsnew file mode 100644index 00000000..947d2a19--- /dev/null+++ b/harness/harness-sdk/src/host_evaluations.rs@@ -0,0 +1,103 @@+//! Acknowledged evaluation writes. No action or model call is replayed on failure.+use super::Host;+use anyhow::{ensure, Context, Result};+use openworker_observability::evaluations::{Evaluation, Input, Query, Target};+use serde::Deserialize;+use serde_json::{json, Value};+#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct Record {+ session_id: String,+ turn_id: String,+ result: Input,+ span_id: Option<String>,+}+impl Host {+ pub(super) async fn evaluation_dispatch(&self, method: &str, p: Value) -> Result<Value> {+ let recorder = self.recording.clone().context("Recording disabled")?;+ ensure!(recorder.options.evaluations, "Evaluation storage disabled");+ if method == "observability/evaluation-record" {+ let q: Record = serde_json::from_value(p)?;+ crate::TraceQuery::turn(&q.session_id, &q.turn_id).validate()?;+ ensure!(+ self.store+ .lock()+ .unwrap()+ .has_turn(&q.session_id, &q.turn_id)?,+ "Evaluation target turn unavailable"+ );+ let session_hash = openworker_runtime::bundle::digest(q.session_id.as_bytes());+ let turn_hash = openworker_runtime::bundle::digest(q.turn_id.as_bytes());+ let permit = self+ .recording_readers+ .clone()+ .try_acquire_owned()+ .context("Recording readers busy")?;+ let value=tokio::task::spawn_blocking(move || -> Result<Value> {+ let _permit=permit;+ let mut store=recorder.options.open()?;+ // Trace retention is independent from result retention. A valid+ // duplicate must not need its old span to remain in raw storage.+ if let Some(old)=store.evaluation(&recorder.options.deployment,&q.result.id)? {+ ensure!(old.target.session_hash==session_hash&&old.target.turn_hash==turn_hash&&old.target.span_id==q.span_id,"Evaluation identity conflict");+ let candidate=Evaluation{result:q.result.clone(),..old.clone()}.redacted();candidate.validate()?;+ ensure!(candidate==old,"Evaluation identity conflict");+ return Ok(json!({"result":old,"duplicate":true}));+ }+ let trace_id=if let Some(span)=&q.span_id {+ let step=store.get(&recorder.options.deployment,span)?.context("Evaluation span unavailable")?;+ ensure!(step.session_hash.as_deref()==Some(&session_hash)&&step.turn_hash.as_deref()==Some(&turn_hash),"Evaluation span scope mismatch");Some(step.trace_id)+ } else {None};+ let e=Evaluation{version:1,deployment:recorder.options.deployment.clone(),target:Target{session_hash,turn_hash,trace_id,span_id:q.span_id},result:q.result}.redacted();+ e.validate()?;+ let cutoff=openworker_runtime::store::now_ms().saturating_sub(u64::from(recorder.options.evaluation_retention_days)*86_400_000);+ store.prune_evaluations(&recorder.options.deployment,cutoff,1000)?;+ let result=store.record_evaluation_bounded(&e,recorder.options.max_pending_evaluations)?;+ Ok(json!({"result":e,"duplicate":result==openworker_observability::Write::Duplicate}))+ }).await??;+ #[cfg(feature = "evaluation-export")]+ if let Some(exporter) = &self.evaluation_exporter {+ exporter.notify();+ }+ return Ok(value);+ }+ if method == "observability/evaluation-list" {+ let q: crate::TraceQuery = serde_json::from_value(p)?;+ q.validate()?;+ ensure!(q.limit <= 50, "Evaluation page limit is 50");+ ensure!(+ self.store+ .lock()+ .unwrap()+ .has_turn(&q.session_id, &q.turn_id)?,+ "Evaluation target turn unavailable"+ );+ let query = Query {+ deployment: recorder.options.deployment.clone(),+ session_hash: openworker_runtime::bundle::digest(q.session_id.as_bytes()),+ turn_hash: openworker_runtime::bundle::digest(q.turn_id.as_bytes()),+ after: q.after,+ limit: q.limit,+ };+ let permit = self+ .recording_readers+ .clone()+ .try_acquire_owned()+ .context("Recording readers busy")?;+ let value = tokio::task::spawn_blocking(move || -> Result<Value> {+ let _permit = permit;+ Ok(json!(recorder.options.open()?.evaluations(&query)?))+ })+ .await??;+ ensure!(+ self.store+ .lock()+ .unwrap()+ .has_turn(&q.session_id, &q.turn_id)?,+ "Evaluation target turn unavailable"+ );+ return Ok(value);+ }+ anyhow::bail!("Unknown evaluation operation")+ }+}diff --git a/harness/harness-sdk/src/host_evidence.rs b/harness/harness-sdk/src/host_evidence.rsnew file mode 100644index 00000000..f108797a--- /dev/null+++ b/harness/harness-sdk/src/host_evidence.rs@@ -0,0 +1,162 @@+//! Opt-in, bounded projection of committed execution events. No attempt deltas,+//! provider-private reasoning, rejected candidates or unselected tool payloads.+use super::Host;+use anyhow::{ensure, Context, Result};+use serde::Deserialize;+use serde_json::{json, Value};+#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct Query {+ session_id: String,+ turn_id: String,+ #[serde(default)]+ after_event_id: i64,+}+fn redact(v: Value) -> Value {+ match v {+ Value::String(s) => json!(openworker_telemetry::redact(&s)),+ Value::Array(a) => Value::Array(a.into_iter().map(redact).collect()),+ Value::Object(o) => Value::Object(o.into_iter().map(|(k, v)| (k, redact(v))).collect()),+ v => v,+ }+}++fn project(kind: &str, data: &Value, policy: &crate::recording::EvidencePolicy) -> Option<Value> {+ match kind {+ "assistant_message" if policy.assistant_output => Some(+ json!({"messages":data["messages"].as_array().into_iter().flatten().map(|m|json!({"role":"assistant","content":m["content"].as_array().into_iter().flatten().filter(|p|p["type"]=="output_text").map(|p|json!({"type":"output_text","text":p["text"]})).collect::<Vec<_>>()})).collect::<Vec<_>>()}),+ ),+ "tool_started" if policy.tool_arguments => Some(+ json!({"name":data["name"],"call_id":data["call_id"],"arguments":data["arguments"]}),+ ),+ "tool_finished"+ if policy.tool_results+ && data["outputBlocked"] != true+ && data["result"]["outputBlocked"] != true =>+ {+ Some(+ json!({"name":data["name"],"call_id":data["call_id"],"result":data["result"],"executed":data["executed"]}),+ )+ }+ "turn_completed" | "turn_recovered" | "turn_failed" | "turn_cancelled"+ | "turn_interrupted" => Some(+ json!({"status":data["status"],"outcome_unknown_count":data["outcome_unknown_count"],"automatic_replay":data["automatic_replay"]}),+ ),+ _ => None,+ }+}+impl Host {+ pub(super) fn evidence_dispatch(&self, p: Value) -> Result<Value> {+ let q: Query = serde_json::from_value(p)?;+ crate::TraceQuery::turn(&q.session_id, &q.turn_id).validate()?;+ ensure!(q.after_event_id >= 0, "Invalid evidence cursor");+ let policy = self+ .config+ .recording+ .as_ref()+ .and_then(|r| r.evidence.as_ref())+ .context("Content evidence is disabled")?;+ let store = self.store.lock().unwrap();+ ensure!(+ store.has_turn(&q.session_id, &q.turn_id)?,+ "Evidence target turn unavailable"+ );+ let session = store.metadata(&q.session_id)?;+ let events = store.events(&q.session_id, q.after_event_id)?;+ let through = store.cursor(&q.session_id)?;+ let current = session.turn.as_ref().filter(|t| t.id == q.turn_id);+ let mut bytes = 0;+ let mut omitted = 0;+ let mut result = Vec::new();+ let input = if q.after_event_id == 0 && policy.user_input {+ current.map(|t| openworker_telemetry::redact(&t.text))+ } else {+ None+ };+ let input = input.filter(|text| {+ if text.len() <= policy.max_bytes {+ bytes += text.len();+ true+ } else {+ omitted += 1;+ false+ }+ });+ let mut after = q.after_event_id;+ for e in events {+ after = e.event_id;+ if e.turn_id.as_deref() != Some(&q.turn_id) {+ continue;+ }+ let payload = project(&e.kind, &e.data, policy);+ if let Some(payload) = payload {+ let projected = json!({"event_id":e.event_id,"kind":e.kind,"data":redact(payload)});+ let size = serde_json::to_vec(&projected)?.len();+ if bytes + size > policy.max_bytes {+ omitted += 1;+ continue;+ }+ bytes += size;+ result.push(projected);+ }+ }+ Ok(+ json!({"version":1,"session_id":q.session_id,"turn_id":q.turn_id,"turn_status":current.map(|t|t.status.as_str()),"configuration":{"agent":session.coworker,"model":session.model,"bundle_digest":session.bundle_digest,"tool_schema_digest":session.schema_digest},"audit_reference":{"session_id":session.id,"turn_id":q.turn_id,"execution_event_through":through,"capture_enabled":session.compliance.is_some()},"input":input,"events":result,"next_event_id":(after<through).then_some(after),"through_event_id":through,"coverage":{"source":"committed_execution_events","input_available":current.is_some()&&policy.user_input,"input_scope":"current_turn_only","omitted_for_size":omitted,"redaction":"best_effort_secret_patterns","captured_fields":policy,"not_an_audit_archive":true}}),+ )+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ fn policy() -> crate::recording::EvidencePolicy {+ crate::recording::EvidencePolicy {+ user_input: true,+ assistant_output: true,+ tool_arguments: true,+ tool_results: true,+ max_bytes: 1024,+ }+ }+ #[test]+ fn evidence_keeps_accepted_text_but_not_reasoning_or_attempt_deltas() {+ let data = json!({"messages":[{"role":"assistant","content":[{"type":"output_text","text":"accepted"},{"type":"reasoning","text":"private"}]}],"provider_private":"hidden"});+ let result = project("assistant_message", &data, &policy())+ .unwrap()+ .to_string();+ assert!(result.contains("accepted"));+ assert!(!result.contains("private"));+ assert!(!result.contains("hidden"));+ assert!(project("assistant_delta", &data, &policy()).is_none());+ assert!(project("reasoning_delta", &data, &policy()).is_none());+ }+ #[test]+ fn evidence_excludes_both_blocked_result_shapes_and_preserves_uncertainty() {+ for data in [+ json!({"outputBlocked":true,"result":{"text":"candidate"}}),+ json!({"result":{"outputBlocked":true,"text":"candidate"}}),+ ] {+ assert!(project("tool_finished", &data, &policy()).is_none());+ }+ let value = project(+ "tool_finished",+ &json!({"call_id":"c","executed":true,"result":{"outcome":"unknown"}}),+ &policy(),+ )+ .unwrap();+ assert_eq!(value["result"]["outcome"], "unknown");+ assert_eq!(value["executed"], true);+ let recovery = project(+ "turn_recovered",+ &json!({"status":"interrupted","outcome_unknown_count":2,"automatic_replay":false}),+ &policy(),+ )+ .unwrap();+ assert_eq!(recovery["outcome_unknown_count"], 2);+ assert_eq!(recovery["automatic_replay"], false);++ let mut disabled = policy();+ disabled.tool_results = false;+ assert!(project("tool_finished", &value, &disabled).is_none());+ }+}diff --git a/harness/harness-sdk/src/host_observability.rs b/harness/harness-sdk/src/host_observability.rsnew file mode 100644index 00000000..ecb4ebf2--- /dev/null+++ b/harness/harness-sdk/src/host_observability.rs@@ -0,0 +1,274 @@+//! Trusted host query boundary. No model tool can call these SDK methods.+use super::Host;+use anyhow::{ensure, Result};+use serde::Deserialize;+use serde_json::{json, Value};++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct Turns {+ session_id: String,+ #[serde(default)]+ after: i64,+ #[serde(default = "limit")]+ limit: usize,+}+fn limit() -> usize {+ 25+}+impl Host {+ pub(super) async fn observation_dispatch(&self, method: &str, p: Value) -> Result<Value> {+ ensure!(!self.stop.is_cancelled(), "Host stopped");+ if method == "observability/health" {+ ensure!(+ p.as_object().is_some_and(|o| o.is_empty()),+ "Health query accepts no parameters"+ );+ let recording: Option<Value> = None;+ #[cfg(feature = "recording")]+ let recording = self+ .recording+ .as_ref()+ .map(|r| json!(r.statistics.snapshot()))+ .or(recording);+ let evaluation_export: Option<Value> = None;+ #[cfg(feature = "evaluation-export")]+ let evaluation_export = self+ .evaluation_exporter+ .as_ref()+ .map(|e| e.health())+ .or(evaluation_export);+ return Ok(+ json!({"evaluation_export":evaluation_export,"scope":"sdk_host","recording_enabled":self.config.recording.is_some(),"recording":recording,"telemetry":self.instrumentation.snapshot(),"exporter_enabled":self.config.telemetry.is_some()}),+ );+ }+ if method == "observability/flush" {+ #[derive(Deserialize)]+ #[serde(deny_unknown_fields)]+ struct Flush {+ timeout_ms: u64,+ }+ let q: Flush = serde_json::from_value(p)?;+ ensure!(+ (1..=5000).contains(&q.timeout_ms),+ "Flush budget must be 1..5000ms"+ );+ let telemetry = self.instrumentation.clone();+ #[cfg(feature = "recording")]+ let recording = self.recording.clone();+ return tokio::task::spawn_blocking(move || {+ let until=std::time::Instant::now()+std::time::Duration::from_millis(q.timeout_ms);+ let exported=telemetry.flush(std::time::Duration::from_millis(q.timeout_ms));+ let drained=true;+ #[cfg(feature="recording")] let drained=if let Some(r)=recording {+ while r.statistics.snapshot().pending>0 && std::time::Instant::now()<until { std::thread::sleep(std::time::Duration::from_millis(10)); }+ r.statistics.snapshot().pending==0+ } else {drained};+ let _=until;+ Ok(json!({"telemetry_barrier":exported,"recording_drained":drained,"guarantee":"bounded_best_effort","evaluation_delivery":"independent_durable_queue"}))+ }).await?;+ }+ ensure!(self.config.recording.is_some(), "Recording is disabled");+ if method == "observability/turns" {+ let q: Turns = serde_json::from_value(p)?;+ ensure!(+ (1..=100).contains(&q.limit) && q.after >= 0,+ "Invalid turn page"+ );+ let store = self.store.lock().unwrap();+ let session = store.metadata(&q.session_id)?;+ // Existing journal reads are bounded at 500 events / 512 KiB. An+ // empty page can still have a next cursor; do not scan unbounded history.+ let events = store.events(&q.session_id, q.after)?;+ let end = store.cursor(&q.session_id)?;+ let mut after = q.after;+ let mut turns = Vec::new();+ for e in events {+ after = e.event_id;+ if e.kind == "turn_accepted" {+ if let Some(id) = e.turn_id {+ let current = session.turn.as_ref().filter(|t| t.id == id);+ turns.push(json!({"session_id":session.id,"turn_id":id,"status":current.map(|t| &t.status),"agent":session.coworker,"name":session.name}));+ if turns.len() == q.limit {+ break;+ }+ }+ }+ }+ return Ok(+ json!({"turns":turns,"next_cursor":(after < end).then_some(after),"through":end}),+ );+ }+ #[cfg(feature = "evaluations")]+ if method.starts_with("observability/evaluation-") {+ return self.evaluation_dispatch(method, p).await;+ }+ #[cfg(feature = "recording")]+ if method == "observability/evidence" {+ return self.evidence_dispatch(p);+ }+ if method == "observability/children" {+ let q: crate::TraceQuery = serde_json::from_value(p)?;+ q.validate()?;+ let children = self+ .store+ .lock()+ .unwrap()+ .observation_children(&q.session_id, &q.turn_id)?;+ return Ok(json!({"children":children,"coverage":"captured_spawn_origins_only"}));+ }+ #[cfg(feature = "recording")]+ if method == "observability/subtree-summary" {+ use openworker_observability::{+ query::{Query, Scope, MAX_SUMMARY_STEPS},+ summary,+ };+ use openworker_runtime::observation_links::{subtree, Run};+ let q: crate::TraceQuery = serde_json::from_value(p)?;+ q.validate()?;+ ensure!(+ q.after == 0,+ "Subtree summary cannot start at a page offset"+ );+ let selection = {+ let store = self.store.lock().unwrap();+ ensure!(+ store.has_turn(&q.session_id, &q.turn_id)?,+ "Trace target turn unavailable"+ );+ subtree(+ Run {+ session_id: q.session_id.clone(),+ turn_id: q.turn_id.clone(),+ },+ |r| store.observation_children(&r.session_id, &r.turn_id),+ )?+ };+ let recording = self+ .recording+ .clone()+ .ok_or_else(|| anyhow::anyhow!("Recording unavailable"))?;+ let read = Query {+ deployment: recording.options.deployment.clone(),+ scopes: selection+ .runs+ .iter()+ .map(|r| Scope {+ session_hash: openworker_runtime::bundle::digest(r.session_id.as_bytes()),+ turn_hash: openworker_runtime::bundle::digest(r.turn_id.as_bytes()),+ })+ .collect(),+ after: 0,+ through: q.through,+ limit: MAX_SUMMARY_STEPS,+ };+ let permit = self+ .recording_readers+ .clone()+ .try_acquire_owned()+ .map_err(|_| anyhow::anyhow!("Recording readers busy"))?;+ let page = tokio::task::spawn_blocking(move || {+ let _permit = permit;+ recording.options.open()?.timeline(&read)+ })+ .await??;+ for run in &selection.runs {+ ensure!(+ self.store+ .lock()+ .unwrap()+ .has_turn(&run.session_id, &run.turn_id)?,+ "Trace target unavailable"+ );+ }+ return Ok(+ json!({"selection":selection,"summary":summary::build(&page,summary::Canonical::default())}),+ );+ }+ #[cfg(feature = "recording")]+ if matches!(method, "observability/timeline" | "observability/summary") {+ use openworker_observability::{+ query::{Query, Scope, MAX_SUMMARY_STEPS},+ summary::{self, Canonical},+ };+ let q: crate::TraceQuery = serde_json::from_value(p)?;+ q.validate()?;+ ensure!(+ self.store+ .lock()+ .unwrap()+ .has_turn(&q.session_id, &q.turn_id)?,+ "Trace target turn unavailable"+ );+ let session = self.store.lock().unwrap().metadata(&q.session_id)?;+ let current = session.turn.as_ref().filter(|t| t.id == q.turn_id);+ let canonical = Canonical {+ terminal: current.is_some_and(|t| openworker_runtime::store::terminal(&t.status)),+ expected_segments: current+ .and_then(|t| t.observation.as_ref())+ .map(|o| o.segments),+ first_started_ms: current+ .and_then(|t| t.observation.as_ref())+ .map(|o| o.first_started_at_ms),+ last_span_id: current.and_then(|t| t.observation.as_ref()).map(|o| {+ o.last_segment+ .span+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect()+ }),+ pending_waits: current.map(|t| {+ session+ .interactions+ .iter()+ .filter(|i| i.turn_id == t.id && i.response.is_none())+ .count() as u64+ }),+ };+ let recording = self+ .recording+ .clone()+ .ok_or_else(|| anyhow::anyhow!("Recording unavailable"))?;+ let is_summary = method == "observability/summary";+ ensure!(+ !is_summary || q.after == 0,+ "Summary cannot start at a page offset"+ );+ let read = Query {+ deployment: recording.options.deployment.clone(),+ scopes: vec![Scope {+ session_hash: openworker_runtime::bundle::digest(q.session_id.as_bytes()),+ turn_hash: openworker_runtime::bundle::digest(q.turn_id.as_bytes()),+ }],+ after: q.after,+ through: q.through,+ limit: if is_summary {+ MAX_SUMMARY_STEPS+ } else {+ q.limit+ },+ };+ let permit = self+ .recording_readers+ .clone()+ .try_acquire_owned()+ .map_err(|_| anyhow::anyhow!("Recording readers busy"))?;+ let page = tokio::task::spawn_blocking(move || {+ let _permit = permit;+ recording.options.open()?.timeline(&read)+ })+ .await??;+ // Recheck after a potentially slow independent database read.+ self.store.lock().unwrap().metadata(&q.session_id)?;+ let run = json!({"session_id":q.session_id,"turn_id":q.turn_id});+ return if is_summary {+ Ok(json!({"run":run,"summary":summary::build(&page,canonical)}))+ } else {+ Ok(+ json!({"run":run,"page":page,"coverage":{"guarantee":"best_effort","order":"recording_cursor","snapshot_across_requests":false}}),+ )+ };+ }+ anyhow::bail!("Unsupported observability operation or capability")+ }+}diff --git a/harness/harness-sdk/src/instrumentation.rs b/harness/harness-sdk/src/instrumentation.rsnew file mode 100644index 00000000..a593149a--- /dev/null+++ b/harness/harness-sdk/src/instrumentation.rs@@ -0,0 +1,153 @@+//! SDK composition of the same recorder/exporter used by the service.+use crate::HostConfig;+use anyhow::Result;+use openworker_telemetry::{Controller, Telemetry};++pub(crate) struct Started {+ #[cfg(feature = "evaluation-export")]+ pub evaluation_exporter: Option<std::sync::Arc<crate::evaluation_export::Controller>>,+ pub telemetry: Telemetry,+ pub controller: Option<Controller>,+ #[cfg(feature = "recording")]+ pub recording: Option<std::sync::Arc<Recording>>,+}+pub(crate) fn start(config: &HostConfig) -> Result<Started> {+ #[cfg(feature = "recording")]+ if let Some(options) = &config.recording {+ use openworker_observability::{Recorder, RecordingSink};+ use openworker_telemetry::{Capture, ContentPolicy, Sink};+ let exporter: Option<Box<dyn Sink>> = None;+ #[cfg(feature = "otlp")]+ let exporter = config+ .telemetry+ .as_ref()+ .map(|c| {+ c.build_sink("sdk-host")+ .map(|s| Box::new(s) as Box<dyn Sink>)+ })+ .transpose()?+ .or(exporter);+ let export_content = config+ .telemetry+ .as_ref()+ .map(|c| c.content.clone())+ .unwrap_or_default();+ let (recorder, controller) = Recorder::start(+ options.open()?,+ options.deployment.clone(),+ options.queue_capacity,+ options.retention_days,+ )?;+ let statistics = recorder.statistics();+ let sink = RecordingSink::new(+ recorder,+ options.deployment.clone(),+ "sdk-host".into(),+ exporter,+ export_content.capture,+ )?;+ let (telemetry, telemetry_controller) = Telemetry::start_with_content(+ sink,+ 2048,+ ContentPolicy {+ capture: export_content.capture.max(Capture::Metadata),+ ..export_content+ },+ )?;+ #[cfg(feature = "evaluation-export")]+ let evaluation_exporter = crate::evaluation_export::start(options.clone())?;+ return Ok(Started {+ #[cfg(feature = "evaluation-export")]+ evaluation_exporter,+ telemetry,+ controller: Some(telemetry_controller),+ recording: Some(std::sync::Arc::new(Recording {+ options: options.clone(),+ statistics,+ controller: std::sync::Mutex::new(Some(controller)),+ })),+ });+ }+ let (telemetry, controller) = match &config.telemetry {+ #[cfg(feature = "otlp")]+ Some(c) => {+ let (t, c) = c.start("sdk-host")?;+ (t, Some(c))+ }+ #[cfg(not(feature = "otlp"))]+ Some(_) => anyhow::bail!("Telemetry exporter not included"),+ None => (Telemetry::default(), None),+ };+ Ok(Started {+ #[cfg(feature = "evaluation-export")]+ evaluation_exporter: None,+ telemetry,+ controller,+ #[cfg(feature = "recording")]+ recording: None,+ })+}++#[cfg(feature = "recording")]+pub(crate) struct Recording {+ pub options: crate::RecordingConfig,+ pub statistics: openworker_observability::Statistics,+ pub controller: std::sync::Mutex<Option<openworker_observability::Controller>>,+}+#[cfg(feature = "recording")]+impl crate::RecordingConfig {+ pub(crate) fn open(&self) -> Result<Box<dyn openworker_observability::Store>> {+ self.validate()?;+ match &self.database {+ crate::RecordingDatabase::Sqlite { path } => Ok(Box::new(+ openworker_observability::sqlite::Sqlite::open(path)?,+ )),+ #[cfg(feature = "recording-postgres")]+ crate::RecordingDatabase::Postgres { .. } => Ok(Box::new(+ openworker_observability::postgres::Postgres::open(&self.postgres()?)?,+ )),+ #[cfg(not(feature = "recording-postgres"))]+ crate::RecordingDatabase::Postgres { .. } => {+ anyhow::bail!("PostgreSQL recording not included")+ }+ }+ }+}++#[cfg(feature = "recording-postgres")]+impl crate::RecordingConfig {+ fn postgres(&self) -> Result<openworker_observability::postgres::Config> {+ use openworker_observability::postgres::{Config, Tls};+ let crate::RecordingDatabase::Postgres {+ connection_env,+ schema,+ ca_file,+ loopback_plaintext,+ } = &self.database+ else {+ anyhow::bail!("PostgreSQL recording configuration required")+ };+ self.validate()?;+ let mut c = Config::new(+ std::env::var(connection_env)+ .map_err(|_| anyhow::anyhow!("Recording database credential unavailable"))?,+ schema.clone(),+ );+ c.tls = if *loopback_plaintext {+ Tls::LoopbackPlaintext+ } else {+ Tls::Verified {+ ca_file: ca_file.clone(),+ }+ };+ Ok(c)+ }+ /// Explicit trusted setup operation. Host startup never migrates PostgreSQL.+ pub async fn migrate_postgres(&self) -> Result<()> {+ let options = self.clone();+ tokio::task::spawn_blocking(move || {+ openworker_observability::postgres::Postgres::migrate(&options.postgres()?)+ })+ .await?+ }+}diff --git a/harness/harness-sdk/src/lib.rs b/harness/harness-sdk/src/lib.rsindex 252cabce..5bdd7705 100644--- a/harness/harness-sdk/src/lib.rs+++ b/harness/harness-sdk/src/lib.rs@@ -1,6 +1,11 @@//! Development Rust API for the same engine embedded or over managed private stdio.//! No request is automatically resubmitted after transport failure.pub mod agents;+#[cfg(feature = "evaluation-export")]+mod evaluation_export;+mod instrumentation;+pub mod recording;+pub use recording::{RecordingConfig, RecordingDatabase, TraceQuery};mod application_tools;mod callbacks;mod provider_callbacks;@@ -34,8 +39,9 @@ pub use openworker_runtime::team_service::{Command as TeamCommand, Create as Teapub use openworker_runtime::teams::{Member as TeamMember, Role as TeamRole};/// OTLP/HTTP export settings: `endpoint` (collector origin, HTTPS or loopback/// HTTP) and `deployment` (label, default `"default"`).-pub use openworker_telemetry::otlp::{- Config as TelemetryConfig, Header as TelemetryHeader, Signal as TelemetrySignal,+pub use openworker_telemetry::config::{+ Acknowledgement as TelemetryAcknowledgement, Config as TelemetryConfig,+ Header as TelemetryHeader, Signal as TelemetrySignal,};/// Opt-in prompt/tool content in telemetry (off by default).pub use openworker_telemetry::{Capture, ContentPolicy};@@ -517,3 +523,6 @@ pub use openworker_runtime::compliance;pub use openworker_security_adapters as security_adapters;pub use openworker_runtime::compliance_export::Sink as ComplianceSink;++#[cfg(feature = "evaluations")]+pub use openworker_observability::evaluations;diff --git a/harness/harness-sdk/src/recording.rs b/harness/harness-sdk/src/recording.rsnew file mode 100644index 00000000..6a9ff83d--- /dev/null+++ b/harness/harness-sdk/src/recording.rs@@ -0,0 +1,432 @@+//! Optional local recording configuration. Does not initialize storage on import.+use anyhow::{ensure, Result};+use serde::{Deserialize, Serialize};+use std::path::PathBuf;++#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct RecordingConfig {+ #[serde(default = "evaluation_capacity")]+ pub max_pending_evaluations: u32,+ #[serde(default = "retention")]+ pub evaluation_retention_days: u16,+ #[serde(default)]+ pub evaluation_sinks: Vec<EvaluationSink>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub evidence: Option<EvidencePolicy>,+ #[serde(default)]+ pub evaluations: bool,+ pub database: RecordingDatabase,+ #[serde(default = "deployment")]+ pub deployment: String,+ #[serde(default = "retention")]+ pub retention_days: u16,+ #[serde(default = "capacity")]+ pub queue_capacity: usize,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]+pub enum RecordingDatabase {+ Sqlite {+ path: PathBuf,+ },+ Postgres {+ connection_env: String,+ schema: String,+ ca_file: Option<PathBuf>,+ #[serde(default)]+ loopback_plaintext: bool,+ },+}+fn deployment() -> String {+ "sdk".into()+}+fn evaluation_capacity() -> u32 {+ 10_000+}+fn retention() -> u16 {+ 30+}+fn capacity() -> usize {+ 1024+}+impl RecordingConfig {+ pub fn sqlite(path: impl Into<PathBuf>) -> Self {+ Self {+ database: RecordingDatabase::Sqlite { path: path.into() },+ evaluations: false,+ max_pending_evaluations: evaluation_capacity(),+ evaluation_retention_days: retention(),+ evaluation_sinks: vec![],+ evidence: None,+ deployment: deployment(),+ retention_days: retention(),+ queue_capacity: capacity(),+ }+ }+ pub fn validate(&self) -> Result<()> {+ ensure!(+ (1..=1_000_000).contains(&self.max_pending_evaluations),+ "Invalid evaluation backlog limit"+ );+ ensure!(+ !self.deployment.is_empty()+ && self.deployment.len() <= 64+ && self+ .deployment+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),+ "Invalid recording deployment"+ );+ ensure!(+ (1..=365).contains(&self.retention_days) && (1..=2048).contains(&self.queue_capacity),+ "Invalid recording retention or queue capacity"+ );+ ensure!(+ (1..=3650).contains(&self.evaluation_retention_days),+ "Invalid evaluation retention"+ );+ ensure!(+ self.evaluation_sinks.len() <= 8,+ "At most eight evaluation destinations"+ );+ let mut names = std::collections::BTreeSet::new();+ for sink in &self.evaluation_sinks {+ sink.validate()?;+ ensure!(+ names.insert(&sink.name),+ "Duplicate evaluation destination name"+ );+ }+ if let Some(p) = &self.evidence {+ anyhow::ensure!(+ (256..=65536).contains(&p.max_bytes),+ "Evidence byte limit must be 256..65536"+ );+ }+ match &self.database {+ RecordingDatabase::Sqlite { path } => ensure!(+ path.is_absolute() && path.file_name().is_some(),+ "Recording path must be an absolute file path"+ ),+ RecordingDatabase::Postgres {+ connection_env,+ schema,+ ca_file,+ ..+ } => {+ ensure!(+ !connection_env.is_empty()+ && connection_env.len() <= 100+ && connection_env+ .bytes()+ .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),+ "Invalid recording credential reference"+ );+ ensure!(+ schema.starts_with("ow_")+ && schema.len() <= 48+ && schema+ .bytes()+ .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'),+ "Invalid recording schema"+ );+ ensure!(+ ca_file.as_ref().is_none_or(|p| p.is_absolute()),+ "Recording CA path must be absolute"+ );+ }+ }+ Ok(())+ }+}++/// A trusted SDK query. Service deployments apply their own session authorization.+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct TraceQuery {+ pub session_id: String,+ pub turn_id: String,+ #[serde(default)]+ pub after: u64,+ pub through: Option<u64>,+ #[serde(default = "page_size")]+ pub limit: u16,+}+fn page_size() -> u16 {+ 100+}+impl TraceQuery {+ pub fn turn(session_id: impl Into<String>, turn_id: impl Into<String>) -> Self {+ Self {+ session_id: session_id.into(),+ turn_id: turn_id.into(),+ after: 0,+ through: None,+ limit: page_size(),+ }+ }+ pub fn validate(&self) -> Result<()> {+ for id in [&self.session_id, &self.turn_id] {+ ensure!(+ !id.is_empty()+ && id.len() <= 80+ && id+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),+ "Invalid trace scope"+ );+ }+ ensure!(+ (1..=200).contains(&self.limit)+ && self.after <= i64::MAX as u64+ && self+ .through+ .is_none_or(|n| n >= self.after && n <= i64::MAX as u64),+ "Invalid trace page"+ );+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn configuration_is_explicit_bounded_and_secret_free() {+ let path = std::env::temp_dir().join("ow-observability-contract.sqlite");+ let mut c = RecordingConfig::sqlite(path);+ assert!(c.validate().is_ok());+ c.queue_capacity = 0;+ assert!(c.validate().is_err());+ assert!(serde_json::from_value::<RecordingConfig>(serde_json::json!({"database":{"kind":"postgres","connection":"secret","connection_env":"PG","schema":"ow_obs"}})).is_err());+ }+ #[test]+ fn trace_query_rejects_raw_hash_scope_and_unbounded_pages() {+ let mut q = TraceQuery::turn("session-1", "turn-2");+ assert!(q.validate().is_ok());+ q.limit = 201;+ assert!(q.validate().is_err());+ assert!(serde_json::from_value::<TraceQuery>(+ serde_json::json!({"session_hash":"other","session_id":"s","turn_id":"t"})+ )+ .is_err());+ }+}++/// Query adapter shared by embedded and managed-process SDK clients.+#[derive(Clone)]+pub struct Observability {+ client: crate::HarnessClient,+}+impl Observability {+ pub(crate) fn new(client: crate::HarnessClient) -> Self {+ Self { client }+ }+ #[cfg(feature = "evaluations")]+ pub async fn record_evaluation(+ &self,+ query: TraceQuery,+ result: openworker_observability::evaluations::Input,+ span_id: Option<String>,+ ) -> crate::Result<serde_json::Value> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ self.client.request("observability/evaluation-record",serde_json::json!({"session_id":query.session_id,"turn_id":query.turn_id,"span_id":span_id,"result":result})).await+ }+ #[cfg(feature = "evaluations")]+ pub async fn evaluations(+ &self,+ query: TraceQuery,+ ) -> crate::Result<openworker_observability::evaluations::Page> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ let value = self+ .client+ .request(+ "observability/evaluation-list",+ serde_json::to_value(query)+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,+ )+ .await?;+ serde_json::from_value(value).map_err(|e| crate::Error::Remote(e.to_string()))+ }+ pub async fn flush(&self, timeout_ms: u64) -> crate::Result<serde_json::Value> {+ self.client+ .request(+ "observability/flush",+ serde_json::json!({"timeout_ms":timeout_ms}),+ )+ .await+ }+ pub async fn health(&self) -> crate::Result<serde_json::Value> {+ self.client+ .request("observability/health", serde_json::json!({}))+ .await+ }+ pub async fn turns(+ &self,+ session_id: &str,+ after: i64,+ limit: u16,+ ) -> crate::Result<serde_json::Value> {+ self.client+ .request(+ "observability/turns",+ serde_json::json!({"session_id":session_id,"after":after,"limit":limit}),+ )+ .await+ }+ pub async fn subtree_summary(&self, query: TraceQuery) -> crate::Result<serde_json::Value> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ self.client+ .request(+ "observability/subtree-summary",+ serde_json::to_value(query)+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,+ )+ .await+ }+ pub async fn children(&self, query: TraceQuery) -> crate::Result<serde_json::Value> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ self.client+ .request(+ "observability/children",+ serde_json::to_value(query)+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,+ )+ .await+ }+ #[cfg(feature = "recording")]+ pub async fn timeline(&self, query: TraceQuery) -> crate::Result<Timeline> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ let v = self+ .client+ .request(+ "observability/timeline",+ serde_json::to_value(query)+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,+ )+ .await?;+ serde_json::from_value(v).map_err(|e| crate::Error::Remote(e.to_string()))+ }+ #[cfg(feature = "recording")]+ pub async fn summary(&self, query: TraceQuery) -> crate::Result<TurnSummary> {+ query+ .validate()+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?;+ let v = self+ .client+ .request(+ "observability/summary",+ serde_json::to_value(query)+ .map_err(|e| crate::Error::InvalidInput(e.to_string()))?,+ )+ .await?;+ serde_json::from_value(v).map_err(|e| crate::Error::Remote(e.to_string()))+ }+}++/// Local evidence access is a separate opt-in from remote telemetry capture.+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct EvidencePolicy {+ #[serde(default)]+ pub user_input: bool,+ #[serde(default)]+ pub assistant_output: bool,+ #[serde(default)]+ pub tool_arguments: bool,+ #[serde(default)]+ pub tool_results: bool,+ pub max_bytes: usize,+}++#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct EvaluationSink {+ pub name: String,+ pub url: String,+ pub authorization_env: String,+ #[serde(default)]+ pub include_explanation: bool,+}+impl EvaluationSink {+ pub fn validate(&self) -> Result<()> {+ ensure!(+ !self.name.is_empty()+ && self.name.len() <= 64+ && self+ .name+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)),+ "Invalid evaluation destination name"+ );+ ensure!(+ !self.authorization_env.is_empty()+ && self.authorization_env.len() <= 100+ && self+ .authorization_env+ .bytes()+ .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),+ "Invalid evaluation credential reference"+ );+ let u = reqwest::Url::parse(&self.url)?;+ ensure!(+ u.username().is_empty()+ && u.password().is_none()+ && u.query().is_none()+ && u.fragment().is_none()+ && u.path() == "/"+ && (u.scheme() == "https"+ || (u.scheme() == "http"+ && u.host_str().is_some_and(|h| h == "localhost"+ || h.trim_matches(['[', ']'])+ .parse::<std::net::IpAddr>()+ .is_ok_and(|ip| ip.is_loopback())))),+ "Evaluation destination must be HTTPS or explicit loopback origin"+ );+ Ok(())+ }+}++#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct RunIdentity {+ pub session_id: String,+ pub turn_id: String,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct RecordedTurn {+ pub session_id: String,+ pub turn_id: String,+ pub status: Option<String>,+ pub agent: String,+ pub name: String,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct TurnPage {+ pub turns: Vec<RecordedTurn>,+ pub next_cursor: Option<i64>,+ pub through: i64,+}+#[cfg(feature = "recording")]+#[derive(Debug, Serialize, Deserialize)]+pub struct Timeline {+ pub run: RunIdentity,+ pub page: openworker_observability::query::Page,+ pub coverage: serde_json::Value,+}+#[cfg(feature = "recording")]+#[derive(Debug, Serialize, Deserialize)]+pub struct TurnSummary {+ pub run: RunIdentity,+ pub summary: openworker_observability::summary::Summary,+}diff --git a/harness/harness-sdk/src/service.rs b/harness/harness-sdk/src/service.rsindex 23191598..b81adfb4 100644--- a/harness/harness-sdk/src/service.rs+++ b/harness/harness-sdk/src/service.rs@@ -19,6 +19,12 @@ pub use openworker_runtime::coding_handoff::config::{pub use openworker_runtime::coding_handoff::ObjectFormat as GitObjectFormat;pub use openworker_runtime::team_definition::Sandbox as TeamSandbox;+#[cfg(feature = "recording")]+#[path = "service_observability.rs"]+mod observability;+#[cfg(feature = "recording")]+pub use observability::Observability;+pub fn new_idempotency_key() -> String {openworker_runtime::id("idem")}diff --git a/harness/harness-sdk/src/service_observability.rs b/harness/harness-sdk/src/service_observability.rsnew file mode 100644index 00000000..5f62b9e6--- /dev/null+++ b/harness/harness-sdk/src/service_observability.rs@@ -0,0 +1,51 @@+//! Typed read-only service queries. Authorization remains on the service.+use super::{error, Result, Service};+use crate::recording::{Timeline, TurnSummary};+use crate::TraceQuery;+use serde::de::DeserializeOwned;+use serde_json::Value;++pub struct Observability<'a> {+ service: &'a Service,+}+impl Service {+ pub fn observability(&self) -> Observability<'_> {+ Observability { service: self }+ }+}+impl Observability<'_> {+ async fn read<T: DeserializeOwned>(&self, query: TraceQuery, kind: &str) -> Result<T> {+ query.validate().map_err(|_| error("InvalidInput"))?;+ let mut path = format!(+ "/v1/sessions/{}/observability/{}/{}",+ query.session_id, query.turn_id, kind+ );+ let mut params = Vec::new();+ if kind == "timeline" {+ params.push(format!("after={}&limit={}", query.after, query.limit));+ }+ if kind != "children" {+ if let Some(through) = query.through {+ params.push(format!("through={through}"));+ }+ }+ if !params.is_empty() {+ path.push('?');+ path.push_str(¶ms.join("&"));+ }+ let value = self.service.json("GET", &path, None, None).await?;+ serde_json::from_value(value).map_err(|_| error("InvalidResponse"))+ }+ pub async fn timeline(&self, query: TraceQuery) -> Result<Timeline> {+ self.read(query, "timeline").await+ }+ pub async fn summary(&self, query: TraceQuery) -> Result<TurnSummary> {+ self.read(query, "summary").await+ }+ pub async fn children(&self, query: TraceQuery) -> Result<Value> {+ self.read(query, "children").await+ }+ pub async fn subtree_summary(&self, query: TraceQuery) -> Result<Value> {+ self.read(query, "subtree-summary").await+ }+}diff --git a/harness/harness-sdk/tests/agents_regressions.rs b/harness/harness-sdk/tests/agents_regressions.rsindex 536b3035..b405921c 100644--- a/harness/harness-sdk/tests/agents_regressions.rs+++ b/harness/harness-sdk/tests/agents_regressions.rs@@ -93,6 +93,7 @@ async fn exercise(postgres: bool, process: bool) {sandbox: Sandbox::Off,scripted: true,telemetry: None,+ recording: None,agents: vec![Agent::new("assistant").instructions("Use tools.")diff --git a/harness/harness-sdk/tests/dynamic_agents.rs b/harness/harness-sdk/tests/dynamic_agents.rsindex 39f42dc0..5c15538e 100644--- a/harness/harness-sdk/tests/dynamic_agents.rs+++ b/harness/harness-sdk/tests/dynamic_agents.rs@@ -51,6 +51,7 @@ async fn exercise(postgres: bool, process: bool) {sandbox: Sandbox::Off,scripted: true,telemetry: None,+ recording: None,agents: vec![],};let h = connect(&config, process).await;@@ -244,6 +245,7 @@ async fn live_configuration_requires_explicit_model_without_calling_provider() {sandbox: Sandbox::Off,scripted: false,telemetry: None,+ recording: None,};let h = connect(&config, false).await;let agent = h@@ -285,6 +287,7 @@ async fn registering_startup_definition_makes_it_durable_without_replacement() {sandbox: Sandbox::Off,scripted: true,telemetry: None,+ recording: None,};let h = connect(&config, false).await;h.create_agent(agent.clone()).await.unwrap();diff --git a/harness/harness-sdk/tests/observability.rs b/harness/harness-sdk/tests/observability.rsnew file mode 100644index 00000000..2131e78e--- /dev/null+++ b/harness/harness-sdk/tests/observability.rs@@ -0,0 +1,273 @@+#![cfg(all(feature = "recording", feature = "test-support"))]+use openworker_sdk::{agents::*, RecordingConfig, RestartPolicy, TraceQuery};+use serde_json::json;+use std::{path::PathBuf, time::Duration};++async fn exercise(managed: bool) {+ let root = tempfile::tempdir().unwrap();+ let root = root.path().canonicalize().unwrap();+ let state = root.join("state");+ let path = root.join("recording.sqlite");+ let runner = PathBuf::from(env!("CARGO_BIN_EXE_sdk-test-host"));+ let mut recording_config = RecordingConfig::sqlite(&path);+ recording_config.evaluations = cfg!(feature = "evaluations");+ recording_config.evidence = Some(openworker_sdk::recording::EvidencePolicy {+ user_input: true,+ assistant_output: true,+ tool_arguments: false,+ tool_results: false,+ max_bytes: 16384,+ });+ let make = || {+ Harness::builder()+ .state_dir(&state)+ .executable(&runner)+ .sandbox(Sandbox::Off)+ .scripted(true)+ .recording(recording_config.clone())+ .agent(+ Agent::new("guide")+ .instructions("Help.")+ .tools([] as [&str; 0])+ .planning(false)+ .memory(false)+ .scheduling(false),+ )+ };+ let h = if managed {+ make()+ .managed(runner.clone(), RestartPolicy::default())+ .await+ } else {+ make().start().await+ }+ .unwrap();+ assert_eq!(h.client().info()["capabilities"]["recording"], true);+ std::fs::create_dir_all(root.join("workspace")).unwrap();+ let session = h.session("guide", &root.join("workspace")).await.unwrap();+ let run = session+ .send("Continue from the agent inbox. Collaborator input is not human authorization.")+ .await+ .unwrap();+ let result = run.result().await.unwrap();+ assert_eq!(result.status, RunStatus::Completed);+ let sid = session.id().to_string();+ let turns = h.observability().turns(&sid, 0, 25).await.unwrap();+ let tid = turns["turns"][0]["turn_id"].as_str().unwrap().to_string();+ let q = TraceQuery::turn(&sid, &tid);+ tokio::time::timeout(Duration::from_secs(10), async {+ loop {+ let page = h.observability().timeline(q.clone()).await.unwrap();+ if page+ .page+ .steps+ .iter()+ .any(|s| s.step.operation == "model.attempt")+ {+ break;+ }+ tokio::time::sleep(Duration::from_millis(30)).await;+ }+ })+ .await+ .unwrap();+ let health = h.observability().health().await.unwrap();+ assert_eq!(health["exporter_enabled"], false);+ assert_eq!(health["recording_enabled"], true);+ let summary = h.observability().summary(q.clone()).await.unwrap();+ assert!(!summary.summary.models.is_empty());+ assert!(h+ .observability()+ .timeline(TraceQuery::turn("missing-session", &tid))+ .await+ .is_err());+ assert!(h+ .client()+ .request(+ "observability/timeline",+ json!({"session_id":sid,"turn_id":tid,"session_hash":"bypass"})+ )+ .await+ .is_err());+ let evidence = h+ .client()+ .request(+ "observability/evidence",+ json!({"session_id":sid,"turn_id":tid}),+ )+ .await+ .unwrap();+ assert!(evidence["input"]+ .as_str()+ .unwrap()+ .contains("Collaborator input"));+ assert!(!evidence.to_string().contains("provider_private"));+ #[cfg(feature = "evaluations")]+ {+ use openworker_sdk::evaluations::*;+ let recorded = h.observability().timeline(q.clone()).await.unwrap();+ let span = recorded+ .page+ .steps+ .iter()+ .find(|r| r.step.operation == "model.attempt")+ .unwrap()+ .step+ .span_id+ .clone();+ let value = Input {+ id: "quality-one".into(),+ name: "correctness".into(),+ evaluator: Evaluator {+ name: "fixture".into(),+ version: "1".into(),+ source: Source::Code,+ model: None,+ usage: None,+ },+ status: Status::Completed,+ score: Some(Score::Boolean(true)),+ created_ms: openworker_runtime::store::now_ms(),+ explanation: Some("sk-syntheticsecret123456789".into()),+ dataset_case: None,+ experiment: None,+ supersedes: None,+ };+ let result = h+ .observability()+ .record_evaluation(q.clone(), value.clone(), Some(span.clone()))+ .await+ .unwrap();+ assert_eq!(result["duplicate"], false);+ assert!(!result.to_string().contains("syntheticsecret"));+ assert_eq!(+ h.observability()+ .record_evaluation(q.clone(), value.clone(), Some(span.clone()))+ .await+ .unwrap()["duplicate"],+ true+ );+ assert!(h+ .observability()+ .record_evaluation(+ TraceQuery::turn(&sid, "missing-turn"),+ value.clone(),+ Some(span.clone())+ )+ .await+ .is_err());+ h.observability().flush(5000).await.unwrap();+ rusqlite::Connection::open(&path)+ .unwrap()+ .execute("DELETE FROM ow_observation_steps WHERE span_id=?", [&span])+ .unwrap();+ assert_eq!(+ h.observability()+ .record_evaluation(q.clone(), value.clone(), Some(span))+ .await+ .unwrap()["duplicate"],+ true+ );+ let mut read = q.clone();+ read.limit = 25;+ assert_eq!(+ h.observability()+ .evaluations(read)+ .await+ .unwrap()+ .results+ .len(),+ 1+ );+ }+ h.shutdown().await.unwrap();+ let reopened = make().start().await.unwrap();+ let page = reopened.observability().timeline(q).await.unwrap();+ assert!(!page.page.steps.is_empty());+ assert!(!serde_json::to_string(&page)+ .unwrap()+ .contains("Collaborator input"));+ reopened.shutdown().await.unwrap();+}+#[tokio::test]+async fn embedded_records_queries_and_reopens() {+ exercise(false).await;+}+#[tokio::test]+async fn managed_records_queries_and_reopens() {+ exercise(true).await;+}+#[tokio::test]+async fn disabled_recording_does_not_create_a_database() {+ let root = tempfile::tempdir().unwrap();+ let state = root.path().join("state");+ let h = Harness::builder()+ .state_dir(&state)+ .executable(env!("CARGO_BIN_EXE_sdk-test-host"))+ .sandbox(Sandbox::Off)+ .start()+ .await+ .unwrap();+ assert_eq!(+ h.observability().health().await.unwrap()["recording_enabled"],+ false+ );+ assert!(h+ .observability()+ .timeline(TraceQuery::turn("s", "t"))+ .await+ .is_err());+ assert!(!state.join("recording.sqlite").exists());+ h.shutdown().await.unwrap();+}++#[cfg(feature = "recording-postgres")]+#[tokio::test]+#[ignore = "disposable loopback PostgreSQL fixture required"]+async fn postgres_recording_is_independent_from_sqlite_execution() {+ let root = tempfile::tempdir().unwrap();+ let schema = format!("ow_sdk_obs_{}", std::process::id());+ let config = RecordingConfig {+ evaluations: false,+ max_pending_evaluations: 10_000,+ evaluation_retention_days: 30,+ evidence: None,+ evaluation_sinks: vec![],+ database: openworker_sdk::RecordingDatabase::Postgres {+ connection_env: "OW_OBSERVABILITY_TEST_URL".into(),+ schema: schema.clone(),+ ca_file: None,+ loopback_plaintext: true,+ },+ deployment: "sdk".into(),+ retention_days: 7,+ queue_capacity: 16,+ };+ config.migrate_postgres().await.unwrap();+ let h = Harness::builder()+ .state_dir(root.path().join("state"))+ .executable(env!("CARGO_BIN_EXE_sdk-test-host"))+ .sandbox(Sandbox::Off)+ .recording(config)+ .start()+ .await+ .unwrap();+ assert_eq!(+ h.observability().health().await.unwrap()["recording_enabled"],+ true+ );+ assert!(root.path().join("state/runtime.sqlite3").exists());+ h.shutdown().await.unwrap();+ tokio::task::spawn_blocking(move || {+ let mut client = postgres::Client::connect(+ &std::env::var("OW_OBSERVABILITY_TEST_URL").unwrap(),+ postgres::NoTls,+ )+ .unwrap();+ client+ .batch_execute(&format!("DROP SCHEMA {schema} CASCADE"))+ .unwrap();+ })+ .await+ .unwrap();+}diff --git a/harness/harness-sdk/tests/standalone_subagents.rs b/harness/harness-sdk/tests/standalone_subagents.rsindex f4f29eda..86d88918 100644--- a/harness/harness-sdk/tests/standalone_subagents.rs+++ b/harness/harness-sdk/tests/standalone_subagents.rs@@ -32,6 +32,10 @@ async fn scenario(managed: bool, execution: &str, approval: u8) {}).allow_subagents(true),);+ #[cfg(feature = "recording")]+ let builder = builder.recording(openworker_sdk::RecordingConfig::sqlite(+ root.join("traces.sqlite"),+ ));let h = if managed {builder.managed(executable, Default::default())@@ -91,6 +95,31 @@ async fn scenario(managed: bool, execution: &str, approval: u8) {.await.unwrap();assert_eq!(run.status, RunStatus::Completed, "{run:?} {events}");+ #[cfg(feature = "recording")]+ {+ let turns = h.observability().turns(parent.id(), 0, 25).await.unwrap();+ let tid = turns["turns"][0]["turn_id"].as_str().unwrap();+ let children = h+ .observability()+ .children(openworker_sdk::TraceQuery::turn(parent.id(), tid))+ .await+ .unwrap();+ assert_eq!(children["children"].as_array().unwrap().len(), 1);+ assert_eq!(children["children"][0]["agent_id"], "reader");+ assert_eq!(children["children"][0]["origin"]["parent_turn_id"], tid);+ assert!(children["children"][0]["origin"]["parent_call_id"].is_string());+ assert!(!children.to_string().contains("CHILD_FILE_EVIDENCE"));+ assert!(children["children"][0]["child_turn_id"].is_string());+ let tree = h+ .observability()+ .subtree_summary(openworker_sdk::TraceQuery::turn(parent.id(), tid))+ .await+ .unwrap();+ assert_eq!(tree["selection"]["runs"].as_array().unwrap().len(), 2);+ assert_eq!(tree["selection"]["selection_truncated"], false);+ assert!(tree["summary"]["wall_elapsed_us"].is_null());+ }+let doc = h.client().request(diff --git a/harness/harness-sdk/tests/storage_teams.rs b/harness/harness-sdk/tests/storage_teams.rsindex c1c625df..95ce690a 100644--- a/harness/harness-sdk/tests/storage_teams.rs+++ b/harness/harness-sdk/tests/storage_teams.rs@@ -106,6 +106,7 @@ async fn exercise(postgres: bool, process: bool) {sandbox: Sandbox::Off,scripted: true,telemetry: None,+ recording: None,agents: vec![],};let c = client(&config, process).await;diff --git a/harness/observability/Cargo.toml b/harness/observability/Cargo.tomlindex 19bcb4ca..492c96cc 100644--- a/harness/observability/Cargo.toml+++ b/harness/observability/Cargo.toml@@ -1,18 +1,23 @@[package]name = "openworker-observability"-version = "0.1.0"+version.workspace = trueedition.workspace = truerust-version.workspace = truelicense.workspace = true-publish = false+publish = ["crates-io"]+readme = "README.md"+repository = "https://github.com/andrewyng/openworker"+description = "Optional bounded recording, trace queries and evaluation storage for OpenWorker"[features]default = []+evaluations = ["dep:openworker-compliance"]postgres = ["dep:postgres", "dep:postgres-native-tls", "dep:native-tls", "dep:rustls-pemfile", "dep:zeroize"]otlp = ["openworker-telemetry/otlp"][dependencies]-openworker-telemetry = { path = "../telemetry" }+openworker-compliance = { path = "../compliance", version = "=0.1.0-beta.2", optional = true }+openworker-telemetry = { path = "../telemetry", version = "=0.1.0-beta.2" }anyhow = "1"serde = { version = "1", features = ["derive"] }serde_json = "1"diff --git a/harness/observability/LICENSE b/harness/observability/LICENSEnew file mode 100644index 00000000..60d38246--- /dev/null+++ b/harness/observability/LICENSE@@ -0,0 +1,18 @@+MIT License++Copyright (c) 2024 Andrew Ng++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and+associated documentation files (the "Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the+following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial+portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.diff --git a/harness/observability/src/evaluations.rs b/harness/observability/src/evaluations.rsnew file mode 100644index 00000000..46c7ccf1--- /dev/null+++ b/harness/observability/src/evaluations.rs@@ -0,0 +1,295 @@+//! Optional, acknowledged evaluation records. Never sent through the lossy trace queue.+use anyhow::{ensure, Result};+pub const DEFAULT_MAX_PENDING: u32 = 10_000;+use serde::{Deserialize, Serialize};++#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Target {+ pub session_hash: String,+ pub turn_hash: String,+ pub trace_id: Option<String>,+ pub span_id: Option<String>,+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Evaluator {+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub model: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub usage: Option<crate::Usage>,+ pub name: String,+ pub version: String,+ pub source: Source,+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Source {+ Human,+ Code,+ Model,+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(+ tag = "type",+ content = "value",+ rename_all = "snake_case",+ deny_unknown_fields+)]+pub enum Score {+ Numeric(f64),+ Boolean(bool),+ Categorical(String),+ Text(String),+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Status {+ Completed,+ Skipped,+ Error,+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Input {+ pub id: String,+ pub name: String,+ pub evaluator: Evaluator,+ pub status: Status,+ pub score: Option<Score>,+ pub created_ms: u64,+ pub explanation: Option<String>,+ pub dataset_case: Option<String>,+ pub experiment: Option<String>,+ /// Corrections append a new ID, preserving the original evaluation.+ pub supersedes: Option<String>,+}+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Evaluation {+ pub version: u8,+ pub deployment: String,+ pub target: Target,+ pub result: Input,+}+impl Evaluation {+ pub fn validate(&self) -> Result<()> {+ ensure!(+ self.version == 1 && crate::identifier(&self.deployment),+ "Unsupported evaluation schema or deployment"+ );+ ensure!(+ crate::hex_id(&self.target.session_hash, 64)+ && crate::hex_id(&self.target.turn_hash, 64),+ "Invalid evaluation scope"+ );+ ensure!(+ self.target+ .trace_id+ .as_ref()+ .is_none_or(|v| crate::hex_id(v, 32))+ && self+ .target+ .span_id+ .as_ref()+ .is_none_or(|v| crate::hex_id(v, 16))+ && (self.target.span_id.is_none() || self.target.trace_id.is_some()),+ "Invalid evaluation trace target"+ );+ for id in [+ &self.result.id,+ &self.result.name,+ &self.result.evaluator.name,+ &self.result.evaluator.version,+ ]+ .into_iter()+ .chain(self.result.dataset_case.iter())+ .chain(self.result.experiment.iter())+ .chain(self.result.supersedes.iter())+ {+ ensure!(crate::label(id), "Invalid evaluation identifier");+ }+ ensure!(+ self.result+ .evaluator+ .model+ .as_ref()+ .is_none_or(|m| crate::label(m)),+ "Invalid evaluator model"+ );+ ensure!(+ (self.result.evaluator.model.is_none() && self.result.evaluator.usage.is_none())+ || self.result.evaluator.source == Source::Model,+ "Only model evaluators carry model accounting"+ );+ if let Some(u) = &self.result.evaluator.usage {+ ensure!(+ openworker_telemetry::TokenUsage {+ input: u.input,+ output: u.output,+ cache_read: u.cache_read,+ cache_write: u.cache_write,+ reasoning: u.reasoning+ }+ .validate(),+ "Invalid evaluator usage"+ );+ }+ ensure!(+ self.result.created_ms > 0 && self.result.created_ms <= 253402300799000,+ "Invalid evaluation timestamp"+ );+ ensure!(+ (self.result.status == Status::Completed) == self.result.score.is_some(),+ "Only completed evaluations carry a score"+ );+ ensure!(+ self.result.supersedes.as_ref() != Some(&self.result.id),+ "An evaluation cannot supersede itself"+ );+ if let Some(score) = &self.result.score {+ match score {+ Score::Numeric(n) => ensure!(n.is_finite(), "Numeric score must be finite"),+ Score::Categorical(s) => {+ ensure!(!s.is_empty() && s.len() <= 256, "Invalid category")+ }+ Score::Text(s) => ensure!(s.len() <= 4000, "Score text exceeds limit"),+ Score::Boolean(_) => {}+ }+ }+ ensure!(+ self.result+ .explanation+ .as_ref()+ .is_none_or(|s| s.len() <= 4000),+ "Evaluation explanation exceeds limit"+ );+ ensure!(+ serde_json::to_vec(self)?.len() <= 16384,+ "Evaluation exceeds record limit"+ );+ Ok(())+ }+ /// Best-effort secret redaction precedes durable storage and export.+ pub fn redacted(mut self) -> Self {+ if let Some(text) = &mut self.result.explanation {+ *text = openworker_telemetry::redact(text);+ }+ if let Some(Score::Text(text) | Score::Categorical(text)) = &mut self.result.score {+ *text = openworker_telemetry::redact(text);+ }+ self+ }+}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Row {+ pub cursor: u64,+ pub evaluation: Evaluation,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Page {+ pub results: Vec<Row>,+ pub next: Option<u64>,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Query {+ pub deployment: String,+ pub session_hash: String,+ pub turn_hash: String,+ pub after: u64,+ pub limit: u16,+}+impl Query {+ pub fn validate(&self) -> Result<()> {+ ensure!(+ crate::identifier(&self.deployment)+ && crate::hex_id(&self.session_hash, 64)+ && crate::hex_id(&self.turn_hash, 64)+ && self.after <= i64::MAX as u64+ && (1..=50).contains(&self.limit),+ "Invalid evaluation query"+ );+ Ok(())+ }+}+pub(crate) fn page(mut results: Vec<Row>, q: &Query) -> Result<Page> {+ for row in &results {+ row.evaluation.validate()?;+ ensure!(+ row.cursor > q.after+ && row.evaluation.deployment == q.deployment+ && row.evaluation.target.session_hash == q.session_hash+ && row.evaluation.target.turn_hash == q.turn_hash,+ "Evaluation scope mismatch"+ );+ }+ let more = results.len() > usize::from(q.limit);+ results.truncate(q.limit.into());+ let next = more.then(|| results.last().unwrap().cursor);+ Ok(Page { results, next })+}++#[cfg(test)]+mod tests {+ use super::*;+ pub fn sample() -> Evaluation {+ Evaluation {+ version: 1,+ deployment: "sdk".into(),+ target: Target {+ session_hash: format!("{:064x}", 1),+ turn_hash: format!("{:064x}", 2),+ trace_id: None,+ span_id: None,+ },+ result: Input {+ id: "e1".into(),+ name: "correctness".into(),+ evaluator: Evaluator {+ name: "fixture".into(),+ version: "1".into(),+ source: Source::Code,+ model: None,+ usage: None,+ },+ status: Status::Completed,+ score: Some(Score::Boolean(true)),+ created_ms: 1,+ explanation: None,+ dataset_case: None,+ experiment: None,+ supersedes: None,+ },+ }+ }+ #[test]+ fn status_and_target_cannot_misrepresent_results() {+ let mut e = sample();+ assert!(e.validate().is_ok());+ e.result.status = Status::Error;+ assert!(e.validate().is_err());+ e.result.score = None;+ assert!(e.validate().is_ok());+ e.target.span_id = Some(format!("{:016x}", 1));+ assert!(e.validate().is_err());+ }+}++/// Reuses the compliance outbox lease/backoff contract; only evaluation delivery+/// is retried. No runtime action is addressed by this checkpoint.+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Delivery {+ pub token: String,+ pub row: Row,+}+pub fn delivery_key(deployment: &str, sink: &str, binding: &str, token: &str) -> Result<()> {+ ensure!(+ crate::identifier(deployment)+ && crate::identifier(sink)+ && crate::hex_id(binding, 64)+ && crate::identifier(token),+ "Invalid evaluation delivery scope"+ );+ Ok(())+}diff --git a/harness/observability/src/lib.rs b/harness/observability/src/lib.rsindex 60404c30..200a6194 100644--- a/harness/observability/src/lib.rs+++ b/harness/observability/src/lib.rs@@ -4,6 +4,8 @@ use anyhow::{ensure, Result};use openworker_telemetry::Record;use serde::{Deserialize, Serialize};use std::time::UNIX_EPOCH;+#[cfg(feature = "evaluations")]+pub mod evaluations;#[cfg(feature = "postgres")]pub mod postgres;pub mod query;@@ -340,6 +342,74 @@ pub struct StoredStep {/// Trusted host interface only. A service must authorize the canonical session/// before querying; the observation database is never an authorization authority.pub trait Store: Send + 'static {+ #[cfg(feature = "evaluations")]+ fn register_evaluation_sink(+ &mut self,+ _deployment: &str,+ _sink: &str,+ _binding: &str,+ ) -> Result<()> {+ anyhow::bail!("Evaluation export unavailable")+ }++ #[cfg(feature = "evaluations")]+ fn prune_evaluations(+ &mut self,+ _deployment: &str,+ _before_ms: u64,+ _limit: u16,+ ) -> Result<u64> {+ anyhow::bail!("Evaluation pruning unsupported")+ }++ #[cfg(feature = "evaluations")]+ fn claim_evaluation_export(+ &mut self,+ _deployment: &str,+ _sink: &str,+ _binding: &str,+ _token: &str,+ _now: u64,+ ) -> Result<Option<evaluations::Delivery>> {+ anyhow::bail!("Evaluation export unavailable")+ }+ #[cfg(feature = "evaluations")]+ fn settle_evaluation_export(+ &mut self,+ _deployment: &str,+ _sink: &str,+ _token: &str,+ _now: u64,+ _success: bool,+ ) -> Result<()> {+ anyhow::bail!("Evaluation export unavailable")+ }++ #[cfg(feature = "evaluations")]+ fn evaluation(+ &mut self,+ _deployment: &str,+ _id: &str,+ ) -> Result<Option<evaluations::Evaluation>> {+ anyhow::bail!("Evaluation storage unavailable")+ }+ #[cfg(feature = "evaluations")]+ fn record_evaluation(&mut self, value: &evaluations::Evaluation) -> Result<Write> {+ self.record_evaluation_bounded(value, evaluations::DEFAULT_MAX_PENDING)+ }+ #[cfg(feature = "evaluations")]+ fn record_evaluation_bounded(+ &mut self,+ _value: &evaluations::Evaluation,+ _max_pending: u32,+ ) -> Result<Write> {+ anyhow::bail!("Evaluation storage unsupported")+ }+ #[cfg(feature = "evaluations")]+ fn evaluations(&mut self, _query: &evaluations::Query) -> Result<evaluations::Page> {+ anyhow::bail!("Evaluation storage unsupported")+ }+/// Read-only bounded session/turn projection; no authorization authority.fn timeline(&mut self, _query: &query::Query) -> Result<query::Page> {anyhow::bail!("Recording queries unsupported by this adapter")diff --git a/harness/observability/src/postgres.rs b/harness/observability/src/postgres.rsindex 4b902ec9..601af64e 100644--- a/harness/observability/src/postgres.rs+++ b/harness/observability/src/postgres.rs@@ -136,6 +136,8 @@ impl Postgres {INSERT INTO observation_version VALUES(TRUE,1);")?;}tx.batch_execute("CREATE INDEX IF NOT EXISTS observation_turn ON observation_steps(deployment,session_hash,(data->>'turn_hash'),cursor)")?;+ #[cfg(feature="evaluations")]+ tx.batch_execute("CREATE TABLE IF NOT EXISTS observation_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data JSONB NOT NULL,PRIMARY KEY(deployment,sink)); CREATE TABLE IF NOT EXISTS observation_evaluations(cursor BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,deployment TEXT NOT NULL,id TEXT NOT NULL,session_hash TEXT NOT NULL,turn_hash TEXT NOT NULL,data JSONB NOT NULL,recorded_ms BIGINT NOT NULL DEFAULT ((extract(epoch FROM clock_timestamp())*1000)::bigint),UNIQUE(deployment,id)); CREATE INDEX IF NOT EXISTS observation_evaluation_scope ON observation_evaluations(deployment,session_hash,turn_hash,cursor)")?;tx.commit()?;Ok(())}@@ -156,6 +158,223 @@ impl Postgres {}}impl Store for Postgres {+ #[cfg(feature = "evaluations")]+ fn register_evaluation_sink(+ &mut self,+ deployment: &str,+ sink: &str,+ binding: &str,+ ) -> Result<()> {+ crate::evaluations::delivery_key(deployment, sink, binding, "registration")?;+ let mut tx = self.client.transaction()?;+ tx.execute(+ "INSERT INTO observation_evaluation_delivery VALUES($1,$2,$3) ON CONFLICT DO NOTHING",+ &[+ &deployment,+ &sink,+ &serde_json::to_value(openworker_compliance::delivery::Checkpoint::new(+ binding.into(),+ ))?,+ ],+ )?;+ let row=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;+ let c: openworker_compliance::delivery::Checkpoint = serde_json::from_value(row.get(0))?;+ ensure!(+ c.binding == binding,+ "Evaluation destination changed; choose a new sink name"+ );+ tx.commit()?;+ Ok(())+ }++ #[cfg(feature = "evaluations")]+ fn prune_evaluations(&mut self, deployment: &str, before_ms: u64, limit: u16) -> Result<u64> {+ crate::query(deployment, None, 0, limit)?;+ ensure!(before_ms <= i64::MAX as u64, "Invalid cutoff");+ Ok(self.client.execute("DELETE FROM observation_evaluations WHERE cursor IN(SELECT e.cursor FROM observation_evaluations e WHERE e.deployment=$1 AND e.recorded_ms<$2 AND NOT EXISTS(SELECT 1 FROM observation_evaluation_delivery d WHERE d.deployment=e.deployment AND (d.data->>'cursor')::bigint<e.cursor) ORDER BY e.cursor LIMIT $3)",&[&deployment,&(before_ms as i64),&i64::from(limit)])?)+ }++ #[cfg(feature = "evaluations")]+ fn claim_evaluation_export(+ &mut self,+ deployment: &str,+ sink: &str,+ binding: &str,+ token: &str,+ now: u64,+ ) -> Result<Option<crate::evaluations::Delivery>> {+ use openworker_compliance::delivery::Checkpoint;+ crate::evaluations::delivery_key(deployment, sink, binding, token)?;+ let mut tx = self.client.transaction()?;+ tx.execute(+ "INSERT INTO observation_evaluation_delivery VALUES($1,$2,$3) ON CONFLICT DO NOTHING",+ &[+ &deployment,+ &sink,+ &serde_json::to_value(Checkpoint::new(binding.into()))?,+ ],+ )?;+ let raw=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;+ let mut checkpoint: Checkpoint = serde_json::from_value(raw.get(0))?;+ ensure!(+ checkpoint.binding == binding,+ "Export destination changed; choose a new sink name"+ );+ let row=tx.query_opt("SELECT cursor,data FROM observation_evaluations WHERE deployment=$1 AND cursor>$2 ORDER BY cursor LIMIT 1",&[&deployment,&checkpoint.cursor])?;+ let Some(row) = row else {+ tx.commit()?;+ return Ok(None);+ };+ let cursor: i64 = row.get(0);+ let claimed = checkpoint.claim(binding, token, now, cursor)?;+ let evaluation: crate::evaluations::Evaluation = serde_json::from_value(row.get(1))?;+ evaluation.validate()?;+ ensure!(+ evaluation.deployment == deployment,+ "Evaluation export scope mismatch"+ );+ tx.execute(+ "UPDATE observation_evaluation_delivery SET data=$1 WHERE deployment=$2 AND sink=$3",+ &[&serde_json::to_value(checkpoint)?, &deployment, &sink],+ )?;+ tx.commit()?;+ Ok(claimed.then(|| crate::evaluations::Delivery {+ token: token.into(),+ row: crate::evaluations::Row {+ cursor: cursor as u64,+ evaluation,+ },+ }))+ }+ #[cfg(feature = "evaluations")]+ fn settle_evaluation_export(+ &mut self,+ deployment: &str,+ sink: &str,+ token: &str,+ now: u64,+ success: bool,+ ) -> Result<()> {+ let mut tx = self.client.transaction()?;+ let raw=tx.query_one("SELECT data FROM observation_evaluation_delivery WHERE deployment=$1 AND sink=$2 FOR UPDATE",&[&deployment,&sink])?;+ let mut c: openworker_compliance::delivery::Checkpoint =+ serde_json::from_value(raw.get(0))?;+ c.settle(token, now, success)?;+ tx.execute(+ "UPDATE observation_evaluation_delivery SET data=$1 WHERE deployment=$2 AND sink=$3",+ &[&serde_json::to_value(c)?, &deployment, &sink],+ )?;+ tx.commit()?;+ Ok(())+ }++ #[cfg(feature = "evaluations")]+ fn evaluation(+ &mut self,+ deployment: &str,+ id: &str,+ ) -> Result<Option<crate::evaluations::Evaluation>> {+ ensure!(+ crate::identifier(deployment) && crate::label(id),+ "Invalid evaluation identity"+ );+ self.client+ .query_opt(+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",+ &[&deployment, &id],+ )?+ .map(|r| {+ let e: crate::evaluations::Evaluation = serde_json::from_value(r.get(0))?;+ e.validate()?;+ ensure!(+ e.deployment == deployment && e.result.id == id,+ "Evaluation identity mismatch"+ );+ Ok(e)+ })+ .transpose()+ }+ #[cfg(feature = "evaluations")]+ fn record_evaluation_bounded(+ &mut self,+ value: &crate::evaluations::Evaluation,+ max_pending: u32,+ ) -> Result<Write> {+ ensure!(+ (1..=1_000_000).contains(&max_pending),+ "Invalid evaluation export backlog limit"+ );+ value.validate()?;+ let value = value.clone().redacted();+ value.validate()?;+ let data = serde_json::to_value(&value)?;+ if let Some(old) = self.evaluation(&value.deployment, &value.result.id)? {+ ensure!(old == value, "Evaluation identity conflict");+ return Ok(Write::Duplicate);+ }+ let mut tx = self.client.transaction()?;+ tx.query_one(+ "SELECT pg_advisory_xact_lock(hashtext($1))",+ &[&format!("eval-capacity:{}", value.deployment)],+ )?;+ let retained: i64 = tx+ .query_one(+ "SELECT count(*) FROM observation_evaluations e WHERE deployment=$1 AND EXISTS(SELECT 1 FROM observation_evaluation_delivery d WHERE d.deployment=e.deployment AND (d.data->>'cursor')::bigint<e.cursor)",+ &[&value.deployment],+ )?+ .get(0);+ let exists:bool=tx.query_one("SELECT EXISTS(SELECT 1 FROM observation_evaluations WHERE deployment=$1 AND id=$2)",&[&value.deployment,&value.result.id])?.get(0);+ ensure!(+ exists || retained < i64::from(max_pending),+ "Pending evaluation export limit reached; restore delivery or raise the configured limit"+ );+ if let Some(id) = &value.result.supersedes {+ let prior = tx+ .query_opt(+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",+ &[&value.deployment, id],+ )?+ .context("Superseded evaluation missing")?;+ let prior: crate::evaluations::Evaluation = serde_json::from_value(prior.get(0))?;+ ensure!(+ prior.target == value.target,+ "Correction must retain its target"+ );+ }+ let inserted=tx.execute("INSERT INTO observation_evaluations(deployment,id,session_hash,turn_hash,data) VALUES($1,$2,$3,$4,$5) ON CONFLICT(deployment,id) DO NOTHING",&[&value.deployment,&value.result.id,&value.target.session_hash,&value.target.turn_hash,&data])?;+ if inserted == 0 {+ let old = tx.query_one(+ "SELECT data FROM observation_evaluations WHERE deployment=$1 AND id=$2",+ &[&value.deployment, &value.result.id],+ )?;+ ensure!(+ serde_json::from_value::<crate::evaluations::Evaluation>(old.get(0))? == value,+ "Evaluation identity conflict"+ );+ }+ tx.commit()?;+ Ok(if inserted == 0 {+ Write::Duplicate+ } else {+ Write::Inserted+ })+ }+ #[cfg(feature = "evaluations")]+ fn evaluations(&mut self, q: &crate::evaluations::Query) -> Result<crate::evaluations::Page> {+ q.validate()?;+ let rows=self.client.query("SELECT cursor,data FROM observation_evaluations WHERE deployment=$1 AND session_hash=$2 AND turn_hash=$3 AND cursor>$4 ORDER BY cursor LIMIT $5",&[&q.deployment,&q.session_hash,&q.turn_hash,&(q.after as i64),&(i64::from(q.limit)+1)])?;+ let rows = rows+ .into_iter()+ .map(|r| {+ Ok(crate::evaluations::Row {+ cursor: r.get::<_, i64>(0) as u64,+ evaluation: serde_json::from_value(r.get(1))?,+ })+ })+ .collect::<Result<Vec<_>>>()?;+ crate::evaluations::page(rows, q)+ }+fn timeline(&mut self, q: &crate::query::Query) -> Result<crate::query::Page> {q.validate()?;let sessions: Vec<_> = q.scopes.iter().map(|s| s.session_hash.as_str()).collect();diff --git a/harness/observability/src/query.rs b/harness/observability/src/query.rsindex 26371133..a2c326a5 100644--- a/harness/observability/src/query.rs+++ b/harness/observability/src/query.rs@@ -44,7 +44,8 @@ pub struct Page {}pub(crate) fn page(rows: Vec<(i64, Step)>, through: u64, q: &Query) -> Result<Page> {let limit = q.limit;- let more = rows.len() > usize::from(limit);+ let mut more = rows.len() > usize::from(limit);+ let mut bytes = 0usize;let mut steps = vec![];for (cursor, step) in rows.into_iter().take(limit.into()) {ensure!(cursor > 0, "Invalid recording cursor");@@ -57,6 +58,12 @@ pub(crate) fn page(rows: Vec<(i64, Step)>, through: u64, q: &Query) -> Result<Pa&& step.turn_hash.as_ref() == Some(&s.turn_hash)),"Recording scope mismatch");+ let size = serde_json::to_vec(&step)?.len();+ if q.limit <= 200 && !steps.is_empty() && bytes + size > 512_000 {+ more = true;+ break;+ }+ bytes += size;steps.push(StoredStep {cursor: cursor as u64,step,diff --git a/harness/observability/src/sqlite.rs b/harness/observability/src/sqlite.rsindex a9d0196d..3c168cb5 100644--- a/harness/observability/src/sqlite.rs+++ b/harness/observability/src/sqlite.rs@@ -41,6 +41,7 @@ impl Sqlite {| OpenFlags::SQLITE_OPEN_NOFOLLOW,)?;db.busy_timeout(Duration::from_millis(500))?;+ ensure!(db.query_row("SELECT count(*) FROM sqlite_master WHERE type='table' AND name IN ('sessions','events','submissions')", [], |r| r.get::<_, i64>(0))? == 0, "Recording database must be separate from execution storage");db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;CREATE TABLE IF NOT EXISTS ow_observation_version(singleton INTEGER PRIMARY KEY CHECK(singleton=1), version INTEGER NOT NULL);")?;let version: Option<i64> = db@@ -75,6 +76,299 @@ impl Sqlite {}}impl Store for Sqlite {+ #[cfg(feature = "evaluations")]+ fn register_evaluation_sink(+ &mut self,+ deployment: &str,+ sink: &str,+ binding: &str,+ ) -> Result<()> {+ crate::evaluations::delivery_key(deployment, sink, binding, "registration")?;+ let tx = self+ .db+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;+ tx.execute(+ "INSERT INTO ow_evaluation_delivery VALUES(?,?,?) ON CONFLICT DO NOTHING",+ params![+ deployment,+ sink,+ serde_json::to_string(&openworker_compliance::delivery::Checkpoint::new(+ binding.into()+ ))?+ ],+ )?;+ let raw: String = tx.query_row(+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",+ params![deployment, sink],+ |r| r.get(0),+ )?;+ let c: openworker_compliance::delivery::Checkpoint = serde_json::from_str(&raw)?;+ ensure!(+ c.binding == binding,+ "Evaluation destination changed; choose a new sink name"+ );+ tx.commit()?;+ Ok(())+ }++ #[cfg(feature = "evaluations")]+ fn prune_evaluations(&mut self, deployment: &str, before_ms: u64, limit: u16) -> Result<u64> {+ crate::query(deployment, None, 0, limit)?;+ ensure!(before_ms <= i64::MAX as u64, "Invalid cutoff");+ if self.db.query_row(+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",+ [],+ |r| r.get::<_, i64>(0),+ )? == 0+ {+ return Ok(0);+ }+ self.db.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;+ Ok(self.db.execute("DELETE FROM ow_observation_evaluations WHERE cursor IN(SELECT e.cursor FROM ow_observation_evaluations e WHERE e.deployment=?1 AND e.recorded_ms<?2 AND NOT EXISTS(SELECT 1 FROM ow_evaluation_delivery d WHERE d.deployment=e.deployment AND json_extract(d.data,'$.cursor')<e.cursor) ORDER BY e.cursor LIMIT ?3)",params![deployment,before_ms as i64,i64::from(limit)])? as u64)+ }++ #[cfg(feature = "evaluations")]+ fn claim_evaluation_export(+ &mut self,+ deployment: &str,+ sink: &str,+ binding: &str,+ token: &str,+ now: u64,+ ) -> Result<Option<crate::evaluations::Delivery>> {+ use openworker_compliance::delivery::Checkpoint;+ crate::evaluations::delivery_key(deployment, sink, binding, token)?;+ let tx = self+ .db+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink))")?;+ let raw: Option<String> = tx+ .query_row(+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",+ params![deployment, sink],+ |r| r.get(0),+ )+ .optional()?;+ let mut checkpoint = raw+ .map(|s| serde_json::from_str::<Checkpoint>(&s))+ .transpose()?+ .unwrap_or_else(|| Checkpoint::new(binding.into()));+ ensure!(+ checkpoint.binding == binding,+ "Export destination changed; choose a new sink name"+ );+ tx.execute(+ "INSERT INTO ow_evaluation_delivery VALUES(?,?,?) ON CONFLICT DO NOTHING",+ params![deployment, sink, serde_json::to_string(&checkpoint)?],+ )?;+ if tx.query_row(+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",+ [],+ |r| r.get::<_, i64>(0),+ )? == 0+ {+ tx.commit()?;+ return Ok(None);+ }+ let row:Option<(i64,String)>=tx.query_row("SELECT cursor,data FROM ow_observation_evaluations WHERE deployment=? AND cursor>? ORDER BY cursor LIMIT 1",params![deployment,checkpoint.cursor],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;+ let Some((cursor, data)) = row else {+ tx.commit()?;+ return Ok(None);+ };+ let claimed = checkpoint.claim(binding, token, now, cursor)?;+ let evaluation: crate::evaluations::Evaluation = serde_json::from_str(&data)?;+ evaluation.validate()?;+ ensure!(+ evaluation.deployment == deployment,+ "Evaluation export scope mismatch"+ );+ tx.execute(+ "UPDATE ow_evaluation_delivery SET data=? WHERE deployment=? AND sink=?",+ params![serde_json::to_string(&checkpoint)?, deployment, sink],+ )?;+ tx.commit()?;+ Ok(claimed.then(|| crate::evaluations::Delivery {+ token: token.into(),+ row: crate::evaluations::Row {+ cursor: cursor as u64,+ evaluation,+ },+ }))+ }+ #[cfg(feature = "evaluations")]+ fn settle_evaluation_export(+ &mut self,+ deployment: &str,+ sink: &str,+ token: &str,+ now: u64,+ success: bool,+ ) -> Result<()> {+ let tx = self+ .db+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;+ let raw: String = tx.query_row(+ "SELECT data FROM ow_evaluation_delivery WHERE deployment=? AND sink=?",+ params![deployment, sink],+ |r| r.get(0),+ )?;+ let mut c: openworker_compliance::delivery::Checkpoint = serde_json::from_str(&raw)?;+ c.settle(token, now, success)?;+ tx.execute(+ "UPDATE ow_evaluation_delivery SET data=? WHERE deployment=? AND sink=?",+ params![serde_json::to_string(&c)?, deployment, sink],+ )?;+ tx.commit()?;+ Ok(())+ }++ #[cfg(feature = "evaluations")]+ fn evaluation(+ &mut self,+ deployment: &str,+ id: &str,+ ) -> Result<Option<crate::evaluations::Evaluation>> {+ ensure!(+ crate::identifier(deployment) && crate::label(id),+ "Invalid evaluation identity"+ );+ if self.db.query_row(+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",+ [],+ |r| r.get::<_, i64>(0),+ )? == 0+ {+ return Ok(None);+ }+ let raw: Option<String> = self+ .db+ .query_row(+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",+ params![deployment, id],+ |r| r.get(0),+ )+ .optional()?;+ raw.map(|raw| {+ let e: crate::evaluations::Evaluation = serde_json::from_str(&raw)?;+ e.validate()?;+ ensure!(+ e.deployment == deployment && e.result.id == id,+ "Evaluation identity mismatch"+ );+ Ok(e)+ })+ .transpose()+ }+ #[cfg(feature = "evaluations")]+ fn record_evaluation_bounded(+ &mut self,+ value: &crate::evaluations::Evaluation,+ max_pending: u32,+ ) -> Result<Write> {+ ensure!(+ (1..=1_000_000).contains(&max_pending),+ "Invalid evaluation export backlog limit"+ );+ use rusqlite::OptionalExtension;+ value.validate()?;+ let value = value.clone().redacted();+ value.validate()?;+ let data = serde_json::to_string(&value)?;+ // Existing corrections remain idempotent even after their predecessor expires.+ if let Some(old) = self.evaluation(&value.deployment, &value.result.id)? {+ ensure!(old == value, "Evaluation identity conflict");+ return Ok(Write::Duplicate);+ }+ let tx = self+ .db+ .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;+ tx.execute_batch("CREATE TABLE IF NOT EXISTS ow_evaluation_delivery(deployment TEXT NOT NULL,sink TEXT NOT NULL,data TEXT NOT NULL,PRIMARY KEY(deployment,sink)); CREATE TABLE IF NOT EXISTS ow_observation_evaluations(cursor INTEGER PRIMARY KEY AUTOINCREMENT,deployment TEXT NOT NULL,id TEXT NOT NULL,session_hash TEXT NOT NULL,turn_hash TEXT NOT NULL,data TEXT NOT NULL,recorded_ms INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)*1000),UNIQUE(deployment,id)); CREATE INDEX IF NOT EXISTS ow_evaluation_scope ON ow_observation_evaluations(deployment,session_hash,turn_hash,cursor)")?;+ let retained: i64 = tx.query_row(+ "SELECT count(*) FROM ow_observation_evaluations e WHERE deployment=? AND EXISTS(SELECT 1 FROM ow_evaluation_delivery d WHERE d.deployment=e.deployment AND json_extract(d.data,'$.cursor')<e.cursor)",+ [&value.deployment],+ |r| r.get(0),+ )?;+ let exists: bool = tx.query_row(+ "SELECT EXISTS(SELECT 1 FROM ow_observation_evaluations WHERE deployment=? AND id=?)",+ params![value.deployment, value.result.id],+ |r| r.get(0),+ )?;+ ensure!(+ exists || retained < i64::from(max_pending),+ "Pending evaluation export limit reached; restore delivery or raise the configured limit"+ );+ if let Some(id) = &value.result.supersedes {+ let prior: Option<String> = tx+ .query_row(+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",+ params![value.deployment, id],+ |r| r.get(0),+ )+ .optional()?;+ let prior: crate::evaluations::Evaluation =+ serde_json::from_str(&prior.context("Superseded evaluation missing")?)?;+ ensure!(+ prior.target == value.target,+ "Correction must retain its target"+ );+ }+ let inserted=tx.execute("INSERT INTO ow_observation_evaluations(deployment,id,session_hash,turn_hash,data) VALUES(?,?,?,?,?) ON CONFLICT(deployment,id) DO NOTHING",params![value.deployment,value.result.id,value.target.session_hash,value.target.turn_hash,data])?;+ if inserted == 0 {+ let old: String = tx.query_row(+ "SELECT data FROM ow_observation_evaluations WHERE deployment=? AND id=?",+ params![value.deployment, value.result.id],+ |r| r.get(0),+ )?;+ ensure!(+ serde_json::from_str::<crate::evaluations::Evaluation>(&old)? == value,+ "Evaluation identity conflict"+ );+ }+ tx.commit()?;+ Ok(if inserted == 0 {+ Write::Duplicate+ } else {+ Write::Inserted+ })+ }+ #[cfg(feature = "evaluations")]+ fn evaluations(&mut self, q: &crate::evaluations::Query) -> Result<crate::evaluations::Page> {+ q.validate()?;+ if self.db.query_row(+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_evaluations'",+ [],+ |r| r.get::<_, i64>(0),+ )? == 0+ {+ return Ok(crate::evaluations::Page {+ results: vec![],+ next: None,+ });+ }+ let mut stmt=self.db.prepare("SELECT cursor,data FROM ow_observation_evaluations WHERE deployment=? AND session_hash=? AND turn_hash=? AND cursor>? ORDER BY cursor LIMIT ?")?;+ let rows = stmt.query_map(+ params![+ q.deployment,+ q.session_hash,+ q.turn_hash,+ q.after as i64,+ i64::from(q.limit) + 1+ ],+ |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),+ )?;+ let rows = rows+ .map(|r| {+ let (cursor, data) = r?;+ Ok(crate::evaluations::Row {+ cursor: cursor as u64,+ evaluation: serde_json::from_str(&data)?,+ })+ })+ .collect::<Result<Vec<_>>>()?;+ crate::evaluations::page(rows, q)+ }+fn timeline(&mut self, q: &crate::query::Query) -> Result<crate::query::Page> {q.validate()?;let tx = self.db.transaction()?;diff --git a/harness/observability/src/summary.rs b/harness/observability/src/summary.rsindex 80bc2ca5..3fb9db8d 100644--- a/harness/observability/src/summary.rs+++ b/harness/observability/src/summary.rs@@ -1,11 +1,11 @@//! Projection v1, rebuilt from bounded immutable raw records. It is not a bill,//! audit ledger, or proof that absent spans/counters never existed.use crate::{query::Page, Step};-use serde::Serialize;+use serde::{Deserialize, Serialize};use std::collections::{BTreeMap, BTreeSet};pub const VERSION: u8 = 1;pub const MAX_GROUPS: usize = 128;-#[derive(Clone, Debug, Default, Serialize)]+#[derive(Clone, Debug, Default, Serialize, Deserialize)]pub struct Measure {pub total: Option<u64>,pub known: u64,@@ -30,7 +30,7 @@ impl Measure {}}}-#[derive(Clone, Debug, Default, Serialize)]+#[derive(Clone, Debug, Default, Serialize, Deserialize)]pub struct Usage {pub input: Measure,pub output: Measure,@@ -47,7 +47,7 @@ impl Usage {self.reasoning.add(u.and_then(|u| u.reasoning));}}-#[derive(Debug, Default, Serialize)]+#[derive(Debug, Default, Serialize, Deserialize)]pub struct Group {pub name: Option<String>,pub purpose: Option<String>,@@ -97,9 +97,9 @@ pub struct Canonical {pub last_span_id: Option<String>,pub pending_waits: Option<u64>,}-#[derive(Debug, Serialize)]+#[derive(Debug, Serialize, Deserialize)]pub struct Coverage {- pub guarantee: &'static str,+ pub guarantee: String,pub scanned_steps: usize,pub scan_truncated: bool,pub group_truncated: bool,@@ -114,7 +114,7 @@ pub struct Coverage {pub clock_or_range_anomaly: bool,pub hierarchy_incomplete: bool,}-#[derive(Debug, Serialize)]+#[derive(Debug, Serialize, Deserialize)]pub struct Summary {pub projection_version: u8,pub through: u64,@@ -124,7 +124,7 @@ pub struct Summary {pub active_segments_accumulated_us: Measure,pub model_accumulated_us: Measure,pub tool_accumulated_us: Measure,- pub exclusive_observed_us: BTreeMap<&'static str, u64>,+ pub exclusive_observed_us: BTreeMap<String, u64>,pub usage: Usage,pub models: Vec<Group>,pub tools: Vec<Group>,@@ -332,7 +332,7 @@ pub fn build(page: &Page, canonical: Canonical) -> Summary {for (at, delta) in changes {if let Some(prev) = previous {let length = at - prev;- *exclusive.entry(phase(&active)).or_insert(0) += length;+ *exclusive.entry(phase(&active).to_owned()).or_insert(0) += length;window += length;}for n in 0..6 {@@ -386,7 +386,7 @@ pub fn build(page: &Page, canonical: Canonical) -> Summary {projection_version: VERSION,through: page.through,coverage: Coverage {- guarantee: "best_effort",+ guarantee: "best_effort".into(),scanned_steps: page.steps.len()diff --git a/harness/observability/tests/evaluations.rs b/harness/observability/tests/evaluations.rsnew file mode 100644index 00000000..842d5da9--- /dev/null+++ b/harness/observability/tests/evaluations.rs@@ -0,0 +1,206 @@+#![cfg(feature = "evaluations")]+use openworker_observability::{evaluations::*, sqlite::Sqlite, Store, Write};+fn sample(id: &str) -> Evaluation {+ Evaluation {+ version: 1,+ deployment: "sdk".into(),+ target: Target {+ session_hash: format!("{:064x}", 1),+ turn_hash: format!("{:064x}", 2),+ trace_id: None,+ span_id: None,+ },+ result: Input {+ id: id.into(),+ name: "correctness".into(),+ evaluator: Evaluator {+ name: "fixture".into(),+ version: "1".into(),+ source: Source::Code,+ model: None,+ usage: None,+ },+ status: Status::Completed,+ score: Some(Score::Boolean(true)),+ created_ms: 1,+ explanation: None,+ dataset_case: None,+ experiment: None,+ supersedes: None,+ },+ }+}+#[test]+fn scores_survive_reopen_and_do_not_rewrite_prior_evidence() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("traces.sqlite");+ let mut store = Sqlite::open(&path).unwrap();+ let e = sample("e1");+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Inserted);+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Duplicate);+ let mut conflict = e.clone();+ conflict.result.score = Some(Score::Boolean(false));+ assert!(store.record_evaluation(&conflict).is_err());+ let mut correction = sample("e2");+ correction.result.supersedes = Some("e1".into());+ correction.result.score = Some(Score::Boolean(false));+ store.record_evaluation(&correction).unwrap();+ drop(store);+ let mut store = Sqlite::open(&path).unwrap();+ let mut query = Query {+ deployment: "sdk".into(),+ session_hash: e.target.session_hash.clone(),+ turn_hash: e.target.turn_hash.clone(),+ after: 0,+ limit: 1,+ };+ let page = store.evaluations(&query).unwrap();+ assert_eq!(page.results[0].evaluation, e);+ query.after = page.next.unwrap();+ let page = store.evaluations(&query).unwrap();+ assert_eq!(page.results[0].evaluation, correction);+ assert!(page.next.is_none());+ query.after = 0;+ query.turn_hash = format!("{:064x}", 3);+ assert!(store.evaluations(&query).unwrap().results.is_empty());+ assert_eq!(+ store.prune_evaluations("sdk", i64::MAX as u64, 1).unwrap(),+ 1+ );+ assert!(store.evaluation("sdk", "e1").unwrap().is_none());+ assert_eq!(+ store.record_evaluation(&correction).unwrap(),+ Write::Duplicate+ );+}++#[cfg(feature = "postgres")]+#[test]+#[ignore = "disposable loopback PostgreSQL fixture required"]+fn postgres_evaluations_are_idempotent_scoped_and_durable() {+ use openworker_observability::postgres::{Config, Postgres, Tls};+ let url = std::env::var("OW_OBSERVABILITY_TEST_URL").unwrap();+ let schema = format!(+ "ow_eval_{}",+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos()+ );+ let mut c = Config::new(url.clone(), schema.clone());+ c.tls = Tls::LoopbackPlaintext;+ Postgres::migrate(&c).unwrap();+ let mut store = Postgres::open(&c).unwrap();+ let e = sample("pg-score");+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Inserted);+ assert_eq!(store.record_evaluation(&e).unwrap(), Write::Duplicate);+ drop(store);+ let mut store = Postgres::open(&c).unwrap();+ let mut q = Query {+ deployment: "sdk".into(),+ session_hash: e.target.session_hash.clone(),+ turn_hash: e.target.turn_hash.clone(),+ after: 0,+ limit: 25,+ };+ assert_eq!(store.evaluations(&q).unwrap().results[0].evaluation, e);+ q.turn_hash = format!("{:064x}", 3);+ assert!(store.evaluations(&q).unwrap().results.is_empty());+ let mut correction = e.clone();+ correction.result.id = "pg-correction".into();+ correction.result.supersedes = Some(e.result.id.clone());+ store.record_evaluation(&correction).unwrap();+ assert_eq!(+ store.prune_evaluations("sdk", i64::MAX as u64, 1).unwrap(),+ 1+ );+ assert_eq!(+ store.record_evaluation(&correction).unwrap(),+ Write::Duplicate+ );+ drop(store);+ let mut outbox = Postgres::open(&c).unwrap();+ // A distinct deployment keeps the earlier storage contract out of this queue.+ outbox+ .prune_evaluations("sdk", i64::MAX as u64, 100)+ .unwrap();+ delivery_contract(&mut outbox);+ drop(outbox);+ let mut cleanup = postgres::Client::connect(&url, postgres::NoTls).unwrap();+ cleanup+ .batch_execute(&format!("DROP SCHEMA {schema} CASCADE"))+ .unwrap();+}++fn delivery_contract(store: &mut dyn Store) {+ let binding = "a".repeat(64);+ store+ .register_evaluation_sink("sdk", "langfuse", &binding)+ .unwrap();+ let first = sample("delivery-one");+ let second = sample("delivery-two");+ store.record_evaluation_bounded(&first, 1).unwrap();+ assert_eq!(+ store.record_evaluation_bounded(&first, 1).unwrap(),+ Write::Duplicate+ );+ assert!(store.record_evaluation_bounded(&second, 1).is_err());+ let claimed = store+ .claim_evaluation_export("sdk", "langfuse", &binding, "first", 100)+ .unwrap()+ .unwrap();+ assert_eq!(claimed.row.evaluation, first);+ assert!(store+ .claim_evaluation_export("sdk", "langfuse", &binding, "other", 101)+ .unwrap()+ .is_none());+ assert_eq!(+ store+ .prune_evaluations("sdk", i64::MAX as u64, 100)+ .unwrap(),+ 0+ );+ assert!(store+ .settle_evaluation_export("sdk", "langfuse", "forged", 102, true)+ .is_err());+ store+ .settle_evaluation_export("sdk", "langfuse", "first", 102, false)+ .unwrap();+ assert!(store+ .claim_evaluation_export("sdk", "langfuse", &binding, "retry", 103)+ .unwrap()+ .is_none());+ let retry = store+ .claim_evaluation_export("sdk", "langfuse", &binding, "retry", 10000)+ .unwrap()+ .unwrap();+ assert_eq!(retry.row.evaluation, first);+ store+ .settle_evaluation_export("sdk", "langfuse", "retry", 10001, true)+ .unwrap();+ // Retained, delivered evaluations must not consume pending admission capacity.+ store.record_evaluation_bounded(&second, 1).unwrap();+ assert_eq!(+ store+ .prune_evaluations("sdk", i64::MAX as u64, 100)+ .unwrap(),+ 1+ );+ assert!(store+ .register_evaluation_sink("sdk", "langfuse", &"b".repeat(64))+ .is_err());+}+#[test]+fn sqlite_outbox_backoff_binding_and_pending_retention() {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("trace.sqlite");+ let mut store = Sqlite::open(&path).unwrap();+ delivery_contract(&mut store);+ drop(store);+ let mut reopened = Sqlite::open(&path).unwrap();+ let next = reopened+ .claim_evaluation_export("sdk", "langfuse", &"a".repeat(64), "restart", 20000)+ .unwrap()+ .unwrap();+ assert_eq!(next.row.evaluation.result.id, "delivery-two");+}diff --git a/harness/observability/tests/storage.rs b/harness/observability/tests/storage.rsindex 59bf6a38..6f8e4cd2 100644--- a/harness/observability/tests/storage.rs+++ b/harness/observability/tests/storage.rs@@ -290,3 +290,22 @@ fn postgres_connection_validation() {c.schema = "ow_obs;DROP SCHEMA public".into();assert!(c.validate().is_err());}++#[test]+fn recording_refuses_execution_database_without_modifying_schema() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("runtime.sqlite3");+ let db = rusqlite::Connection::open(&path).unwrap();+ db.execute_batch("CREATE TABLE sessions(id TEXT PRIMARY KEY)")+ .unwrap();+ assert!(Sqlite::open(&path).is_err());+ assert_eq!(+ db.query_row(+ "SELECT count(*) FROM sqlite_master WHERE name='ow_observation_version'",+ [],+ |r| r.get::<_, i64>(0)+ )+ .unwrap(),+ 0+ );+}diff --git a/harness/release/README.md b/harness/release/README.mdindex e47b1e34..4648baad 100644--- a/harness/release/README.md+++ b/harness/release/README.md@@ -10,7 +10,7 @@ See [versions and compatibility](../../docs/harness/guides/releases.md).npm --prefix harness/typescript-sdk cinpm --prefix harness/typescript-sdk run buildcargo build --manifest-path harness/Cargo.toml --locked --release \- -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export+ -p openworker-sdk --bin openworker-harness --features security-adapters,compliance-export,recording-postgres,evaluation-exportnode harness/release/prepare.mjs \/absolute/target/release/openworker-harness aarch64-apple-darwin /absolute/new-candidate```@@ -42,14 +42,14 @@ hook or fetches an executable during startup.## Rust archives-Package the nine crates listed in `manifest.json` together, in dependency order.+Package the crates listed in `manifest.json` together, in dependency order.Cargo rewrites path+version dependencies into registry dependencies. For example:```shcargo package --manifest-path harness/Cargo.toml --no-verify \-p openworker-artifacts -p openworker-compliance -p openworker-guardrails \-p openworker-telemetry -p openworker-tools -p openworker-runtime \- -p openworker-security-adapters -p openworker-store-postgres -p openworker-sdk+ -p openworker-security-adapters -p openworker-store-postgres -p openworker-observability -p openworker-sdknode harness/release/check-examples.mjs /absolute/new-examplespython3 harness/release/verify-crates.py \--packages /absolute/target/package --out /absolute/new-consumer \diff --git a/harness/release/manifest.json b/harness/release/manifest.jsonindex d7d08356..16d4250f 100644--- a/harness/release/manifest.json+++ b/harness/release/manifest.json@@ -34,6 +34,10 @@"name": "openworker-store-postgres","directory": "store-postgres"},+ {+ "name": "openworker-observability",+ "directory": "observability"+ },{"name": "openworker-sdk","directory": "harness-sdk"diff --git a/harness/release/smoke-npm.mjs b/harness/release/smoke-npm.mjsindex 1e635c29..fffab1d3 100644--- a/harness/release/smoke-npm.mjs+++ b/harness/release/smoke-npm.mjs@@ -19,5 +19,15 @@ try {assert.equal(JSON.parse(await readFile(join(dir,'new-app/openworker.json'),'utf8')).sdkRoot,null);// The async fixture must keep its own event loop running while the app executes.execFileSync(process.execPath,[join(root,'release/run-examples.mjs'),dir,'-',process.execPath],{cwd:dir,stdio:'inherit'});+ await writeFile(join(dir,'observability.mjs'), `+ import assert from 'node:assert/strict';+ import {Harness,langfuseTelemetry} from '@openworkerai/harness-sdk';+ import {evidenceJsonl} from '@openworkerai/harness-sdk/evaluation-export';+ import {join} from 'node:path';+ const h=await Harness.start({stateDir:join(process.cwd(),'observation-state'),sandbox:'off',recording:{database:{type:'sqlite',path:join(process.cwd(),'observation.sqlite')},evaluations:true}});+ try {assert.equal((await h.observability.health()).recording_enabled,true);assert.equal((await h.observability.flush()).recording_drained,true);assert.equal(langfuseTelemetry('http://127.0.0.1:1','AUTH').acknowledgement,'langfuse_v4');assert(evidenceJsonl({selection:{sessionId:'s',turnId:'t'},watermark:0,timeline:{},summary:{}}).includes('selected_pages_only'));}+ finally {await h.shutdown();}+ `);+ execFileSync(process.execPath,[join(dir,'observability.mjs')],{cwd:dir,stdio:'inherit'});console.log('Packed npm client, native runtime, starter and exact TypeScript guide verified');} finally {await rm(dir,{recursive:true,force:true});}diff --git a/harness/release/verify-crates.py b/harness/release/verify-crates.pyindex de4207a9..cd7c2f20 100644--- a/harness/release/verify-crates.py+++ b/harness/release/verify-crates.py@@ -10,7 +10,7 @@ for item in m['rustPackages']:name=item['name'];filename=f'{name}-{m["version"]}'with tarfile.open(pathlib.Path(a.packages)/(filename+'.crate')) as archive:archive.extractall(out/'crates',filter='data')patch.append(f'{name} = {{ path = {json.dumps(str(out/"crates"/filename))} }}')-(out/'Cargo.toml').write_text('[package]\nname="openworker-registry-consumer"\nversion="0.0.0"\nedition="2021"\n[workspace]\n[dependencies]\nopenworker-sdk = {version="='+m['version']+'", features=["security-adapters","compliance-export"]}\ntokio = {version="1", features=["macros","rt-multi-thread"]}\nfutures-util="0.3"\nanyhow="1"\n[patch.crates-io]\n'+'\n'.join(patch)+'\n')+(out/'Cargo.toml').write_text('[package]\nname="openworker-registry-consumer"\nversion="0.0.0"\nedition="2021"\n[workspace]\n[dependencies]\nopenworker-sdk = {version="='+m['version']+'", features=["security-adapters","compliance-export","recording-postgres","evaluation-export"]}\ntokio = {version="1", features=["macros","rt-multi-thread"]}\nfutures-util="0.3"\nanyhow="1"\n[patch.crates-io]\n'+'\n'.join(patch)+'\n')(out/'src/main.rs').write_text(pathlib.Path(a.example).read_text() if a.example else 'fn main() { let _ = openworker_sdk::agents::Harness::builder(); }\n')if a.fetch:subprocess.run(['cargo','fetch','--manifest-path',str(out/'Cargo.toml')],check=True)diff --git a/harness/runtime/src/agent_team.rs b/harness/runtime/src/agent_team.rsindex 192366d8..f19f7fc7 100644--- a/harness/runtime/src/agent_team.rs+++ b/harness/runtime/src/agent_team.rs@@ -26,6 +26,8 @@ pub struct Wait {}#[derive(Clone, Debug, Serialize, Deserialize)]pub struct Document {+ #[serde(default)]+ pub trace_origins: BTreeMap<String, crate::observation_links::SpawnOrigin>,pub id: String,pub lead: String,pub roster: rules::Roster,@@ -365,6 +367,7 @@ pub fn apply(}}let doc = Document {+ trace_origins: BTreeMap::new(),id: id.clone(),lead: lead_id.clone(),roster,@@ -1037,6 +1040,26 @@ pub fn apply(};let reservation = doc.roster.reserve(request.clone(), || crate::id("s"))?;if reservation.created {+ if let Some(turn) = &s.turn {+ doc.trace_origins.insert(+ request.agent_id.clone(),+ crate::observation_links::SpawnOrigin {+ parent_session_id: s.id.clone(),+ parent_turn_id: turn.id.clone(),+ parent_call_id: turn+ .queue+ .first()+ .filter(|c| {+ matches!(+ c["name"].as_str(),+ Some("team_spawn" | "create_sub_agent")+ )+ })+ .and_then(|c| c["call_id"].as_str())+ .map(str::to_owned),+ },+ );+ }doc.sandbox_choices.insert(request.agent_id.clone(), sandbox.clone());if let Some(binding) = binding {@@ -1420,6 +1443,7 @@ pub fn public_snapshot(mut value: Value) -> Value {if let Some(object) = value.as_object_mut() {for key in ["standalone",+ "trace_origins","commands","tasks","charges",diff --git a/harness/runtime/src/connections.rs b/harness/runtime/src/connections.rsindex 96e26466..a58c389a 100644--- a/harness/runtime/src/connections.rs+++ b/harness/runtime/src/connections.rs@@ -160,7 +160,7 @@ impl Manager {bail!("Credential database cannot be a link");}let db = Connection::open(path)?;- db.execute_batch("PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL; PRAGMA secure_delete=ON;+ db.execute_batch("PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL; PRAGMA secure_delete=ON; BEGIN IMMEDIATE;CREATE TABLE IF NOT EXISTS connections(id TEXT PRIMARY KEY,label TEXT NOT NULL,identity TEXT NOT NULL,persistence TEXT NOT NULL,secret BLOB);CREATE TABLE IF NOT EXISTS settings(id INTEGER PRIMARY KEY CHECK(id=1),value TEXT NOT NULL);")?;let has_provider = db@@ -198,6 +198,10 @@ impl Manager {"ALTER TABLE endpoints ADD COLUMN prompt_caching INTEGER NOT NULL DEFAULT 1 CHECK(prompt_caching IN (0,1))",)?;}+ // Multiple service controllers may prepare the same immutable child type.+ // Serialize check-and-alter migrations so a duplicate initializer cannot+ // turn valid child provisioning into a permanent definition failure.+ db.execute_batch("COMMIT")?;Ok(Self {scope: root.canonicalize()?.to_string_lossy().into(),state: Mutex::new(State {diff --git a/harness/runtime/src/lib.rs b/harness/runtime/src/lib.rsindex 4148484e..d8894a41 100644--- a/harness/runtime/src/lib.rs+++ b/harness/runtime/src/lib.rs@@ -71,3 +71,5 @@ pub use openworker_compliance as compliance;pub mod compliance_export;pub mod subagents;++pub mod observation_links;diff --git a/harness/runtime/src/observation_links.rs b/harness/runtime/src/observation_links.rsnew file mode 100644index 00000000..79d4ed54--- /dev/null+++ b/harness/runtime/src/observation_links.rs@@ -0,0 +1,118 @@+//! Durable spawn provenance. Contains identifiers only, never a child brief.+use serde::{Deserialize, Serialize};+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct SpawnOrigin {+ pub parent_session_id: String,+ pub parent_turn_id: String,+ pub parent_call_id: Option<String>,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct ChildLink {+ pub origin: SpawnOrigin,+ pub child_session_id: String,+ /// First accepted task only; later inbox turns are not charged to the spawn.+ #[serde(default)]+ pub child_turn_id: Option<String>,+ pub agent_id: String,+ pub phase: crate::team_coordination::Phase,+}+pub fn children(doc: &crate::agent_team::Document, session: &str, turn: &str) -> Vec<ChildLink> {+ doc.trace_origins+ .iter()+ .filter(|(_, o)| o.parent_session_id == session && o.parent_turn_id == turn)+ .filter_map(|(id, origin)| {+ doc.roster.instance(id).map(|i| ChildLink {+ origin: origin.clone(),+ child_session_id: i.session_id.clone(),+ child_turn_id: None,+ agent_id: id.clone(),+ phase: i.phase,+ })+ })+ .collect()+}++/// A bounded selection of causally linked initial tasks. No transcript is read.+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]+pub struct Run {+ pub session_id: String,+ pub turn_id: String,+}+#[derive(Debug, Serialize, Deserialize)]+pub struct Subtree {+ pub runs: Vec<Run>,+ pub selection_truncated: bool,+ pub scope: String,+}+pub fn subtree(+ root: Run,+ mut children: impl FnMut(&Run) -> anyhow::Result<Vec<ChildLink>>,+) -> anyhow::Result<Subtree> {+ let mut runs = vec![root.clone()];+ let mut seen = std::collections::BTreeSet::from([root]);+ let mut truncated = false;+ let mut index = 0;+ while index < runs.len() {+ for child in children(&runs[index])? {+ let Some(turn_id) = child.child_turn_id else {+ continue;+ };+ let run = Run {+ session_id: child.child_session_id,+ turn_id,+ };+ if seen.contains(&run) {+ continue;+ }+ if runs.len() == 25 {+ truncated = true;+ continue;+ }+ seen.insert(run.clone());+ runs.push(run);+ }+ index += 1;+ }+ Ok(Subtree {+ runs,+ selection_truncated: truncated,+ scope: "root_and_visible_spawned_initial_turns".into(),+ })+}+#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn nested_links_are_bounded_and_cycles_do_not_double_count() {+ let root = Run {+ session_id: "s0".into(),+ turn_id: "t".into(),+ };+ let result = subtree(root, |r| {+ Ok((0..40)+ .map(|n| ChildLink {+ origin: SpawnOrigin {+ parent_session_id: r.session_id.clone(),+ parent_turn_id: r.turn_id.clone(),+ parent_call_id: None,+ },+ child_session_id: format!("s{n}"),+ child_turn_id: Some("t".into()),+ agent_id: format!("a{n}"),+ phase: crate::team_coordination::Phase::Ready,+ })+ .collect())+ })+ .unwrap();+ assert_eq!(result.runs.len(), 25);+ assert!(result.selection_truncated);+ assert_eq!(+ result+ .runs+ .iter()+ .collect::<std::collections::BTreeSet<_>>()+ .len(),+ 25+ );+ }+}diff --git a/harness/runtime/src/scoped_store.rs b/harness/runtime/src/scoped_store.rsindex 6338e17f..89819174 100644--- a/harness/runtime/src/scoped_store.rs+++ b/harness/runtime/src/scoped_store.rs@@ -82,6 +82,10 @@ impl StateStore for ScopedStore {.unwrap().assistant_call(sid, tid, call, args)}+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {+ self.metadata(sid)?;+ self.inner.lock().unwrap().has_turn(sid, tid)+ }fn agent_run(&self, sid: &str, submission: Option<&str>) -> Result<Option<Value>> {self.metadata(sid)?;self.inner.lock().unwrap().agent_run(sid, submission)diff --git a/harness/runtime/src/store.rs b/harness/runtime/src/store.rsindex 3459ce3a..42a29e72 100644--- a/harness/runtime/src/store.rs+++ b/harness/runtime/src/store.rs@@ -213,6 +213,17 @@ pub struct Event {/// the runtime safe for shared workers. `recover` may only touch sessions owned/// exclusively by this store instance. No automatic replay of tool effects.pub trait StateStore: Send {+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {+ Ok(self.metadata(sid)?.turn.is_some_and(|t| t.id == tid))+ }++ fn observation_children(+ &self,+ _sid: &str,+ _tid: &str,+ ) -> Result<Vec<crate::observation_links::ChildLink>> {+ bail!("Spawn trace queries unsupported")+ }fn pending_observers(&self, _sid: &str) -> Result<Vec<Value>> {Ok(vec![])}@@ -1068,6 +1079,40 @@ impl Store {}impl StateStore for Store {+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {+ self.metadata(sid)?;+ Ok(self.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM submissions WHERE session_id=? AND turn_id=?)",+ params![sid, tid],+ |r| r.get(0),+ )?)+ }++ fn observation_children(+ &self,+ sid: &str,+ tid: &str,+ ) -> Result<Vec<crate::observation_links::ChildLink>> {+ let session = self.metadata(sid)?;+ let Some(member) = session.agent_team else {+ return Ok(vec![]);+ };+ let raw: String = self.db.query_row(+ "SELECT data FROM agent_teams WHERE id=?",+ [member.team_id],+ |r| r.get(0),+ )?;+ let mut children =+ crate::observation_links::children(&serde_json::from_str(&raw)?, sid, tid);+ for child in &mut children {+ child.child_turn_id = self.db.query_row(+ "SELECT turn_id FROM events WHERE session_id=? AND kind='turn_accepted' ORDER BY id LIMIT 1",+ [&child.child_session_id], |r| r.get(0),+ ).optional()?.flatten();+ }+ Ok(children)+ }+fn pending_observers(&self, sid: &str) -> Result<Vec<Value>> {Store::pending_observers(self, sid)}diff --git a/harness/runtime/tests/connections.rs b/harness/runtime/tests/connections.rsindex 3d37a1fa..8ef0f600 100644--- a/harness/runtime/tests/connections.rs+++ b/harness/runtime/tests/connections.rs@@ -465,3 +465,29 @@ fn macos_keychain_roundtrip_and_removal() {.any(|w| w == b"keychain-synthetic-test-key"));assert!(m.key(&selected).is_err());}++#[test]+fn concurrent_controller_connection_cache_initialization() {+ // Two service controllers can prepare the same immutable child definition.+ // Their local connection-cache schema must be initialized atomically.+ for _ in 0..12 {+ let root = tempfile::tempdir().unwrap();+ let barrier = std::sync::Arc::new(std::sync::Barrier::new(4));+ std::thread::scope(|scope| {+ let tasks = (0..4)+ .map(|_| {+ let barrier = barrier.clone();+ let path = root.path();+ scope.spawn(move || {+ barrier.wait();+ openworker_runtime::connections::Manager::open(path)+ .map(|m| m.list().unwrap())+ })+ })+ .collect::<Vec<_>>();+ for task in tasks {+ task.join().unwrap().unwrap();+ }+ });+ }+}diff --git a/harness/server-reference/src/recording_query.rs b/harness/server-reference/src/recording_query.rsindex f9ef1285..112dd0ea 100644--- a/harness/server-reference/src/recording_query.rs+++ b/harness/server-reference/src/recording_query.rs@@ -48,7 +48,7 @@ pub(crate) fn authority(method: &str, path: &str) -> bool {_,"observability",_,- "timeline" | "summary"+ "timeline" | "summary" | "children" | "subtree-summary"])}@@ -70,6 +70,14 @@ pub(crate) fn router(prefix: &str) -> Router<App> {&format!("{prefix}/sessions/:sid/observability/:turn/timeline"),get(timeline),)+ .route(+ &format!("{prefix}/sessions/:sid/observability/:turn/children"),+ get(children),+ )+ .route(+ &format!("{prefix}/sessions/:sid/observability/:turn/subtree-summary"),+ get(subtree_summary),+ ).route(&format!("{prefix}/sessions/:sid/observability/:turn/summary"),get(summary),@@ -267,6 +275,45 @@ async fn summary(let summary = summary::build(&page, canonical(&run));Ok(Json(json!({"run":run,"summary":summary})))}+async fn subtree_summary(+ Extension(app): Extension<App>,+ Path((sid, turn)): Path<(String, String)>,+ Query(q): Query<Summary>,+) -> Result<Json<Value>, ApiError> {+ use openworker_runtime::observation_links::{subtree, Run as LinkRun};+ app.store.observation_run(&sid, &turn)?;+ let selection = subtree(+ LinkRun {+ session_id: sid,+ turn_id: turn,+ },+ |r| app.store.observation_children(&r.session_id, &r.turn_id),+ )?;+ for run in &selection.runs {+ app.store.observation_run(&run.session_id, &run.turn_id)?;+ }+ let page = read(+ &app,+ Read {+ deployment: app.config.schema.clone(),+ scopes: selection+ .runs+ .iter()+ .map(|r| scope(&r.session_id, &r.turn_id))+ .collect(),+ after: 0,+ through: q.through,+ limit: openworker_observability::query::MAX_SUMMARY_STEPS,+ },+ )+ .await?;+ for run in &selection.runs {+ app.store.observation_run(&run.session_id, &run.turn_id)?;+ }+ Ok(Json(+ json!({"selection":selection,"summary":summary::build(&page,Canonical::default())}),+ ))+}async fn grouped(Extension(app): Extension<App>,Query(q): Query<Search>,@@ -329,6 +376,18 @@ async fn grouped())}+async fn children(+ Extension(app): Extension<App>,+ Path((sid, turn)): Path<(String, String)>,+) -> Result<Json<Value>, ApiError> {+ app.store.observation_run(&sid, &turn)?;+ let children = app.store.observation_children(&sid, &turn)?;+ app.store.observation_run(&sid, &turn)?;+ Ok(Json(+ json!({"children":children,"coverage":"authorized_captured_spawn_origins_only"}),+ ))+}+#[cfg(test)]mod tests {use super::*;@@ -341,6 +400,8 @@ mod tests {"/observability/health","/sessions/s/observability/t/timeline","/sessions/s/observability/t/summary",+ "/sessions/s/observability/t/children",+ "/sessions/s/observability/t/subtree-summary",] {let path = format!("{prefix}{path}");assert!(authority("GET", &path));diff --git a/harness/server-reference/tests/service_v2.rs b/harness/server-reference/tests/service_v2.rsindex a0e5fa22..2431c1fb 100644--- a/harness/server-reference/tests/service_v2.rs+++ b/harness/server-reference/tests/service_v2.rs@@ -165,6 +165,13 @@ async fn qualify(standalone: bool, live: bool) {.unwrap();let mut value = json!({"service":{"id":"m2-test","namespace":"test","agent":"builder"},"database_env":"OW_PG_TEST_URL","schema":format!("ow_service_{}",uuid::Uuid::new_v4().simple()),"observability":{"schema":format!("ow_record_{}",uuid::Uuid::new_v4().simple())},"database_loopback_plaintext":true,"state_root":root.path().join("state"),"bundle":bundle,"runner":std::env::var("OW_PROBE_RUNNER").unwrap(),"model":"fixture","scripted":true,"api_token_file":token_file,"actor":"tester","openshell":{"identity":"m2-test","endpoint":std::env::var("OW_PROBE_ENDPOINT").unwrap(),"certificates":std::env::var("OW_PROBE_CERTS").unwrap(),"tls_server_name":std::env::var("OW_PROBE_TLS_NAME").ok(),"workspace":"default","image":std::env::var("OW_PROBE_IMAGE").unwrap()}});+ if let Ok(endpoint) = std::env::var("OW_TEST_OTLP_ENDPOINT") {+ assert!(+ endpoint.starts_with("http://127.0.0.1:"),+ "Qualification exporter must be loopback"+ );+ value["telemetry"] = json!({"endpoint":endpoint,"signals":["traces"],"acknowledgement":"langfuse_v4","headers":[{"name":"Authorization","value_env":"OW_TEST_OTLP_AUTH"},{"name":"x-langfuse-ingestion-version","value":"4"}],"content":{"capture":"metadata"}});+ }if live {value["model"] = json!(std::env::var("OW_STANDALONE_LIVE_MODEL").expect("Provide an authorized model ID")diff --git a/harness/server-reference/tests/support/service_subagents.rs b/harness/server-reference/tests/support/service_subagents.rsindex f03db1ec..3734636c 100644--- a/harness/server-reference/tests/support/service_subagents.rs+++ b/harness/server-reference/tests/support/service_subagents.rs@@ -51,6 +51,52 @@ pub(super) async fn qualify(.await.unwrap();assert_eq!(run.result().await.unwrap().status, "completed");+ let links = service+ .json(+ "GET",+ &format!(+ "/v1/sessions/{}/observability/{}/children",+ parent.id(),+ run.id+ ),+ None,+ None,+ )+ .await+ .unwrap();+ assert_eq!(links["children"].as_array().unwrap().len(), 3);+ for link in links["children"].as_array().unwrap() {+ assert_eq!(link["origin"]["parent_turn_id"], run.id);+ assert!(link["origin"]["parent_call_id"].is_string());+ }+ // A parent-only read credential cannot discover child identifiers or usage.+ let cfg: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();+ let admin_token = std::fs::read_to_string(cfg["api_token_file"].as_str().unwrap()).unwrap();+ let read_token = uuid::Uuid::new_v4().simple().to_string().repeat(2);+ _db.authorized(admin_token.trim()).unwrap().auth_issue("trace-parent-only",serde_json::from_value(json!({"principal":"trace-reader","role":"observer","scope":{"kind":"sessions","ids":[parent.id()]}})).unwrap(),&read_token).unwrap();+ let restricted = _db.authorized(&read_token).unwrap();+ assert!(restricted+ .observation_children(parent.id(), &run.id)+ .unwrap()+ .is_empty());+ let initial_scope = format!("/v1/sessions/{}/observability/{}", parent.id(), run.id);+ let initial_tree = service+ .json(+ "GET",+ &format!("{initial_scope}/subtree-summary"),+ None,+ None,+ )+ .await+ .unwrap();+ assert_eq!(+ initial_tree["selection"]["runs"].as_array().unwrap().len(),+ 4+ );+ assert!(initial_tree["summary"]["wall_elapsed_us"].is_null());+ for link in links["children"].as_array().unwrap() {+ assert!(link["child_turn_id"].is_string());+ }let original = doc(config, parent.id());assert!(original.standalone.is_some());assert_eq!(original.roster.instances().count(), 4);@@ -181,6 +227,50 @@ pub(super) async fn qualify(.unwrap().iter().any(|e| e["type"] == "tool_started"));+ // Follow-up messages must not change the initial spawn selection.+ let after_tree = service+ .json(+ "GET",+ &format!("{initial_scope}/subtree-summary"),+ None,+ None,+ )+ .await+ .unwrap();+ assert_eq!(after_tree["selection"], initial_tree["selection"]);+ if let Ok(destination) = std::env::var("OW_OBSERVABILITY_EVIDENCE") {+ let timeline = service+ .json(+ "GET",+ &format!("{initial_scope}/timeline?limit=200"),+ None,+ None,+ )+ .await+ .unwrap();+ let summary = service+ .json("GET", &format!("{initial_scope}/summary"), None, None)+ .await+ .unwrap();+ let mut child_traces = Vec::new();+ for link in links["children"].as_array().unwrap() {+ let base = format!(+ "/v1/sessions/{}/observability/{}",+ link["child_session_id"].as_str().unwrap(),+ link["child_turn_id"].as_str().unwrap()+ );+ let timeline = service+ .json("GET", &format!("{base}/timeline?limit=200"), None, None)+ .await+ .unwrap();+ let summary = service+ .json("GET", &format!("{base}/summary"), None, None)+ .await+ .unwrap();+ child_traces.push(json!({"timeline":timeline,"summary":summary}));+ }+ std::fs::write(destination,serde_json::to_vec_pretty(&json!({"timeline":timeline,"summary":summary,"children":links,"subtree":after_tree,"childTraces":child_traces})).unwrap()).unwrap();+ }eprintln!("PASS standalone service releases: two async builders, sync restricted verifier, shared OpenShell, worker restart, durable follow-up");}diff --git a/harness/service-ui/public/app.js b/harness/service-ui/public/app.jsindex 3608b984..253854a1 100644--- a/harness/service-ui/public/app.js+++ b/harness/service-ui/public/app.js@@ -563,6 +563,7 @@ async function render() {request: (path, signal) =>api(path, "GET", undefined, undefined, signal),isAdmin: can.admin(),+ children: true,});return;}diff --git a/harness/service-ui/public/observability.js b/harness/service-ui/public/observability.jsindex ccd78ea6..9573abf3 100644--- a/harness/service-ui/public/observability.js+++ b/harness/service-ui/public/observability.js@@ -116,7 +116,11 @@ const stepKind = (s) =>? "question_wait": "other_active";-export function mountObservability(root, { request, isAdmin = false }) {+let viewerSequence = 0;+export function mountObservability(root, { request, isAdmin = false, initialRun = null, directory = true, openSession = true, children = false }) {+ const instance = `ow-trace-${++viewerSequence}`;+ const domId = (id) => `${instance}-${id}`;+ const ids = (html) => html.replace(/(id|for|aria-controls|aria-labelledby)="(obs[^" ]*)"/g, (_, attr, id) => `${attr}="${domId(id)}"`);const lifetime = new AbortController();let stopped = false,serial = 0,@@ -124,7 +128,8 @@ export function mountObservability(root, { request, isAdmin = false }) {timer,loadController,searchController;- let chosen = null,+ let childLinks=[], ancestors=[], subtree=null;+ let chosen = initialRun,tab = "usage",summary = null,selection = null,@@ -155,8 +160,10 @@ export function mountObservability(root, { request, isAdmin = false }) {controllers.delete(c);}}- root.innerHTML = `<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder & exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`;- const $ = (s) => root.querySelector(s);+ root.innerHTML = ids(`<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder & exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`);+ const $ = (s) => root.querySelector(s.replace(/#(obs[\w-]*)/g, (_, id) => `#${domId(id)}`));+ if (!directory) $(".obs-picker").hidden = true;+ root.addEventListener("ow-trace-refresh", () => load(), {signal:lifetime.signal});const content = $("#obs-content"),input = $("#obs-run-search"),menu = $("#obs-run-menu");@@ -168,7 +175,7 @@ export function mountObservability(root, { request, isAdmin = false }) {}function scope() {$("#obs-scope").innerHTML = chosen- ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc(chosen.agent)} · ${esc(chosen.status)} · ${esc(new Date(chosen.created_ms).toLocaleString())}</span></div><button class="obs-text" data-obs="recent">Recent runs</button><a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a></div>`+ ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc([chosen.agent, chosen.status, known(chosen.created_ms) ? new Date(chosen.created_ms).toLocaleString() : null].filter(Boolean).join(" · "))}</span></div>${directory ? `<button class="obs-text" data-obs="recent">Recent runs</button>` : ""}${openSession ? `<a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a>` : ""}</div>`: `<p class="obs-hint obs-scope">${selection ? `${selection.selected_runs} recent authorized runs${selection.selection_truncated ? " · more available" : ""}. Summary covers this page only.` : "Recent authorized runs"}</p>`;if (summary?.coverage.scan_truncated || summary?.coverage.group_truncated) {$("#obs-scope").insertAdjacentHTML(@@ -176,13 +183,14 @@ export function mountObservability(root, { request, isAdmin = false }) {'<span class="obs-partial">Partial summary</span>',);}- content.setAttribute("aria-labelledby", `obs-${tab}-tab`);+ content.setAttribute("aria-labelledby", domId(`obs-${tab}-tab`));for (const b of root.querySelectorAll("[data-obs=usage],[data-obs=timeline]",)) {const active = b.dataset.obs === tab;b.classList.toggle("active", active);b.setAttribute("aria-selected", String(active));+ b.tabIndex = active ? 0 : -1;}}function draw() {@@ -219,7 +227,7 @@ export function mountObservability(root, { request, isAdmin = false }) {).join("",- )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}`;+ )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}${chosen&&children?`<section class="obs-panel"><h2>Sub-agents</h2>${ancestors.length?`<button class="button" data-obs="parent">Back to parent turn</button>`:""}${childLinks.map((child,i)=>`<button class="button" data-obs-child="${i}">${esc(child.agent_id)} · ${esc(child.phase)}</button>`).join(" ")||hint("No visible recorded child links for this turn.")}${subtree ? hint(`Linked work: ${count(subtree.selection.runs.length)} visible initial tasks${subtree.selection.selection_truncated ? " · truncated" : ""} · ${measurement(subtree.summary.usage.input)} input / ${measurement(subtree.summary.usage.output)} output tokens. Later inbox turns are separate.`) : ""}${hint("Usage above covers this turn. Open a child to inspect its own requests and usage.")}</section>`:""}`;widths();}function usageView(s) {@@ -274,7 +282,7 @@ export function mountObservability(root, { request, isAdmin = false }) {})).filter((r) => r.value > 0),"Run phase duration",- )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="obs-step-detail">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="obs-timeline-error" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;+ )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="${domId("obs-step-detail")}">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="${domId("obs-timeline-error")}" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;}function stepDetail(r) {const s = r.step;@@ -338,6 +346,12 @@ export function mountObservability(root, { request, isAdmin = false }) {selection = value;summary = selection.summary;}+ childLinks=[];subtree=null;+ if(chosen && children) {+ try {const links=await read(perRun(chosen)+"/children",c); if(stopped||n!==serial)return;childLinks=links.children||[];} catch {childLinks=[];}+ if (stopped || n !== serial) return;+ try {const value=await read(perRun(chosen)+"/subtree-summary?"+query({through:timelineThrough}),c);if(stopped||n!==serial)return;subtree=value;} catch {subtree=null;}+ }draw();} catch (e) {if (stopped || n !== serial || c.signal.aborted) return;@@ -356,12 +370,12 @@ export function mountObservability(root, { request, isAdmin = false }) {$("#obs-run-options").innerHTML = results.map((r, i) =>- `<button type="button" class="obs-run-option" role="option" id="obs-option-${i}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,+ `<button type="button" class="obs-run-option" role="option" id="${domId(`obs-option-${i}`)}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,).join("");$("#obs-search-more").hidden = !resultNext;if (focusedIndex >= 0)- input.setAttribute("aria-activedescendant", `obs-option-${focusedIndex}`);+ input.setAttribute("aria-activedescendant", domId(`obs-option-${focusedIndex}`));else input.removeAttribute("aria-activedescendant");}async function search(more = false) {@@ -401,7 +415,8 @@ export function mountObservability(root, { request, isAdmin = false }) {$("#obs-search-state").textContent = "Unable to search runs. Try again.";}}- function choose(run) {+ function choose(run, keepParents=false) {+ if(!keepParents) ancestors=[];if (!run) return;chosen = run;tool = null;@@ -498,14 +513,29 @@ export function mountObservability(root, { request, isAdmin = false }) {draw();return;}+ const childButton=e.target.closest("[data-obs-child]");+ if(childButton){+ const child=childLinks[Number(childButton.dataset.obsChild)];if(!child)return;+ const selectedSerial=serial;const selectedParent=chosen;+ if(child.child_turn_id){ancestors.push(selectedParent);choose({session_id:child.child_session_id,turn_id:child.child_turn_id},true);return;}+ childButton.disabled=true;+ const c=controller();+ read("/v1/observability/runs?"+query({q:child.child_session_id,limit:100}),c).then(page=>{+ if(stopped||serial!==selectedSerial)return;+ const run=child.child_turn_id ? {session_id:child.child_session_id,turn_id:child.child_turn_id} : page.runs.find(r=>r.session_id===child.child_session_id);+ if(run){ancestors.push(selectedParent);choose(run,true);}else{content.prepend(notice("No accessible child turn is available yet."));}+ }).catch(()=>{if(!stopped)content.prepend(notice("Child traces are unavailable."));}).finally(()=>{childButton.disabled=false;});+ return;+ }const action = e.target.closest("[data-obs]")?.dataset.obs;+ if(action==="parent" && ancestors.length){choose(ancestors.pop(),true);return;}if (action === "refresh") {health = null;healthError = false;if ($("#obs-health")?.open) loadHealth();load();}- if (action === "recent") {+ if (action === "recent" && directory) {chosen = null;tool = null;tab = "usage";@@ -534,6 +564,15 @@ export function mountObservability(root, { request, isAdmin = false }) {},{ signal: lifetime.signal },);+ root.addEventListener("keydown", e => {+ if (e.target.getAttribute?.("role") !== "tab" || !["ArrowLeft", "ArrowRight", "Home", "End"].includes(e.key)) return;+ const tabs = [...root.querySelectorAll('[role="tab"]')];+ let index = tabs.indexOf(e.target);+ if (e.key === "Home") index = 0;+ else if (e.key === "End") index = tabs.length - 1;+ else index = (index + (e.key === "ArrowRight" ? 1 : -1) + tabs.length) % tabs.length;+ e.preventDefault(); tab = tabs[index].dataset.obs; draw(); tabs[index].focus();+ }, {signal:lifetime.signal});input.addEventListener("input",() => {diff --git a/harness/service-ui/server.mjs b/harness/service-ui/server.mjsindex c757f506..92e6b2d9 100644--- a/harness/service-ui/server.mjs+++ b/harness/service-ui/server.mjs@@ -104,8 +104,10 @@ function allowedRecording(path) {`^/v1/sessions/${id}/observability/${id}/timeline$`,).test(u.pathname);const summary = new RegExp(- `^/v1/sessions/${id}/observability/${id}/summary$`,+ `^/v1/sessions/${id}/observability/${id}/(?:summary|subtree-summary)$`,).test(u.pathname);+ const children = new RegExp(`^/v1/sessions/${id}/observability/${id}/children$`).test(u.pathname);+ if(children) return !u.search;const health = u.pathname === "/v1/observability/health";if (!listing && !timeline && !summary && !health) return false;try {diff --git a/harness/store-postgres/src/db.rs b/harness/store-postgres/src/db.rsindex 9a43b713..badeae80 100644--- a/harness/store-postgres/src/db.rs+++ b/harness/store-postgres/src/db.rs@@ -509,6 +509,23 @@ impl Database {if tx.execute("INSERT INTO sessions(id,data,owner,epoch,expires) VALUES($1,'{}',$2,1,clock_timestamp()+$3::bigint*interval '1 millisecond') ON CONFLICT DO NOTHING", &[&s.id,&self.owner,&self.lease_ms])?!=1 {bail!("Session already exists; claim it explicitly");}1};+ // Controllers acquire team -> environment. Terminal reconciliation also+ // needs the team row, so take it BEFORE write_metadata locks the bound+ // environment. Otherwise a child save and a reception controller can+ // deadlock (environment -> team versus team -> environment). This is a+ // lock-order fix, not permission to retry any uncertain external action.+ if s.closed+ || s.turn+ .as_ref()+ .is_some_and(|t| openworker_runtime::store::terminal(&t.status))+ {+ if let Some(member) = &s.agent_team {+ tx.query_one(+ "SELECT data FROM agent_teams WHERE id=$1 FOR UPDATE",+ &[&member.team_id],+ )?;+ }+ }append_history(&mut tx, s)?;write_metadata(&mut tx, s)?;openworker_runtime::subagents::terminal_saved(&mut crate::agent_team::Tx(&mut tx), s)?;diff --git a/harness/store-postgres/src/lib.rs b/harness/store-postgres/src/lib.rsindex 8e63150e..7e216238 100644--- a/harness/store-postgres/src/lib.rs+++ b/harness/store-postgres/src/lib.rs@@ -583,6 +583,16 @@ impl StateStore for PostgresStore {) -> Result<Value> {PostgresStore::compliance_page(self, sid, q)}+ fn observation_children(+ &self,+ sid: &str,+ tid: &str,+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {+ PostgresStore::observation_children(self, sid, tid)+ }+ fn has_turn(&self, sid: &str, tid: &str) -> Result<bool> {+ self.observation_run(sid, tid).map(|_| true)+ }fn supports_agent_teams(&self) -> bool {true}diff --git a/harness/store-postgres/src/run_query.rs b/harness/store-postgres/src/run_query.rsindex 026ebe30..3ab8ae91 100644--- a/harness/store-postgres/src/run_query.rs+++ b/harness/store-postgres/src/run_query.rs@@ -209,3 +209,52 @@ impl PostgresStore {self.call(move |db| db.observation_run(&sid, &tid))}}++impl Database {+ pub(crate) fn observation_children(+ &mut self,+ sid: &str,+ tid: &str,+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {+ let mut tx = self.client.transaction()?;+ auth::guard(&mut tx, self.auth.as_ref(), Action::Read, Some(sid), None)?;+ let row=tx.query_opt("SELECT t.data FROM sessions s JOIN agent_teams t ON t.id=s.data->'agent_team'->>'team_id' WHERE s.id=$1",&[&sid])?;+ let mut visible = Vec::new();+ if let Some(row) = row {+ let doc = serde_json::from_value(row.get(0))?;+ for mut child in openworker_runtime::observation_links::children(&doc, sid, tid) {+ // Parent visibility never grants access to a child's identifiers or usage.+ match auth::guard(+ &mut tx,+ self.auth.as_ref(),+ Action::Read,+ Some(&child.child_session_id),+ None,+ ) {+ Ok(_) => {+ child.child_turn_id=tx.query_opt("SELECT turn_id FROM submissions WHERE session_id=$1 AND turn_id IS NOT NULL ORDER BY sequence LIMIT 1", &[&child.child_session_id])?.and_then(|r|r.get(0));+ visible.push(child);+ }+ Err(e)+ if matches!(+ e.downcast_ref::<auth::Error>(),+ Some(auth::Error::Forbidden)+ ) => {}+ Err(e) => return Err(e),+ }+ }+ }+ tx.commit()?;+ Ok(visible)+ }+}+impl PostgresStore {+ pub fn observation_children(+ &self,+ sid: &str,+ tid: &str,+ ) -> Result<Vec<openworker_runtime::observation_links::ChildLink>> {+ let (sid, tid) = (sid.to_owned(), tid.to_owned());+ self.call(move |db| db.observation_children(&sid, &tid))+ }+}diff --git a/harness/store-postgres/tests/environments.rs b/harness/store-postgres/tests/environments.rsindex 2581d7fc..fc7fb7e2 100644--- a/harness/store-postgres/tests/environments.rs+++ b/harness/store-postgres/tests/environments.rs@@ -421,3 +421,68 @@ async fn resilience_recovered_runtime_shutdown_retires_old_attachments_without_d);assert!(!next.get("recovered").unwrap().closed);}++#[test]+#[ignore = "requires isolated OW_PG_TEST_URL"]+fn terminal_team_save_does_not_hold_environment_while_waiting_for_team() {+ let c = config();+ let mut store = PostgresStore::open(c.clone()).unwrap();+ let binding = ready(&store, Sharing::Shared, Management::Harness);+ let lead = session("lock-order-lead", binding.clone());+ store.save(&lead, "created", json!({})).unwrap();+ let team = store+ .agent_team(+ &lead.id,+ "create",+ &json!({"agent_id":"lead","agent_types":["builder"]}),+ None,+ )+ .unwrap();+ let team_id = team["id"].as_str().unwrap().to_owned();+ let mut closed = store.get(&lead.id).unwrap();+ closed.closed = true;+ let mut controller = admin(&c);+ let mut monitor = admin(&c);+ let mut tx = controller.transaction().unwrap();+ tx.query_one(+ "SELECT data FROM agent_teams WHERE id=$1 FOR UPDATE",+ &[&team_id],+ )+ .unwrap();+ let worker = std::thread::spawn(move || store.save(&closed, "session_closed", json!({})));+ // Wait until this exact schema's worker is blocked on the held team row.+ let relation = format!("{}.agent_teams", c.schema);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let waiting = loop {+ let blocked:bool=monitor.query_one("SELECT EXISTS(SELECT 1 FROM pg_stat_activity a JOIN pg_locks l ON l.pid=a.pid WHERE l.relation=$1::text::regclass AND a.wait_event_type='Lock' AND a.query LIKE '%agent_teams%FOR UPDATE%')",&[&relation]).unwrap().get(0);+ if blocked {+ break true;+ }+ if std::time::Instant::now() > deadline {+ break false;+ }+ std::thread::sleep(std::time::Duration::from_millis(10));+ };+ tx.batch_execute("SET LOCAL lock_timeout='500ms'").unwrap();+ let can_read_environment = tx+ .query_one(+ "SELECT data FROM environments WHERE id=$1 FOR SHARE",+ &[&binding.environment_id],+ )+ .is_ok();+ // Always release test locks before joining or asserting, even on failure.+ tx.rollback().unwrap();+ let saved = worker.join().unwrap();+ drop(controller);+ drop(monitor);+ postgres::Client::connect(&c.connection, postgres::NoTls)+ .unwrap()+ .batch_execute(&format!("DROP SCHEMA {} CASCADE", c.schema))+ .unwrap();+ assert!(waiting, "Worker never reached the team lock");+ assert!(+ can_read_environment,+ "Worker held environment while waiting for team: reversed lock order"+ );+ saved.unwrap();+}diff --git a/harness/telemetry/src/config.rs b/harness/telemetry/src/config.rsnew file mode 100644index 00000000..300e8756--- /dev/null+++ b/harness/telemetry/src/config.rs@@ -0,0 +1,108 @@+//! Export configuration remains available without linking any exporter.+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Config {+ /// Collector or vendor OTLP base URL. Signal paths (`/v1/traces` ...) are+ /// appended. A base path is allowed, e.g. `https://cloud.langfuse.com/api/public/otel`.+ pub endpoint: String,+ #[serde(default = "default_deployment")]+ pub deployment: String,+ /// Extra request headers, typically authentication. Requires HTTPS unless+ /// the endpoint is loopback.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ pub headers: Vec<Header>,+ /// Signals to export. Defaults to all three; vendors that only ingest+ /// traces (e.g. Langfuse) should select `traces` only.+ #[serde(default = "all_signals")]+ pub signals: Vec<Signal>,+ /// Explicit receiver dialect; standard OTLP remains the default.+ #[serde(default)]+ pub acknowledgement: Acknowledgement,+ /// Opt-in content capture. Off by default: records stay content-free.+ #[serde(default)]+ pub content: crate::ContentPolicy,+}+impl Default for Config {+ fn default() -> Self {+ Self {+ endpoint: String::new(),+ deployment: default_deployment(),+ headers: vec![],+ signals: all_signals(),+ acknowledgement: Acknowledgement::default(),+ content: crate::ContentPolicy::default(),+ }+ }+}+fn default_deployment() -> String {+ "default".into()+}+fn all_signals() -> Vec<Signal> {+ vec![Signal::Traces, Signal::Metrics, Signal::Logs]+}+#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum Signal {+ Traces,+ Metrics,+ Logs,+}+/// Langfuse v4 currently returns a queue-job JSON receipt for nonempty batches.+/// Select it explicitly; arbitrary HTTP 200 bodies are never assumed successful.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Acknowledgement {+ #[default]+ Otlp,+ LangfuseV4,+}+/// One request header. Exactly one of `value` (non-secret literal, e.g. a+/// protocol version) or `value_env` (name of an environment variable holding+/// the secret, e.g. `Basic <base64>` or an API key) must be set.+#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]+#[serde(deny_unknown_fields)]+pub struct Header {+ pub name: String,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub value: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub value_env: Option<String>,+}+impl Header {+ /// Secret header whose value is read from environment variable `env`.+ pub fn secret(name: impl Into<String>, env: impl Into<String>) -> Self {+ Self {+ name: name.into(),+ value_env: Some(env.into()),+ ..Self::default()+ }+ }+ /// Non-secret literal header.+ pub fn literal(name: impl Into<String>, value: impl Into<String>) -> Self {+ Self {+ name: name.into(),+ value: Some(value.into()),+ ..Self::default()+ }+ }+}++impl Config {+ /// Direct Langfuse v4 export. The trusted host resolves the credential environment variable.+ pub fn langfuse(endpoint: impl Into<String>, authorization_env: impl Into<String>) -> Self {+ Self {+ endpoint: endpoint.into(),+ signals: vec![Signal::Traces],+ acknowledgement: Acknowledgement::LangfuseV4,+ headers: vec![+ Header::secret("Authorization", authorization_env),+ Header::literal("x-langfuse-ingestion-version", "4"),+ ],+ content: crate::ContentPolicy {+ capture: crate::Capture::Metadata,+ ..Default::default()+ },+ ..Default::default()+ }+ }+}diff --git a/harness/telemetry/src/lib.rs b/harness/telemetry/src/lib.rsindex 1fa3aa8e..ad82ef8a 100644--- a/harness/telemetry/src/lib.rs+++ b/harness/telemetry/src/lib.rs@@ -12,6 +12,7 @@ use std::{};mod accounting;pub mod attributes;+pub mod config;mod observation;pub use accounting::{ModelPurpose, TokenUsage};#[cfg(feature = "otel")]@@ -391,6 +392,7 @@ pub trait Sink: Send + 'static {}}enum Message {+ Flush(mpsc::SyncSender<bool>),Record(Record),Observations([Observation; Gauge::ALL.len()], usize),}@@ -485,11 +487,13 @@ impl Telemetry {break;}if !batch.is_empty() {+ let mut acknowledgements = Vec::new();let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {let mut records = Vec::new();let mut observations = Vec::new();for message in batch.drain(..) {match message {+ Message::Flush(ack) => acknowledgements.push(ack),Message::Record(r) => records.push(r),Message::Observations(o, n) => {observations.extend_from_slice(&o[..n])@@ -505,6 +509,9 @@ impl Telemetry {if !ok {s.failures.fetch_add(1, Ordering::Relaxed);}+ for ack in acknowledgements {+ let _ = ack.try_send(ok);+ }batch.clear();}if t.load(Ordering::Acquire) {@@ -532,6 +539,19 @@ impl Telemetry {},))}+ /// Bounded barrier for records already enqueued. Success is not durable or+ /// lossless delivery; inspect counters for earlier drops/rejections. Never retries.+ pub fn flush(&self, budget: Duration) -> bool {+ let Some(inner) = &self.0 else {+ return true;+ };+ let (ack, rx) = mpsc::sync_channel(1);+ if inner.stop.load(Ordering::Acquire) || inner.tx.try_send(Message::Flush(ack)).is_err() {+ return false;+ }+ rx.recv_timeout(budget.min(Duration::from_secs(5)))+ .unwrap_or(false)+ }pub fn enabled(&self) -> bool {self.0.is_some()}diff --git a/harness/telemetry/src/otlp.rs b/harness/telemetry/src/otlp.rsindex a2da5ec7..815b242d 100644--- a/harness/telemetry/src/otlp.rs+++ b/harness/telemetry/src/otlp.rs@@ -14,92 +14,8 @@ use opentelemetry_sdk::{};use std::{sync::Arc, time::Duration};-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]-#[serde(deny_unknown_fields)]-pub struct Config {- /// Collector or vendor OTLP base URL. Signal paths (`/v1/traces` ...) are- /// appended. A base path is allowed, e.g. `https://cloud.langfuse.com/api/public/otel`.- pub endpoint: String,- #[serde(default = "default_deployment")]- pub deployment: String,- /// Extra request headers, typically authentication. Requires HTTPS unless- /// the endpoint is loopback.- #[serde(default, skip_serializing_if = "Vec::is_empty")]- pub headers: Vec<Header>,- /// Signals to export. Defaults to all three; vendors that only ingest- /// traces (e.g. Langfuse) should select `traces` only.- #[serde(default = "all_signals")]- pub signals: Vec<Signal>,- /// Explicit receiver dialect; standard OTLP remains the default.- #[serde(default)]- pub acknowledgement: Acknowledgement,- /// Opt-in content capture. Off by default: records stay content-free.- #[serde(default)]- pub content: crate::ContentPolicy,-}-impl Default for Config {- fn default() -> Self {- Self {- endpoint: String::new(),- deployment: default_deployment(),- headers: vec![],- signals: all_signals(),- acknowledgement: Acknowledgement::default(),- content: crate::ContentPolicy::default(),- }- }-}-fn default_deployment() -> String {- "default".into()-}-fn all_signals() -> Vec<Signal> {- vec![Signal::Traces, Signal::Metrics, Signal::Logs]-}-#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]-#[serde(rename_all = "lowercase")]-pub enum Signal {- Traces,- Metrics,- Logs,-}-/// Langfuse v4 currently returns a queue-job JSON receipt for nonempty batches.-/// Select it explicitly; arbitrary HTTP 200 bodies are never assumed successful.-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]-#[serde(rename_all = "snake_case")]-pub enum Acknowledgement {- #[default]- Otlp,- LangfuseV4,-}-/// One request header. Exactly one of `value` (non-secret literal, e.g. a-/// protocol version) or `value_env` (name of an environment variable holding-/// the secret, e.g. `Basic <base64>` or an API key) must be set.-#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]-#[serde(deny_unknown_fields)]-pub struct Header {- pub name: String,- #[serde(default, skip_serializing_if = "Option::is_none")]- pub value: Option<String>,- #[serde(default, skip_serializing_if = "Option::is_none")]- pub value_env: Option<String>,-}+pub use crate::config::{Acknowledgement, Config, Header, Signal};impl Header {- /// Secret header whose value is read from environment variable `env`.- pub fn secret(name: impl Into<String>, env: impl Into<String>) -> Self {- Self {- name: name.into(),- value_env: Some(env.into()),- ..Self::default()- }- }- /// Non-secret literal header.- pub fn literal(name: impl Into<String>, value: impl Into<String>) -> Self {- Self {- name: name.into(),- value: Some(value.into()),- ..Self::default()- }- }fn resolve(&self) -> Result<(http::HeaderName, http::HeaderValue)> {let name = http::HeaderName::from_bytes(self.name.trim().to_ascii_lowercase().as_bytes()).context("Invalid telemetry header name")?;diff --git a/harness/telemetry/tests/boundaries.rs b/harness/telemetry/tests/boundaries.rsindex 7415b492..dbd403bb 100644--- a/harness/telemetry/tests/boundaries.rs+++ b/harness/telemetry/tests/boundaries.rs@@ -232,3 +232,19 @@ fn shared_registry_bounds_names_and_content_and_reserves_typed_fields() {.iter().all(|(k, v)| export_attribute(k, v) && v.len() <= 67));}++#[test]+fn flush_is_a_bounded_barrier_without_stopping_instrumentation() {+ let rows = Arc::new(Mutex::new(vec![]));+ let (t, c) = Telemetry::start(Capture(rows.clone()), 32).unwrap();+ t.event(Operation::Command, Outcome::Ok, None, None);+ assert!(t.flush(Duration::from_secs(2)));+ assert_eq!(rows.lock().unwrap().len(), 1);+ t.event(Operation::Command, Outcome::Ok, None, None);+ assert!(t.flush(Duration::from_secs(2)));+ assert_eq!(rows.lock().unwrap().len(), 2);+ assert_eq!(t.snapshot().accepted, 2);+ assert!(c.shutdown(Duration::from_secs(2)));+ assert!(!t.flush(Duration::from_millis(10)));+ assert!(Telemetry::default().flush(Duration::ZERO));+}diff --git a/harness/tests/sdk-storage-shard.mjs b/harness/tests/sdk-storage-shard.mjsindex a635fba9..8b2c41ad 100644--- a/harness/tests/sdk-storage-shard.mjs+++ b/harness/tests/sdk-storage-shard.mjs@@ -27,7 +27,9 @@ if (process.argv.includes("--list")) {if (result.error) throw result.error;if (result.status !== 0) process.exit(result.status ?? 1);};- run(["test", "--locked", "-p", "openworker-sdk", "--features", "test-support",+ process.env.OW_OBSERVABILITY_TEST_URL = process.env.OW_PG_TEST_URL;+ const features = shard === "core" ? "test-support,recording-postgres,evaluation-export" : shard === "teams" ? "test-support,recording" : "test-support";+ run(["test", "--locked", "-p", "openworker-sdk", "--features", features,...(shard === "core" ? ["--lib", "--bins"] : []),...groups[shard].flatMap((name) => ["--test", name]),"--", "--include-ignored", "--test-threads=1"]);diff --git a/harness/trace-viewer/LICENSE b/harness/trace-viewer/LICENSEnew file mode 100644index 00000000..60d38246--- /dev/null+++ b/harness/trace-viewer/LICENSE@@ -0,0 +1,18 @@+MIT License++Copyright (c) 2024 Andrew Ng++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and+associated documentation files (the "Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the+following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial+portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.diff --git a/harness/trace-viewer/README.md b/harness/trace-viewer/README.mdnew file mode 100644index 00000000..8ccb084d--- /dev/null+++ b/harness/trace-viewer/README.md@@ -0,0 +1,50 @@+# Optional turn trace viewer++This standalone browser component uses the management UI's timeline, usage and+model/tool summaries. It adds no dependency to the Rust harness or TypeScript SDK.+It is a local source package pending release review, not a published npm package.++```js+import {mountTurnTrace} from './src/index.js';+import './src/style.css';++const view = mountTurnTrace(document.querySelector('#trace'), {+ adapter, // timeline(query), summary(query); see src/index.d.ts+ sessionId, turnId,+ onChild(child) { openTrace(child.child_session_id, child.child_turn_id); },+});+// On route change: view.dispose(). To reload: view.refresh().+```++The browser adapter calls your application's authenticated API. Keep harness,+Langfuse and database credentials in the trusted backend. Check session ownership+on every request. Optional adapter methods add `children`, `subtreeSummary` and+`evaluations` panels. The viewer never starts an agent or calls an evaluator.++Each instance has independent DOM IDs, requests and disposal. Timeline pages load+100 records at a time, to a browser cap of 1,000. Summaries use the server's separate+bounded scan; missing counters and truncation remain visible. Evaluations show the+first 25 results; later pages remain available through the SDK query API.++Linked work means the root turn and authorized spawned initial turns (up to 25),+not every later message in those child sessions. Tokens count model attempts once;+cache/reasoning counts are subsets. Concurrent task durations are not wall time.++`node sync.mjs` copies the canonical modules into the management UI's static+assets. Run `node sync.mjs --check` to catch drift. These are generated copies;+edit this package's source and sync it instead of maintaining two viewers.++For a loopback preview of qualification data:++```sh+node examples/serve.mjs /absolute/path/to/qualification.json 59602+```++Browser qualification (using an existing Playwright installation):++```sh+node test/browser.mjs http://127.0.0.1:59602 /absolute/path/to/playwright/index.mjs /tmp/trace-screenshots+```++This checks keyboard tabs, exact child navigation, independent mounted instances,+HTML escaping, disposal, and the 1,000-record pagination limit.diff --git a/harness/trace-viewer/examples/index.html b/harness/trace-viewer/examples/index.htmlnew file mode 100644index 00000000..8906d860--- /dev/null+++ b/harness/trace-viewer/examples/index.html@@ -0,0 +1,6 @@+<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>OpenWorker · Turn trace</title><link rel="stylesheet" href="/src/style.css"><style>body{margin:0;background:#f6f7f9}header{padding:20px 40px;background:white;border-bottom:1px solid #e7e9ee;font:15px -apple-system,BlinkMacSystemFont,sans-serif;color:#20242b}header span{color:#737b89;margin-left:16px;font-size:13px}main{margin:32px auto;border-radius:16px;max-width:1120px;overflow:hidden;box-shadow:0 4px 24px #17243508}</style><header><strong>OpenWorker</strong><span>SDK turn trace</span></header><main id="trace"></main><script type="module">+import {mountTurnTrace} from '/src/index.js';+const evidence=await fetch('/trace.json').then(r=>r.json());+const adapter={timeline:async()=>evidence.timeline,summary:async()=>evidence.summary,evaluations:async()=>({results:evidence.evaluationRecord?[{cursor:1,evaluation:evidence.evaluationRecord}]:[],next:null})};+window.viewer=mountTurnTrace(document.querySelector('#trace'),{adapter,sessionId:evidence.timeline.run.session_id,turnId:evidence.timeline.run.turn_id});+</script></html>diff --git a/harness/trace-viewer/examples/serve.mjs b/harness/trace-viewer/examples/serve.mjsnew file mode 100644index 00000000..49adec6b--- /dev/null+++ b/harness/trace-viewer/examples/serve.mjs@@ -0,0 +1,10 @@+// Loopback-only viewer for a selected synthetic qualification result.+import {createServer} from 'node:http';+import {readFile} from 'node:fs/promises';+const [evidence,port='59602']=process.argv.slice(2);+if(!evidence)throw Error('Usage: node examples/serve.mjs EVIDENCE_JSON [PORT]');+const files=new Map([['/',new URL('index.html',import.meta.url)],...['index.js','observability.js','observability-data.js','style.css'].map(f=>[`/src/${f}`,new URL(`../src/${f}`,import.meta.url)])]);+const types={js:'text/javascript',html:'text/html',css:'text/css',json:'application/json'};+createServer(async(req,res)=>{+ try {const path=new URL(req.url,'http://local').pathname;const file=path==='/trace.json'?evidence:files.get(path);if(!file){res.writeHead(404);res.end();return;}const body=await readFile(file);res.writeHead(200,{'content-type':path==='/'?'text/html':types[path.split('.').pop()]??'text/plain','cache-control':'no-store'});res.end(body);}catch{res.writeHead(500);res.end('Unavailable');}+}).listen(Number(port),'127.0.0.1',()=>console.log(`Viewer: http://127.0.0.1:${port}`));diff --git a/harness/trace-viewer/package.json b/harness/trace-viewer/package.jsonnew file mode 100644index 00000000..785f5a03--- /dev/null+++ b/harness/trace-viewer/package.json@@ -0,0 +1,25 @@+{+ "name": "@openworkerai/harness-trace-viewer",+ "version": "0.1.0-beta.2",+ "private": true,+ "type": "module",+ "description": "Optional browser trace viewer for OpenWorker SDK applications",+ "exports": {+ ".": {+ "types": "./src/index.d.ts",+ "import": "./src/index.js"+ },+ "./style.css": "./src/style.css"+ },+ "files": [+ "src",+ "README.md",+ "LICENSE"+ ],+ "scripts": {+ "sync": "node sync.mjs",+ "check": "node sync.mjs --check"+ },+ "types": "./src/index.d.ts",+ "license": "MIT"+}diff --git a/harness/trace-viewer/src/index.d.ts b/harness/trace-viewer/src/index.d.tsnew file mode 100644index 00000000..ef2b7af7--- /dev/null+++ b/harness/trace-viewer/src/index.d.ts@@ -0,0 +1,10 @@+export interface TraceAdapter {+ timeline(query: {sessionId:string;turnId:string;after?:number;through?:number;limit?:number}): Promise<unknown>;+ summary(query: {sessionId:string;turnId:string;through?:number}): Promise<unknown>;+ children?(query: {sessionId:string;turnId:string}): Promise<unknown>;+ subtreeSummary?(query: {sessionId:string;turnId:string}): Promise<unknown>;+ evaluations?(query: {sessionId:string;turnId:string;limit?:number}): Promise<unknown>;+}+export interface ChildLink {child_session_id:string;child_turn_id:string|null;agent_id:string;phase:string}+export function mountTurnTrace(root:HTMLElement, options:{adapter:TraceAdapter;sessionId:string;turnId:string;onChild?:(child:ChildLink)=>void}): {refresh():void;dispose():void};+export function mountObservability(root:HTMLElement, options:{request:(path:string,signal?:AbortSignal)=>Promise<any>;isAdmin?:boolean;initialRun?:{session_id:string;turn_id:string}|null;directory?:boolean;openSession?:boolean;children?:boolean}):()=>void;diff --git a/harness/trace-viewer/src/index.js b/harness/trace-viewer/src/index.jsnew file mode 100644index 00000000..968d61f2--- /dev/null+++ b/harness/trace-viewer/src/index.js@@ -0,0 +1,75 @@+import {mountObservability} from './observability.js';+import {count, measurement} from './observability-data.js';+export {mountObservability};++/** Mount one turn. The trusted application owns authorization and transport. */+export function mountTurnTrace(root, {adapter, sessionId, turnId, onChild}) {+ root.classList.add('ow-trace-viewer');+ const query = {sessionId, turnId};+ const request = async path => {+ const url = new URL(path, 'https://viewer.invalid');+ const match = /^\/v1\/sessions\/([^/]+)\/observability\/([^/]+)\/(timeline|summary)$/.exec(url.pathname);+ if (!match || decodeURIComponent(match[1]) !== sessionId || decodeURIComponent(match[2]) !== turnId)+ throw Error('Unsupported trace scope');+ const q = {...query};+ for (const key of ['after', 'through', 'limit'])+ if (url.searchParams.has(key)) q[key] = Number(url.searchParams.get(key));+ return adapter[match[3]](q);+ };+ const unmount = mountObservability(root, {request, initialRun: {session_id: sessionId, turn_id: turnId}, directory: false, openSession: false});+ const extra = document.createElement('div'); root.append(extra);+ let disposed = false, generation = 0;+ const paragraph = text => { const p = document.createElement('p'); p.textContent = text; return p; };+ const section = title => {+ const node = document.createElement('section'); node.className = 'obs-panel';+ const h = document.createElement('h2'); h.textContent = title; node.append(h); return node;+ };+ async function details() {+ const mine = ++generation;+ extra.replaceChildren();+ const tasks = [];+ if (adapter.children) tasks.push(['Sub-agents', async () => {+ const page = await adapter.children(query), node = section('Sub-agents');+ for (const child of page.children) {+ if (onChild && child.child_turn_id) {+ const b = document.createElement('button'); b.className = 'button';+ b.textContent = `${child.agent_id} · ${child.phase}`;+ b.addEventListener('click', () => onChild(child)); node.append(b);+ } else node.append(paragraph(`${child.agent_id} · ${child.phase}`));+ }+ if (!page.children.length) node.append(paragraph('No visible recorded child links for this turn.'));+ return node;+ }]);+ if (adapter.subtreeSummary) tasks.push(['Linked work', async () => {+ const tree = await adapter.subtreeSummary(query), node = section('Linked work');+ node.append(paragraph(`${count(tree.selection.runs.length)} visible initial tasks${tree.selection.selection_truncated ? ' · selection limit reached' : ''}`));+ node.append(paragraph(`Input: ${measurement(tree.summary.usage.input)} · Output: ${measurement(tree.summary.usage.output)} tokens`));+ node.append(paragraph('Includes this turn and spawned initial tasks. Later messages are separate turns; concurrent durations are not elapsed time.'));+ return node;+ }]);+ if (adapter.evaluations) tasks.push(['Evaluations', async () => {+ const page = await adapter.evaluations({...query, limit: 25}), node = section('Evaluations');+ for (const row of page.results) {+ const e = row.evaluation.result;+ const details = document.createElement('details'); details.className = 'obs-details';+ const summary = document.createElement('summary');+ summary.textContent = `${e.name} · ${e.status === 'completed' ? String(e.score?.value) : e.status}`;+ details.append(summary, paragraph(`${e.evaluator.name} · ${e.evaluator.version} · ${e.evaluator.source}`));+ if (e.explanation) details.append(paragraph(e.explanation));+ if (e.supersedes) details.append(paragraph(`Corrects ${e.supersedes}; original evidence retained.`));+ node.append(details);+ }+ if (!page.results.length) node.append(paragraph('No recorded evaluations.'));+ if (page.next !== null) node.append(paragraph('Showing the first 25 evaluations. Use the query API to read later pages.'));+ return node;+ }]);+ const results = await Promise.all(tasks.map(async ([title, task]) => {+ try {return await task();} catch {const node = section(title); node.append(paragraph('Unavailable or not enabled for this application.')); return node;}+ }));+ if (!disposed && mine === generation) extra.replaceChildren(...results);+ }+ const refresh = () => {root.dispatchEvent(new Event('ow-trace-refresh')); details();};+ const click = e => {if (e.target.closest('[data-obs="refresh"]')) details();};+ root.addEventListener('click', click); details();+ return {refresh, dispose() {disposed = true; generation++; root.removeEventListener('click', click); unmount(); root.replaceChildren(); root.classList.remove('ow-trace-viewer');}};+}diff --git a/harness/trace-viewer/src/observability-data.js b/harness/trace-viewer/src/observability-data.jsnew file mode 100644index 00000000..0ce44467--- /dev/null+++ b/harness/trace-viewer/src/observability-data.js@@ -0,0 +1,153 @@+// Pure presentation helpers. Provider-reported values stay optional; they are+// neither a bill nor inferred per-tool token costs.+export const esc = (s) =>+ String(s ?? "").replace(+ /[&<>"']/g,+ (c) =>+ ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[+ c+ ],+ );+export const known = (n) =>+ typeof n === "number" &&+ Number.isFinite(n) &&+ n >= 0 &&+ n <= Number.MAX_SAFE_INTEGER;+export const count = (n) =>+ known(n) ? n.toLocaleString("en-US", { maximumFractionDigits: 0 }) : "—";+export function duration(us) {+ if (!known(us)) return "—";+ if (us === 0) return "0 ms";+ if (us < 1000) return "<1 ms";+ if (us < 1000000)+ return `${(us / 1000).toLocaleString("en-US", { maximumFractionDigits: 1 })} ms`;+ if (us < 60000000)+ return `${(us / 1000000).toLocaleString("en-US", { maximumFractionDigits: 1 })} s`;+ return `${(us / 60000000).toLocaleString("en-US", { maximumFractionDigits: 1 })} min`;+}+export const measurement = (m, formatter = count) =>+ `${formatter(m?.total)}${m?.unknown > 0 && known(m.total) ? " + unknown" : ""}`;+export const average = (m) =>+ m?.known > 0 && known(m.total) ? m.total / m.known : null;+export const runKey = (r) => `${r.session_id}/${r.turn_id}`;+export const purpose = (p) =>+ ({+ main: "Main agent",+ reviewer: "Auto-approver",+ auto_approver: "Auto-approver",+ compaction: "Compaction",+ summary: "Compaction",+ helper: "Helper",+ title: "Title",+ })[p] ||+ p ||+ "Unspecified";+export const phases = [+ ["model", "Model requests"],+ ["tool", "Tool execution"],+ ["policy", "Policy checks"],+ ["approval_wait", "Waiting for approval"],+ ["question_wait", "Waiting for answer"],+ ["other_active", "Other activity"],+ ["parallel", "Parallel work"],+ ["unobserved_gap", "Unobserved gaps"],+];+export function usageTotal(u) {+ // A combined token number is only useful if both counters were reported.+ if (!known(u?.input?.total) || !known(u?.output?.total)) return null;+ const total = u.input.total + u.output.total;+ return Number.isSafeInteger(total) ? total : null;+}+export function purposeUsage(models = []) {+ const groups = new Map();+ for (const m of models) {+ const name = purpose(m.purpose),+ current = groups.get(name) || { name, total: 0, missing: 0, known: 0 };+ const total = usageTotal(m.usage);+ if (known(total)) {+ current.total += total;+ current.known++;+ }+ if (!known(total) || m.usage?.input?.unknown || m.usage?.output?.unknown)+ current.missing++;+ groups.set(name, current);+ }+ return [...groups.values()].map((g) => ({+ ...g,+ total: g.known && Number.isSafeInteger(g.total) ? g.total : null,+ }));+}+export function coverageNotes(c = {}) {+ const notes = [];+ if (!c.scanned_steps) notes.push("No recorded steps for this selection.");+ if (c.scan_truncated) notes.push("The 4,096-step summary limit was reached.");+ if (c.group_truncated)+ notes.push(+ "Only the first 128 model/tool groups are listed; group sums may differ from totals.",+ );+ if (known(c.expected_segments) && c.recorded_segments !== c.expected_segments)+ notes.push(+ `${count(c.recorded_segments)} of ${count(c.expected_segments)} execution segments recorded.`,+ );+ if (c.scanned_steps && !c.all_visible_segments_closed)+ notes.push("Visible execution segments are unfinished or missing.");+ if (c.pending_waits > 0)+ notes.push(`${count(c.pending_waits)} pending human response(s).`);+ if (c.requests_without_completion > 0)+ notes.push(+ `${count(c.requests_without_completion)} model request(s) without a completion record.`,+ );+ if (c.completions_without_start > 0)+ notes.push(+ `${count(c.completions_without_start)} model completion(s) without a start record.`,+ );+ if (c.unknown_timing_steps > 0)+ notes.push(+ `${count(c.unknown_timing_steps)} step(s) without measured duration.`,+ );+ if (c.unknown_outcome_steps > 0)+ notes.push(+ `${count(c.unknown_outcome_steps)} step(s) with unknown outcome.`,+ );+ if (c.clock_or_range_anomaly)+ notes.push("Clock or timing range anomalies were detected.");+ if (c.hierarchy_incomplete)+ notes.push(+ "Parent relationships are missing or cyclic; phase attribution may be incomplete.",+ );+ return notes;+}+export function timelineLayout(records) {+ const steps = records+ .filter((r) => known(r.step.started_ms))+ .slice()+ .sort(+ (a, b) => a.step.started_ms - b.step.started_ms || a.cursor - b.cursor,+ );+ const starts = steps.map((r) => r.step.started_ms * 1000);+ const from = starts.length ? Math.min(...starts) : 0;+ const to = steps.length+ ? Math.max(+ ...steps.map(+ (r) =>+ r.step.started_ms * 1000 ++ (known(r.step.duration_us) ? r.step.duration_us : 0),+ ),+ )+ : from;+ const span = Math.max(1, to - from);+ return {+ from,+ to,+ rows: steps.map((r) => ({+ ...r,+ left: Math.max(+ 0,+ Math.min(100, ((r.step.started_ms * 1000 - from) / span) * 100),+ ),+ width: known(r.step.duration_us)+ ? Math.max(0, Math.min(100, (r.step.duration_us / span) * 100))+ : 0,+ })),+ };+}diff --git a/harness/trace-viewer/src/observability.js b/harness/trace-viewer/src/observability.jsnew file mode 100644index 00000000..9573abf3--- /dev/null+++ b/harness/trace-viewer/src/observability.js@@ -0,0 +1,685 @@+import {+ esc,+ known,+ count,+ duration,+ measurement,+ average,+ runKey,+ purpose,+ phases,+ usageTotal,+ purposeUsage,+ coverageNotes,+ timelineLayout,+} from "./observability-data.js";+const query = (params) =>+ new URLSearchParams(+ Object.entries(params).filter(+ ([, v]) => v !== null && v !== undefined && v !== "",+ ),+ ).toString();+const perRun = (r) =>+ `/v1/sessions/${encodeURIComponent(r.session_id)}/observability/${encodeURIComponent(r.turn_id)}`;+const cell = (value) => `<td>${esc(value)}</td>`;+const hint = (text) => `<p class="obs-hint">${esc(text)}</p>`;+const notice = (text) =>+ `<div class="obs-notice" role="status">${esc(text)}</div>`;+function failure(e) {+ if (e.status === 403) return "This credential cannot read the selected run.";+ if (e.status === 404) return "This run is no longer available.";+ if (e.status === 429)+ return "Recording readers are busy. Try Refresh shortly.";+ if (e.status === 503)+ return "Recordings are unavailable or disabled. You can still search runs.";+ return "Unable to load recordings. Try Refresh.";+}+const rowTable = (headers, rows, label) =>+ `<div class="obs-table-wrap"><table class="table" aria-label="${label}"><thead><tr>${headers.map((h) => `<th>${h}</th>`).join("")}</tr></thead><tbody>${rows.join("") || `<tr><td colspan="${headers.length}" class="obs-empty-cell">No recorded data</td></tr>`}</tbody></table></div>`;+function bars(rows, label, formatter = duration, type = "phase") {+ const max = Math.max(+ 1,+ ...rows.filter((r) => known(r.value)).map((r) => r.value),+ );+ return `<div class="obs-bars" aria-label="${label}">${rows.map((r) => `<div class="obs-bar-row"><span>${esc(r.name)}</span><div class="obs-track"><i class="obs-fill obs-${r.key || type}" data-obs-width="${known(r.value) ? Math.min(100, (r.value / max) * 100) : 0}"></i></div><span class="obs-number">${esc(formatter(r.value))}${r.missing ? "*" : ""}</span></div>`).join("") || hint("No recorded data")}</div>`;+}+function tokensByPurpose(s) {+ const rows = purposeUsage(s.models).map((r) => ({+ ...r,+ value: r.total,+ key: "model",+ }));+ return (+ bars(rows, "Tokens by purpose", count) ++ (s.coverage.group_truncated ? hint("Group list is truncated.") : "") ++ (rows.some((r) => r.missing)+ ? hint("* Some request counters are unknown.")+ : "") ++ hint(+ `Cached input: ${measurement(s.usage.cache_read)} · already included in input.`,+ )+ );+}+function modelTable(s) {+ return rowTable(+ ["Model / purpose", "Calls", "Input", "Output", "First output · avg."],+ s.models.map(+ (m) =>+ `<tr><td>${esc(m.name || "Unspecified")}<span class="cell-subtitle">${esc(purpose(m.purpose))}</span></td>${cell(count(m.calls))}${cell(measurement(m.usage?.input))}${cell(measurement(m.usage?.output))}<td>${esc(duration(average(m.first_output_us)))}${m.first_output_us.unknown ? `<span class="cell-subtitle">${count(m.first_output_us.unknown)} unknown</span>` : ""}</td></tr>`,+ ),+ "Model usage",+ );+}+function toolTable(s) {+ return rowTable(+ ["Tool", "Calls", "Accumulated", "Avg. known", "Result bytes"],+ [...s.tools]+ .sort(+ (a, b) =>+ (b.accumulated_us.total ?? -1) - (a.accumulated_us.total ?? -1),+ )+ .map(+ (t) =>+ `<tr><td><button class="obs-text" data-obs-tool="${esc(t.name || "")}">${esc(t.name || "Unspecified")}</button></td>${cell(count(t.calls))}${cell(measurement(t.accumulated_us, duration))}${cell(duration(average(t.accumulated_us)))}${cell(measurement(t.returned_bytes))}</tr>`,+ ),+ "Tool summary",+ );+}+function coverage(s) {+ const c = s.coverage,+ notes = coverageNotes(c);+ return `<details class="obs-details"><summary>Coverage${notes.length ? ` · ${notes.length} ${notes.length === 1 ? "note" : "notes"}` : ""}</summary><p>${count(c.scanned_steps)} steps scanned. Recording is best effort; missing counters are unknown, not zero.</p>${notes.length ? `<ul>${notes.map((n) => `<li>${esc(n)}</li>`).join("")}</ul>` : ""}<p>Late commits and retention can change a refreshed view. A closed run is not proof of lossless accounting.</p></details>`;+}+const stepName = (s) =>+ s.operation === "model.attempt"+ ? s.model || "Model request"+ : s.operation === "model.requested"+ ? "Request started"+ : s.operation === "tool.execute"+ ? s.tool || "Tool execution"+ : {+ "agent.execute": "Execution segment",+ "policy.review": "Policy check",+ "approval.wait": "Approval wait",+ "question.wait": "Question wait",+ }[s.operation] || s.operation;+const stepKind = (s) =>+ s.operation.startsWith("model.")+ ? "model"+ : s.operation.startsWith("tool.")+ ? "tool"+ : s.operation === "policy.review"+ ? "policy"+ : s.operation === "approval.wait"+ ? "approval_wait"+ : s.operation === "question.wait"+ ? "question_wait"+ : "other_active";++let viewerSequence = 0;+export function mountObservability(root, { request, isAdmin = false, initialRun = null, directory = true, openSession = true, children = false }) {+ const instance = `ow-trace-${++viewerSequence}`;+ const domId = (id) => `${instance}-${id}`;+ const ids = (html) => html.replace(/(id|for|aria-controls|aria-labelledby)="(obs[^" ]*)"/g, (_, attr, id) => `${attr}="${domId(id)}"`);+ const lifetime = new AbortController();+ let stopped = false,+ serial = 0,+ searchSerial = 0,+ timer,+ loadController,+ searchController;+ let childLinks=[], ancestors=[], subtree=null;+ let chosen = initialRun,+ tab = "usage",+ summary = null,+ selection = null,+ timeline = [],+ timelinePage = null,+ timelineThrough = null,+ selectedStep = null,+ tool = null;+ let results = [],+ resultNext = null,+ searchText = "",+ focusedIndex = -1,+ pageCursor = null,+ pageStack = [];+ let health = null,+ healthError = false;+ const controllers = new Set();+ function controller() {+ const c = new AbortController();+ controllers.add(c);+ return c;+ }+ async function read(path, c) {+ controllers.add(c);+ try {+ return await request(path, c.signal);+ } finally {+ controllers.delete(c);+ }+ }+ root.innerHTML = ids(`<section class="observability" id="observability"><div class="page-heading"><div><h1>Observability</h1><p>Where agent time and tokens go.</p></div><button class="button" data-obs="refresh">Refresh</button></div><div class="obs-toolbar"><div class="tabs obs-tabs" role="tablist" aria-label="Observability views"><button class="tab active" role="tab" id="obs-usage-tab" aria-controls="obs-content" aria-selected="true" data-obs="usage">Usage</button><button class="tab" role="tab" id="obs-timeline-tab" aria-controls="obs-content" aria-selected="false" data-obs="timeline">Timeline</button></div><div class="obs-picker"><label for="obs-run-search">Find a run</label><input id="obs-run-search" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-controls="obs-run-options" autocomplete="off" placeholder="Search name or ID…"><div id="obs-run-menu" class="obs-run-menu" hidden><div id="obs-run-options" role="listbox" aria-label="Matching runs"></div><button class="obs-more" id="obs-search-more" hidden>More results</button><p id="obs-search-state" role="status"></p></div></div></div><div id="obs-scope"></div><div id="obs-content" role="tabpanel" aria-labelledby="obs-usage-tab" aria-live="polite"></div><details class="obs-details"><summary>How time and usage are counted</summary><p>Input and output are provider-reported tokens. Cached input and reasoning are subsets, not extra tokens. Tools have duration and result bytes; tokens cannot be attributed exactly to a tool.</p><p>Accumulated work adds individual durations. Exclusive phases partition the observed window, with parallel work shown separately. Unobserved gaps are not assigned to model or tool work. First-output time includes transport and provider queueing, not just inference.</p></details>${isAdmin ? '<details class="obs-details" id="obs-health"><summary>Recorder & exporter health</summary><div id="obs-health-content"></div></details>' : ""}</section>`);+ const $ = (s) => root.querySelector(s.replace(/#(obs[\w-]*)/g, (_, id) => `#${domId(id)}`));+ if (!directory) $(".obs-picker").hidden = true;+ root.addEventListener("ow-trace-refresh", () => load(), {signal:lifetime.signal});+ const content = $("#obs-content"),+ input = $("#obs-run-search"),+ menu = $("#obs-run-menu");+ function widths() {+ for (const el of root.querySelectorAll("[data-obs-width]"))+ el.style.width = `${el.dataset.obsWidth}%`;+ for (const el of root.querySelectorAll("[data-obs-left]"))+ el.style.left = `${el.dataset.obsLeft}%`;+ }+ function scope() {+ $("#obs-scope").innerHTML = chosen+ ? `<div class="obs-scope"><div><strong>${esc(chosen.name || chosen.turn_id)}</strong><span>${esc([chosen.agent, chosen.status, known(chosen.created_ms) ? new Date(chosen.created_ms).toLocaleString() : null].filter(Boolean).join(" · "))}</span></div>${directory ? `<button class="obs-text" data-obs="recent">Recent runs</button>` : ""}${openSession ? `<a class="obs-text" href="#sessions/${encodeURIComponent(chosen.session_id)}">Open session ↗</a>` : ""}</div>`+ : `<p class="obs-hint obs-scope">${selection ? `${selection.selected_runs} recent authorized runs${selection.selection_truncated ? " · more available" : ""}. Summary covers this page only.` : "Recent authorized runs"}</p>`;+ if (summary?.coverage.scan_truncated || summary?.coverage.group_truncated) {+ $("#obs-scope").insertAdjacentHTML(+ "beforeend",+ '<span class="obs-partial">Partial summary</span>',+ );+ }+ content.setAttribute("aria-labelledby", domId(`obs-${tab}-tab`));+ for (const b of root.querySelectorAll(+ "[data-obs=usage],[data-obs=timeline]",+ )) {+ const active = b.dataset.obs === tab;+ b.classList.toggle("active", active);+ b.setAttribute("aria-selected", String(active));+ b.tabIndex = active ? 0 : -1;+ }+ }+ function draw() {+ if (stopped) return;+ scope();+ if (!summary) return;+ const s = summary,+ tokenTotal = usageTotal(s.usage),+ tokensMissing = s.usage.input.unknown + s.usage.output.unknown;+ content.innerHTML = `<div class="stats obs-stats">${[+ [+ chosen ? "Run elapsed" : "Observed window",+ duration(chosen ? s.wall_elapsed_us : s.observed_window_us),+ ],+ [+ "Reported tokens",+ count(tokenTotal) ++ (known(tokenTotal) && tokensMissing ? " + unknown" : ""),+ ],+ [+ "Model requests",+ count(s.models.reduce((n, g) => n + g.calls, 0)) ++ (s.coverage.group_truncated ? "+" : ""),+ ],+ [+ "Tool calls",+ count(s.tools.reduce((n, g) => n + g.calls, 0)) ++ (s.coverage.group_truncated ? "+" : ""),+ ],+ ]+ .map(+ ([label, value]) =>+ `<div class="stat"><div class="stat-label">${label}</div><div class="stat-main"><span class="stat-value">${esc(value)}</span></div></div>`,+ )+ .join(+ "",+ )}</div>${s.coverage.scanned_steps === 0 ? notice("No recorded steps yet. The run directory is still available below.") : ""}${tab === "timeline" ? timelineView(s) : usageView(s)}${coverage(s)}${runDirectory()}${chosen&&children?`<section class="obs-panel"><h2>Sub-agents</h2>${ancestors.length?`<button class="button" data-obs="parent">Back to parent turn</button>`:""}${childLinks.map((child,i)=>`<button class="button" data-obs-child="${i}">${esc(child.agent_id)} · ${esc(child.phase)}</button>`).join(" ")||hint("No visible recorded child links for this turn.")}${subtree ? hint(`Linked work: ${count(subtree.selection.runs.length)} visible initial tasks${subtree.selection.selection_truncated ? " · truncated" : ""} · ${measurement(subtree.summary.usage.input)} input / ${measurement(subtree.summary.usage.output)} output tokens. Later inbox turns are separate.`) : ""}${hint("Usage above covers this turn. Open a child to inspect its own requests and usage.")}</section>`:""}`;+ widths();+ }+ function usageView(s) {+ return `<div class="obs-grid"><section class="obs-panel"><h2>Where time went</h2>${hint("Exclusive phases · observed time")}${bars(+ phases+ .map(([key, name]) => ({+ key,+ name,+ value: s.exclusive_observed_us[key],+ }))+ .filter((r) => r.value > 0),+ "Exclusive phase duration",+ )}${hint(`Observed window: ${duration(s.observed_window_us)}${chosen ? "" : " · union across selected runs, not summed run time"}`)}</section><section class="obs-panel"><h2>Tokens by purpose</h2>${hint("Reported input + output")}${tokensByPurpose(s)}</section></div><div class="obs-grid"><section class="obs-panel"><h2>Accumulated work</h2>${hint("May exceed elapsed time when calls overlap")}${bars(+ [+ {+ name: "Model requests",+ value: s.model_accumulated_us.total,+ key: "model",+ },+ {+ name: "Tool executions",+ value: s.tool_accumulated_us.total,+ key: "tool",+ },+ ],+ "Accumulated work",+ )}${s.model_accumulated_us.unknown || s.tool_accumulated_us.unknown ? hint("Some durations are unknown; bars show measured work only.") : ""}</section><section class="obs-panel"><h2>Model response time</h2>${hint("Average request start → first output · known samples")}${bars(+ s.models.map((m) => ({+ name: `${m.name || "Unspecified"} · ${purpose(m.purpose)}`,+ value: average(m.first_output_us),+ missing: m.first_output_us.unknown,+ key: "model",+ })),+ "Average model first output",+ )}</section></div><section class="obs-panel"><h2>Tools</h2>${hint("Select a tool to inspect its calls in a run")}${toolTable(s)}</section><section class="obs-panel"><h2>Model usage</h2>${modelTable(s)}</section>`;+ }+ function timelineView(s) {+ if (!chosen) return notice("Choose a run to see its timeline.");+ const records = tool+ ? timeline.filter((r) => r.step.tool === tool)+ : timeline;+ const layout = timelineLayout(records);+ const selected = timeline.find(+ (r) => String(r.cursor) === String(selectedStep),+ );+ return `<div class="obs-grid"><section class="obs-panel"><h2>Time by phase</h2>${bars(+ phases+ .map(([key, name]) => ({+ key,+ name,+ value: s.exclusive_observed_us[key],+ }))+ .filter((r) => r.value > 0),+ "Run phase duration",+ )}</section><section class="obs-panel"><h2>Usage by purpose</h2>${tokensByPurpose(s)}</section></div><section class="obs-panel obs-timeline"><div class="obs-panel-title"><div><h2>Turn timeline</h2>${hint(`${timeline.length} loaded records · select a step for details`)}</div>${tool ? `<button class="obs-text" data-obs="clear-tool">Clear tool filter</button>` : ""}</div>${tool ? hint(`Tool: ${tool}`) : ""}<div class="obs-axis"><span>0</span><span>${duration(layout.to - layout.from)}</span></div><div class="obs-lanes">${layout.rows.map((r) => `<button class="obs-lane" data-obs-step="${r.cursor}" aria-pressed="${String(selectedStep) === String(r.cursor)}"><span class="obs-lane-name" title="${esc(stepName(r.step))}">${esc(stepName(r.step))}<small>${esc(r.step.operation === "model.attempt" ? purpose(r.step.purpose) : r.step.outcome)}</small></span><span class="obs-lane-track"><i class="obs-mark obs-${stepKind(r.step)}" data-obs-left="${r.left}" data-obs-width="${r.width}"></i></span><span class="obs-number">${duration(r.step.duration_us)}</span></button>`).join("") || hint("No timeline records in this view.")}</div>${timelinePage?.next !== null && timelinePage?.next !== undefined ? (timeline.length < 1000 ? '<button class="button" data-obs="more-timeline">Load more records</button>' : hint("The browser limit of 1,000 records was reached. The summary uses its separate bounded scan.")) : ""}<div id="${domId("obs-step-detail")}">${selected ? stepDetail(selected) : hint("Select a step to inspect timing, usage and outcome.")}</div><div id="${domId("obs-timeline-error")}" role="status"></div>${hint("Bars use recorded starts and measured durations. Unknown duration is a marker, not zero. Late records can appear before earlier pages.")}</section>`;+ }+ function stepDetail(r) {+ const s = r.step;+ return `<div class="obs-step-detail"><div><strong>${esc(stepName(s))}</strong><span>${esc(s.outcome)} · ${esc(s.operation)}</span></div><dl>${[+ ["Duration", duration(s.duration_us)],+ ["First output", duration(s.first_output_us)],+ ["Input tokens", count(s.usage?.input)],+ ["Output tokens", count(s.usage?.output)],+ ["Result bytes", count(s.output_bytes)],+ ["Attempt", count(s.attempt)],+ ]+ .map(([k, v]) => `<div><dt>${k}</dt><dd>${v}</dd></div>`)+ .join(+ "",+ )}</dl><details><summary>Record details</summary><pre>${esc(JSON.stringify(r, null, 2))}</pre></details></div>`;+ }+ function runDirectory() {+ if (chosen) return "";+ const runs = selection?.selection?.runs || [];+ return `<section class="obs-panel"><h2>Runs</h2>${rowTable(+ ["Run", "Agent", "Status", "Submitted"],+ runs.map(+ (r, i) =>+ `<tr><td><button class="obs-text obs-run-name" data-obs-run="${i}">${esc(r.name || r.turn_id)}</button><span class="cell-subtitle mono">${esc(r.turn_id)}</span></td>${cell(r.agent)}${cell(r.status)}${cell(new Date(r.created_ms).toLocaleString())}</tr>`,+ ),+ "Authorized runs",+ )}<div class="obs-pagination">${pageStack.length ? '<button class="button" data-obs="previous">Previous page</button>' : ""}${selection?.selection?.next_cursor ? '<button class="button" data-obs="next">Next 25 runs</button>' : ""}</div></section>`;+ }+ async function load() {+ const n = ++serial;+ loadController?.abort();+ const c = controller();+ loadController = c;+ summary = null;+ content.innerHTML = notice("Loading recordings…");+ scope();+ try {+ if (chosen) {+ const page = await read(perRun(chosen) + "/timeline?limit=100", c);+ if (stopped || n !== serial) return;+ timeline = page.page.steps;+ timelinePage = page.page;+ timelineThrough = page.page.through;+ selectedStep = null;+ // One watermark aligns the first timeline page and summary. It is not+ // a cross-request database snapshot; late commits/retention can differ.+ const value = await read(+ perRun(chosen) + "/summary?" + query({ through: timelineThrough }),+ c,+ );+ if (stopped || n !== serial) return;+ chosen = value.run;+ summary = value.summary;+ } else {+ const value = await read(+ "/v1/observability/summary?" ++ query({ limit: 25, after: pageCursor }),+ c,+ );+ if (stopped || n !== serial) return;+ selection = value;+ summary = selection.summary;+ }+ childLinks=[];subtree=null;+ if(chosen && children) {+ try {const links=await read(perRun(chosen)+"/children",c); if(stopped||n!==serial)return;childLinks=links.children||[];} catch {childLinks=[];}+ if (stopped || n !== serial) return;+ try {const value=await read(perRun(chosen)+"/subtree-summary?"+query({through:timelineThrough}),c);if(stopped||n!==serial)return;subtree=value;} catch {subtree=null;}+ }+ draw();+ } catch (e) {+ if (stopped || n !== serial || c.signal.aborted) return;+ // Old successful metrics are removed before each read; never retain them+ // as a silent fallback after scope change, logout or recording failure.+ content.innerHTML = notice(failure(e));+ scope();+ }+ }+ function closeSearch() {+ menu.hidden = true;+ input.setAttribute("aria-expanded", "false");+ input.removeAttribute("aria-activedescendant");+ }+ function drawResults() {+ $("#obs-run-options").innerHTML = results+ .map(+ (r, i) =>+ `<button type="button" class="obs-run-option" role="option" id="${domId(`obs-option-${i}`)}" aria-selected="${i === focusedIndex}" data-obs-option="${i}"><strong>${esc(r.name || r.turn_id)}</strong><span>${esc(r.agent)} · ${esc(r.status)}</span><small>${esc(r.turn_id)}</small></button>`,+ )+ .join("");+ $("#obs-search-more").hidden = !resultNext;+ if (focusedIndex >= 0)+ input.setAttribute("aria-activedescendant", domId(`obs-option-${focusedIndex}`));+ else input.removeAttribute("aria-activedescendant");+ }+ async function search(more = false) {+ const n = ++searchSerial;+ searchController?.abort();+ const c = controller();+ searchController = c;+ menu.hidden = false;+ input.setAttribute("aria-expanded", "true");+ $("#obs-search-state").textContent = "Searching…";+ try {+ const page = await read(+ "/v1/observability/runs?" ++ query({ q: searchText, limit: 10, after: more ? resultNext : null }),+ c,+ );+ if (stopped || n !== searchSerial) return;+ results = more+ ? [+ ...results,+ ...page.runs.filter(+ (r) => !results.some((x) => runKey(x) === runKey(r)),+ ),+ ]+ : page.runs;+ resultNext = page.next_cursor;+ focusedIndex = -1;+ drawResults();+ $("#obs-search-state").textContent = results.length+ ? `${results.length} matching runs${resultNext ? " · more available" : ""}`+ : "No matching runs.";+ } catch (e) {+ if (stopped || n !== searchSerial || c.signal.aborted) return;+ results = [];+ resultNext = null;+ drawResults();+ $("#obs-search-state").textContent = "Unable to search runs. Try again.";+ }+ }+ function choose(run, keepParents=false) {+ if(!keepParents) ancestors=[];+ if (!run) return;+ chosen = run;+ tool = null;+ tab = "timeline";+ input.value = "";+ searchText = "";+ closeSearch();+ ++searchSerial;+ searchController?.abort();+ clearTimeout(timer);+ load();+ }+ async function moreTimeline(button) {+ if (!chosen || !timelinePage?.next || button.disabled) return;+ button.disabled = true;+ const n = serial,+ c = controller();+ try {+ const value = await read(+ perRun(chosen) ++ "/timeline?" ++ query({+ after: timelinePage.next,+ through: timelineThrough,+ limit: Math.min(100, 1000 - timeline.length),+ }),+ c,+ );+ if (stopped || n !== serial) return;+ const cursors = new Set(timeline.map((r) => r.cursor));+ timeline.push(...value.page.steps.filter((r) => !cursors.has(r.cursor)));+ timelinePage = value.page;+ draw();+ } catch (e) {+ if (!stopped && n === serial && !c.signal.aborted) {+ $("#obs-timeline-error").textContent = failure(e);+ button.disabled = false;+ }+ }+ }+ async function loadHealth() {+ if (health || healthError || stopped) return;+ const el = $("#obs-health-content");+ el.textContent = "Loading…";+ const c = controller();+ try {+ health = await read("/v1/observability/health", c);+ if (stopped) return;+ el.innerHTML =+ hint(+ "Serving API instance only. Worker and other API counters are not aggregated here.",+ ) ++ `<dl class="obs-health-values"><div><dt>Recorder</dt><dd>${health.recording_enabled ? (health.recording ? "Enabled" : "Not reporting") : "Disabled"}</dd></div><div><dt>Exporter</dt><dd>${health.exporter_enabled ? "Enabled" : "Disabled"}</dd></div><div><dt>Recording drops</dt><dd>${count(health.recording?.dropped)}</dd></div><div><dt>Write failures</dt><dd>${count(health.recording?.failed_writes)}</dd></div><div><dt>Export failures</dt><dd>${count(health.exporter?.export_failures)}</dd></div></dl><details><summary>Process counters</summary><pre>${esc(JSON.stringify(health, null, 2))}</pre></details>`;+ } catch (e) {+ if (!stopped && !c.signal.aborted) {+ healthError = true;+ el.textContent =+ e.status === 403+ ? "Administrator access required."+ : "Process health unavailable. Refresh to try again.";+ }+ }+ }+ root.addEventListener(+ "click",+ (e) => {+ const option = e.target.closest("[data-obs-option]");+ if (option) {+ choose(results[Number(option.dataset.obsOption)]);+ return;+ }+ const run = e.target.closest("[data-obs-run]");+ if (run) {+ choose(selection?.selection.runs[Number(run.dataset.obsRun)]);+ return;+ }+ const step = e.target.closest("[data-obs-step]");+ if (step) {+ selectedStep = step.dataset.obsStep;+ draw();+ root.querySelector(`[data-obs-step="${selectedStep}"]`)?.focus();+ return;+ }+ const t = e.target.closest("[data-obs-tool]");+ if (t) {+ if (!chosen) {+ content.prepend(+ notice("Choose a run to inspect individual tool calls."),+ );+ return;+ }+ tool = t.dataset.obsTool;+ tab = "timeline";+ draw();+ return;+ }+ const childButton=e.target.closest("[data-obs-child]");+ if(childButton){+ const child=childLinks[Number(childButton.dataset.obsChild)];if(!child)return;+ const selectedSerial=serial;const selectedParent=chosen;+ if(child.child_turn_id){ancestors.push(selectedParent);choose({session_id:child.child_session_id,turn_id:child.child_turn_id},true);return;}+ childButton.disabled=true;+ const c=controller();+ read("/v1/observability/runs?"+query({q:child.child_session_id,limit:100}),c).then(page=>{+ if(stopped||serial!==selectedSerial)return;+ const run=child.child_turn_id ? {session_id:child.child_session_id,turn_id:child.child_turn_id} : page.runs.find(r=>r.session_id===child.child_session_id);+ if(run){ancestors.push(selectedParent);choose(run,true);}else{content.prepend(notice("No accessible child turn is available yet."));}+ }).catch(()=>{if(!stopped)content.prepend(notice("Child traces are unavailable."));}).finally(()=>{childButton.disabled=false;});+ return;+ }+ const action = e.target.closest("[data-obs]")?.dataset.obs;+ if(action==="parent" && ancestors.length){choose(ancestors.pop(),true);return;}+ if (action === "refresh") {+ health = null;+ healthError = false;+ if ($("#obs-health")?.open) loadHealth();+ load();+ }+ if (action === "recent" && directory) {+ chosen = null;+ tool = null;+ tab = "usage";+ pageCursor = null;+ pageStack = [];+ load();+ }+ if (action === "usage" || action === "timeline") {+ tab = action;+ draw();+ }+ if (action === "next") {+ pageStack.push(pageCursor);+ pageCursor = selection.selection.next_cursor;+ load();+ }+ if (action === "previous") {+ pageCursor = pageStack.pop() ?? null;+ load();+ }+ if (action === "more-timeline") moreTimeline(e.target.closest("button"));+ if (action === "clear-tool") {+ tool = null;+ draw();+ }+ },+ { signal: lifetime.signal },+ );+ root.addEventListener("keydown", e => {+ if (e.target.getAttribute?.("role") !== "tab" || !["ArrowLeft", "ArrowRight", "Home", "End"].includes(e.key)) return;+ const tabs = [...root.querySelectorAll('[role="tab"]')];+ let index = tabs.indexOf(e.target);+ if (e.key === "Home") index = 0;+ else if (e.key === "End") index = tabs.length - 1;+ else index = (index + (e.key === "ArrowRight" ? 1 : -1) + tabs.length) % tabs.length;+ e.preventDefault(); tab = tabs[index].dataset.obs; draw(); tabs[index].focus();+ }, {signal:lifetime.signal});+ input.addEventListener(+ "input",+ () => {+ searchText = input.value;+ clearTimeout(timer);+ ++searchSerial;+ searchController?.abort();+ results = [];+ resultNext = null;+ focusedIndex = -1;+ drawResults();+ menu.hidden = false;+ input.setAttribute("aria-expanded", "true");+ $("#obs-search-state").textContent = "Searching…";+ timer = setTimeout(() => search(), 250);+ },+ { signal: lifetime.signal },+ );+ input.addEventListener(+ "focus",+ () => {+ if (menu.hidden) {+ searchText = input.value;+ search();+ }+ },+ { signal: lifetime.signal },+ );+ input.addEventListener(+ "keydown",+ (e) => {+ if (e.key === "Escape") {+ ++searchSerial;+ searchController?.abort();+ clearTimeout(timer);+ closeSearch();+ e.preventDefault();+ }+ if (["ArrowDown", "ArrowUp"].includes(e.key)) {+ e.preventDefault();+ if (menu.hidden) {+ search();+ return;+ }+ focusedIndex = Math.max(+ 0,+ Math.min(+ results.length - 1,+ focusedIndex + (e.key === "ArrowDown" ? 1 : -1),+ ),+ );+ drawResults();+ $("#obs-run-options")+ .querySelector('[aria-selected="true"]')+ ?.scrollIntoView({ block: "nearest" });+ }+ if (e.key === "Enter" && focusedIndex >= 0) {+ e.preventDefault();+ choose(results[focusedIndex]);+ }+ },+ { signal: lifetime.signal },+ );+ $("#obs-search-more").addEventListener(+ "click",+ () => {+ if (results.length >= 100) {+ $("#obs-search-state").textContent =+ "Narrow your search to see more matches.";+ return;+ }+ search(true);+ },+ { signal: lifetime.signal },+ );+ document.addEventListener(+ "click",+ (e) => {+ if (!root.querySelector(".obs-picker").contains(e.target)) {+ closeSearch();+ }+ },+ { signal: lifetime.signal },+ );+ document.addEventListener(+ "focusin",+ (e) => {+ if (!root.querySelector(".obs-picker").contains(e.target)) closeSearch();+ },+ { signal: lifetime.signal },+ );+ $("#obs-health")?.addEventListener(+ "toggle",+ () => {+ if ($("#obs-health").open) loadHealth();+ },+ { signal: lifetime.signal },+ );+ load();+ return () => {+ stopped = true;+ serial++;+ searchSerial++;+ clearTimeout(timer);+ lifetime.abort();+ for (const c of controllers) c.abort();+ controllers.clear();+ root.replaceChildren();+ };+}diff --git a/harness/trace-viewer/src/style.css b/harness/trace-viewer/src/style.cssnew file mode 100644index 00000000..6f91452b--- /dev/null+++ b/harness/trace-viewer/src/style.css@@ -0,0 +1,508 @@+/* Optional viewer styles; no global reset. */+.ow-trace-viewer { color:#24272c;background:#fff;font:15px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;max-width:1120px;margin:auto;padding:28px;--line:#e5e7eb;--border:#e5e7eb;--muted:#667085;--surface:#f6f7f9;--text:#24272c;--accent:#405f99; }+.ow-trace-viewer * {box-sizing:border-box}+.ow-trace-viewer [hidden] {display:none!important}+.ow-trace-viewer button,.ow-trace-viewer input {font:inherit}+.ow-trace-viewer button {cursor:pointer;color:inherit}+.ow-trace-viewer button:focus-visible,.ow-trace-viewer input:focus-visible {outline:2px solid #405f99;outline-offset:3px}+.ow-trace-viewer .page-heading {display:flex;align-items:center;justify-content:space-between;margin-bottom:24px}+.ow-trace-viewer h1 {font-size:27px;letter-spacing:-.7px;margin:0}+.ow-trace-viewer h2 {font-size:17px;margin:0 0 16px}+.ow-trace-viewer p {color:#667085}+.ow-trace-viewer .button {background:white;border:1px solid #d9dde4;border-radius:8px;padding:8px 14px}+.ow-trace-viewer .tabs {display:flex;gap:4px;background:#f1f2f4;border-radius:9px;padding:4px}+.ow-trace-viewer .tab {border:0;background:transparent;border-radius:6px;padding:7px 15px}+.ow-trace-viewer .tab.active {background:white;box-shadow:0 1px 4px #0001}+.ow-trace-viewer .table {width:100%;border-collapse:collapse;text-align:left}+.ow-trace-viewer th,.ow-trace-viewer td {padding:12px;border-bottom:1px solid #eee;font-size:13px}+.ow-trace-viewer th {font-weight:500;color:#667085}+.ow-trace-viewer .cell-subtitle {display:block;color:#667085;font-size:12px}+.ow-trace-viewer pre {white-space:pre-wrap;overflow-wrap:anywhere;font-size:12px;padding:16px;background:#f6f7f9;border-radius:8px}+/* Observability: shared typography, neutral surfaces and restrained phase color. */+.observability {+ --obs-model: #687fba;+ --obs-tool: #6c958d;+ --obs-policy: #9b8aae;+ --obs-wait: #b39763;+}+.obs-toolbar {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: 24px;+ border-bottom: 1px solid var(--line);+ margin-bottom: 20px;+}+.obs-tabs {+ margin: 0;+ border: 0;+ align-self: stretch;+}+.obs-tabs .tab {+ padding: 16px 2px;+ margin-right: 22px;+}+.obs-picker {+ position: relative;+ display: flex;+ align-items: center;+ gap: 12px;+ padding: 10px 0;+ min-width: 310px;+}+.obs-picker label {+ color: var(--muted);+ font-size: 12px;+ white-space: nowrap;+}+.obs-picker input {+ width: 250px;+ border: 1px solid var(--line);+ border-radius: 7px;+ padding: 9px 11px;+ background: var(--canvas);+ color: var(--ink);+}+.obs-run-menu {+ position: absolute;+ top: calc(100% - 2px);+ right: 0;+ width: 370px;+ max-width: 90vw;+ background: white;+ border: 1px solid var(--line);+ border-radius: 9px;+ z-index: 10;+ box-shadow: 0 10px 35px #20232815;+ padding: 6px;+}+#obs-run-options {+ max-height: 310px;+ overflow-y: auto;+}+.obs-run-option {+ display: block;+ text-align: left;+ width: 100%;+ padding: 10px 11px;+ border-radius: 5px;+}+.obs-run-option:hover,+.obs-run-option[aria-selected="true"] {+ background: var(--wash);+}+.obs-run-option strong {+ display: block;+ font-weight: 500;+ overflow-wrap: anywhere;+}+.obs-run-option span,+.obs-run-option small {+ display: block;+ color: var(--muted);+ margin-top: 4px;+ font-size: 11px;+ overflow-wrap: anywhere;+}+.obs-run-menu p {+ color: var(--muted);+ padding: 8px 11px;+ font-size: 11px;+}+.obs-more {+ width: 100%;+ border-top: 1px solid var(--line);+ text-align: left;+ padding: 10px 11px;+}+.obs-scope {+ display: flex;+ align-items: center;+ gap: 18px;+ margin: 0 0 22px;+ min-height: 26px;+}+.obs-scope > div {+ flex: 1;+ min-width: 0;+}+.obs-scope strong {+ display: block;+ font-weight: 550;+ font-size: 15px;+ overflow-wrap: anywhere;+}+.obs-scope span {+ display: block;+ margin-top: 5px;+ color: var(--muted);+ font-size: 12px;+}+.obs-stats {+ grid-template-columns: repeat(4, minmax(0, 1fr));+ margin-bottom: 12px;+}+.obs-stats .stat-value {+ font-size: 26px;+ font-weight: 500;+ letter-spacing: -0.8px;+ overflow-wrap: anywhere;+}+.obs-stats .stat {+ min-width: 0;+}+.obs-grid {+ display: grid;+ grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);+ gap: 36px;+}+.obs-panel {+ padding: 26px 0;+ border-bottom: 1px solid var(--line);+ min-width: 0;+}+.obs-panel h2 {+ margin-bottom: 8px;+}+.obs-hint {+ color: var(--muted);+ font-size: 12px;+ line-height: 1.6;+ margin-top: 7px;+}+.obs-bars {+ margin: 20px 0 12px;+}+.obs-bar-row {+ display: grid;+ grid-template-columns: minmax(95px, 1fr) minmax(60px, 1.4fr) minmax(+ 56px,+ auto+ );+ align-items: center;+ gap: 14px;+ margin-top: 14px;+ font-size: 12px;+}+.obs-bar-row > span:first-child {+ overflow-wrap: anywhere;+}+.obs-track {+ height: 12px;+ background: var(--wash);+ border-radius: 3px;+ overflow: hidden;+}+.obs-fill {+ display: block;+ height: 100%;+ border-radius: 3px;+ background: var(--slate);+}+.obs-model {+ background: var(--obs-model);+}+.obs-tool {+ background: var(--obs-tool);+}+.obs-policy {+ background: var(--obs-policy);+}+.obs-approval_wait,+.obs-question_wait {+ background: var(--obs-wait);+}+.obs-other_active {+ background: #9aa0ab;+}+.obs-parallel {+ background: repeating-linear-gradient(+ 135deg,+ #868b97 0px,+ #868b97 3px,+ #b2b5bd 3px,+ #b2b5bd 6px+ );+}+.obs-unobserved_gap {+ background: #dadde3;+}+.obs-number {+ font-variant-numeric: tabular-nums;+ text-align: right;+ white-space: nowrap;+}+.obs-table-wrap {+ overflow-x: auto;+}+.obs-table-wrap .table {+ margin-top: 15px;+ min-width: 480px;+}+.obs-table-wrap .table td {+ vertical-align: top;+ font-variant-numeric: tabular-nums;+}+.obs-table-wrap .table th {+ white-space: nowrap;+}+.obs-table-wrap .table td:first-child {+ max-width: 320px;+ overflow-wrap: anywhere;+}+.obs-text {+ color: var(--slate);+ font-weight: 450;+ text-align: left;+ padding: 0;+}+.obs-text:hover {+ color: var(--ink);+ text-decoration: underline;+}+.obs-run-name {+ color: var(--ink);+ font-weight: 500;+}+.obs-pagination {+ display: flex;+ justify-content: flex-end;+ gap: 10px;+ margin-top: 16px;+}+.obs-notice {+ padding: 18px 0;+ color: var(--muted);+ line-height: 1.6;+}+.obs-empty-cell {+ color: var(--muted);+ padding: 20px 0 !important;+}+.obs-panel-title {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: 16px;+}+.obs-axis {+ display: flex;+ justify-content: space-between;+ margin: 20px 85px 4px 190px;+ color: var(--muted);+ font-size: 11px;+}+.obs-lanes {+ max-height: 560px;+ overflow: auto;+ padding-right: 4px;+ margin: 8px 0 16px;+}+.obs-lane {+ display: grid;+ grid-template-columns: 174px minmax(100px, 1fr) 70px;+ gap: 16px;+ width: 100%;+ padding: 8px 0;+ text-align: left;+ align-items: center;+ border-radius: 4px;+}+.obs-lane:hover,+.obs-lane[aria-pressed="true"] {+ background: var(--wash);+}+.obs-lane-name {+ font-size: 12px;+ padding-left: 6px;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+}+.obs-lane-name small {+ display: block;+ color: var(--muted);+ font-size: 10px;+ margin-top: 4px;+}+.obs-lane-track {+ display: block;+ height: 26px;+ position: relative;+ background: repeating-linear-gradient(+ to right,+ var(--line) 0,+ var(--line) 1px,+ transparent 1px,+ transparent 25%+ );+ border-bottom: 1px solid var(--line);+}+.obs-mark {+ position: absolute;+ height: 14px;+ top: 5px;+ border-radius: 3px;+ min-width: 3px;+ max-width: 100%;+ transform: translateX(-1px);+}+.obs-lane[aria-pressed="true"] .obs-mark {+ outline: 1px solid var(--ink);+ outline-offset: 2px;+}+.obs-lane .obs-number {+ font-size: 11px;+ padding-right: 4px;+}+.obs-step-detail {+ background: var(--wash);+ padding: 16px 18px;+ border-radius: 7px;+ margin: 14px 0;+}+.obs-step-detail strong {+ font-weight: 550;+}+.obs-step-detail > div > span {+ display: block;+ color: var(--muted);+ font-size: 12px;+ margin-top: 5px;+}+.obs-step-detail dl {+ display: grid;+ grid-template-columns: repeat(3, minmax(0, 1fr));+ gap: 16px;+ margin: 20px 0;+}+.obs-step-detail dt,+.obs-health-values dt {+ color: var(--muted);+ font-size: 11px;+ margin-bottom: 5px;+}+.obs-step-detail dd,+.obs-health-values dd {+ margin: 0;+ font-variant-numeric: tabular-nums;+}+.obs-details {+ border-bottom: 1px solid var(--line);+ padding: 17px 0;+ font-size: 12px;+}+.obs-details summary,+.obs-step-detail summary {+ cursor: pointer;+ color: var(--slate);+}+.obs-details p,+.obs-details ul {+ max-width: 760px;+ margin: 12px 0 0;+ color: var(--muted);+ line-height: 1.7;+}+.obs-details ul {+ padding-left: 18px;+}+.obs-details pre,+.obs-step-detail pre {+ max-height: 300px;+ overflow: auto;+ font-size: 11px;+ line-height: 1.6;+ white-space: pre-wrap;+ overflow-wrap: anywhere;+ padding: 14px;+ background: var(--wash);+ border-radius: 6px;+}+.obs-health-values {+ display: flex;+ gap: 50px;+ margin: 16px 0;+}+@media (max-width: 1100px) {+ .obs-bar-row {+ grid-template-columns: minmax(90px, 1fr) minmax(35px, 1fr) auto;+ gap: 9px;+ }+ .obs-grid {+ gap: 24px;+ }+}+@media (max-width: 800px) {+ .obs-grid {+ grid-template-columns: minmax(0, 1fr);+ gap: 0;+ }+ .obs-toolbar {+ flex-wrap: wrap;+ gap: 0;+ }+ .obs-picker {+ min-width: 0;+ }+ .obs-stats .stat-value {+ font-size: 22px;+ }+}+@media (max-width: 560px) {+ .obs-stats {+ grid-template-columns: repeat(2, minmax(0, 1fr));+ }+ .obs-picker {+ width: 100%;+ padding-top: 0;+ }+ .obs-picker input {+ flex: 1;+ min-width: 0;+ width: auto;+ }+ .obs-run-menu {+ width: 100%;+ }+ .obs-scope {+ flex-wrap: wrap;+ gap: 12px;+ }+ .obs-scope > div {+ flex-basis: 100%;+ }+ .obs-lane {+ grid-template-columns: 108px minmax(30px, 1fr) 58px;+ gap: 8px;+ }+ .obs-axis {+ margin-left: 116px;+ margin-right: 66px;+ }+ .obs-step-detail dl {+ grid-template-columns: repeat(2, minmax(0, 1fr));+ }+}+.obs-partial {+ display: inline-block;+ border: 1px solid var(--line);+ border-radius: 4px;+ padding: 3px 7px;+ font-size: 11px;+ color: var(--muted);+ margin-bottom: 12px;+}+.obs-health-values {+ flex-wrap: wrap;+ gap: 20px 40px;+}+.ow-trace-viewer .stats {display:grid;grid-template-columns:repeat(4,minmax(0,1fr));border:1px solid var(--line);border-radius:10px;margin:0 0 24px}+.ow-trace-viewer .stat {padding:20px 24px}+.ow-trace-viewer .stat+.stat {border-left:1px solid var(--line)}+.ow-trace-viewer .stat-label {font-size:12px;color:#737b88}+.ow-trace-viewer .stat-main {margin-top:10px}+.ow-trace-viewer .stat-value {font-size:29px;letter-spacing:-1px;font-weight:550;font-variant-numeric:tabular-nums}+.ow-trace-viewer .obs-lane {appearance:none;border:0;border-bottom:1px solid #eef0f3;background:white;text-align:left;border-radius:4px}+.ow-trace-viewer .obs-lane:hover {background:#f7f8fa}+.ow-trace-viewer .obs-lane[aria-pressed=true] {background:#f0f3f8;box-shadow:inset 3px 0 #687fba}diff --git a/harness/trace-viewer/sync.mjs b/harness/trace-viewer/sync.mjsnew file mode 100644index 00000000..c2054ea9--- /dev/null+++ b/harness/trace-viewer/sync.mjs@@ -0,0 +1,9 @@+// One authoritative viewer implementation, copied into the standalone UI image.+import {readFile,writeFile} from 'node:fs/promises';+import assert from 'node:assert/strict';+for(const file of ['observability.js','observability-data.js']) {+ const source=await readFile(new URL(`src/${file}`,import.meta.url),'utf8');+ const target=new URL(`../service-ui/public/${file}`,import.meta.url);+ if(process.argv.includes('--check'))assert.equal(await readFile(target,'utf8'),source,`${file}: run npm run sync in trace-viewer`);+ else await writeFile(target,source);+}diff --git a/harness/trace-viewer/test/browser.mjs b/harness/trace-viewer/test/browser.mjsnew file mode 100644index 00000000..735bfa71--- /dev/null+++ b/harness/trace-viewer/test/browser.mjs@@ -0,0 +1,80 @@+// Run against the loopback preview with a caller-supplied Playwright installation.+// No dependency or browser download is added to the SDK/runtime package.+import assert from 'node:assert/strict';+import {mkdir} from 'node:fs/promises';+import {resolve} from 'node:path';+import {pathToFileURL} from 'node:url';+const [url,modulePath,output]=process.argv.slice(2);+assert(url&&modulePath&&output,'Usage: browser.mjs LOOPBACK_URL PLAYWRIGHT_MODULE OUTPUT_DIRECTORY');+assert(['127.0.0.1','localhost'].includes(new URL(url).hostname));+const {chromium}=await import(pathToFileURL(resolve(modulePath)).href);+const browser=await chromium.launch({headless:true});+try {+ const page=await browser.newPage({viewport:{width:1440,height:1100}});+ const errors=[];page.on('pageerror',e=>errors.push(e.message));+ await page.goto(url);await page.getByText('Reported tokens',{exact:true}).waitFor();+ await mkdir(output,{recursive:true});+ await page.screenshot({path:resolve(output,'sdk-trace-usage.png'),fullPage:true});+ await page.getByRole('tab',{name:'Timeline',exact:true}).click();+ await page.getByRole('heading',{name:'Turn timeline'}).waitFor();+ await page.locator('[data-obs-step]').first().click();+ await page.getByText('Record details',{exact:true}).waitFor();+ await page.screenshot({path:resolve(output,'sdk-trace-timeline.png'),fullPage:true});+ // Multiple independently mounted instances, hostile text, subtree/eval panels.+ await page.evaluate(async()=>{+ const {mountTurnTrace}=await import('/src/index.js');+ const evidence=await fetch('/trace.json').then(r=>r.json());+ const adapter={timeline:async()=>evidence.timeline,summary:async()=>evidence.summary,+ children:async()=>({children:[{child_session_id:'child',child_turn_id:'initial',agent_id:'<img src=x onerror=alert(1)>',phase:'ready'}]}),+ subtreeSummary:async()=>({selection:{runs:[{},{}],selection_truncated:false},summary:evidence.summary.summary}),+ evaluations:async()=>({results:[{evaluation:{result:{name:'correctness',status:'completed',score:{value:true},evaluator:{name:'fixture',version:'1',source:'code'},explanation:'<script>alert(1)</script>'}}}],next:null})};+ window.testViews=[];+ for(let i=0;i<2;i++){const root=document.createElement('main');root.id=`test-${i}`;document.body.append(root);window.testViews.push(mountTurnTrace(root,{adapter,sessionId:evidence.timeline.run.session_id,turnId:evidence.timeline.run.turn_id,onChild:child=>{window.selectedChild=child;}}));}+ });+ await page.locator('#test-0').getByRole('heading',{name:'Linked work',exact:true}).waitFor();+ const ids=await page.locator('[id]').evaluateAll(nodes=>nodes.map(n=>n.id));assert.equal(new Set(ids).size,ids.length,'Mounted viewers must have unique DOM IDs');+ assert.equal(await page.locator('#test-0 img, #test-0 script').count(),0,'Content must be escaped');+ await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).focus();await page.keyboard.press('ArrowRight');+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Timeline',exact:true}).getAttribute('aria-selected'),'true');+ await page.keyboard.press('Home');+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).getAttribute('aria-selected'),'true');+ const child=page.locator('#test-0').getByRole('button',{name:/img src/});await child.focus();await page.keyboard.press('Enter');+ assert.equal(await page.evaluate(()=>window.selectedChild.child_turn_id),'initial');+ await page.locator('#test-1').getByRole('tab',{name:'Timeline',exact:true}).click();+ assert.equal(await page.locator('#test-0').getByRole('tab',{name:'Usage',exact:true}).getAttribute('aria-selected'),'true');+ // The same component's service adapter follows an exact child turn without listing all runs.+ await page.evaluate(async()=>{+ const {mountObservability}=await import('/src/index.js');const data=await fetch('/trace.json').then(r=>r.json());+ const root=document.createElement('main');root.id='management';document.body.append(root);window.managementRequests=[];+ window.managementDispose=mountObservability(root,{initialRun:data.timeline.run,directory:false,openSession:false,children:true,request:async path=>{+ window.managementRequests.push(path);const u=new URL(path,location.origin),parts=u.pathname.split('/');const run={session_id:parts[3],turn_id:parts[5]};+ if(parts[6]==='children')return {children:run.session_id==='child'?[]:[{child_session_id:'child',child_turn_id:'initial',agent_id:'builder',phase:'ready'}]};+ if(parts[6]==='subtree-summary')return {selection:{runs:[{},{}],selection_truncated:false},summary:data.summary.summary};+ if(parts[6]==='timeline')return {...data.timeline,run};if(parts[6]==='summary')return {...data.summary,run};throw Error('Unexpected service request');+ }});+ });+ await page.locator('#management [data-obs-child]').waitFor();await page.locator('#management [data-obs-child]').click();+ await page.waitForFunction(()=>window.managementRequests.some(p=>p.includes('/sessions/child/observability/initial/summary')));+ assert(!(await page.evaluate(()=>window.managementRequests)).some(p=>p.includes('/observability/runs')));+ await page.locator('#management [data-obs="parent"]').click();+ await page.evaluate(()=>window.managementDispose());+ // A large trace stays paged and stops at the documented browser bound.+ await page.evaluate(async()=>{+ const {mountTurnTrace}=await import('/src/index.js');const data=await fetch('/trace.json').then(r=>r.json());+ const root=document.createElement('main');root.id='large';document.body.append(root);+ const base=data.timeline.page.steps[0].step;+ const rows=Array.from({length:1001},(_,i)=>({cursor:i+1,step:{...base,span_id:(i+1).toString(16).padStart(16,'0')}}));+ window.largeRequests=[];+ window.largeView=mountTurnTrace(root,{sessionId:data.timeline.run.session_id,turnId:data.timeline.run.turn_id,adapter:{summary:async()=>data.summary,timeline:async q=>{window.largeRequests.push(q);const start=q.after??0;return {...data.timeline,page:{steps:rows.slice(start,start+100),through:1001,next:start+100<1001?start+100:null}};}}});+ });+ await page.locator('#large').getByText('Reported tokens',{exact:true}).waitFor();+ await page.locator('#large').getByRole('tab',{name:'Timeline',exact:true}).click();+ for(let i=0;i<9;i++) {await page.locator('#large').getByRole('button',{name:'Load more records',exact:true}).click();await page.waitForFunction(n=>document.querySelectorAll('#large [data-obs-step]').length===n,(i+2)*100);}+ assert.equal(await page.locator('#large [data-obs-step]').count(),1000);+ assert.equal(await page.locator('#large').getByRole('button',{name:'Load more records',exact:true}).count(),0);+ assert.equal(await page.evaluate(()=>window.largeRequests.length),10);+ await page.evaluate(()=>window.largeView.dispose());+ await page.evaluate(()=>window.testViews.forEach(v=>v.dispose()));+ assert.equal(await page.locator('#test-0').textContent(),'');assert.deepEqual(errors,[]);+ console.log('Viewer passed: usage, timeline/detail, unique IDs, independent instances, child keyboard navigation, escaped evaluations, disposal.');+}finally{await browser.close();}diff --git a/harness/typescript-sdk/README.md b/harness/typescript-sdk/README.mdindex 3b06305c..95df5efd 100644--- a/harness/typescript-sdk/README.md+++ b/harness/typescript-sdk/README.md@@ -518,3 +518,14 @@ tool subset. The legacy `subagents: Agent[]` form remains unchanged. This requira matching feature runtime; it is not in published runtime 0.1.0-beta.2.See [standalone sub-agents](../../docs/harness/guides/sessions.md#standalone-sub-agents)for limits, approvals and shared-sandbox behavior.+++## Optional turn traces and evaluation evidence++The source candidate includes opt-in recording, per-turn timeline/grouped queries,+child drill-down and durable evaluation results. Recording and evaluation delivery+are disabled until configured; browser UI is a separate optional component.+See the [observability guide](../../docs/harness/guides/observability.md) for local+SQLite/PostgreSQL, Langfuse export, content selection and audit references. These+new APIs require matching local candidate builds and are not in the published+0.1.0-beta.2 packages yet.diff --git a/harness/typescript-sdk/package.json b/harness/typescript-sdk/package.jsonindex 13cde785..0e7d61d8 100644--- a/harness/typescript-sdk/package.json+++ b/harness/typescript-sdk/package.json@@ -11,6 +11,10 @@".": {"types": "./dist/index.d.ts","import": "./dist/index.js"+ },+ "./evaluation-export": {+ "types": "./dist/evaluation-export.d.ts",+ "import": "./dist/evaluation-export.js"}},"files": [@@ -37,7 +41,9 @@"test:provider-auth": "npm run build && node --test test/provider-auth.test.mjs","prepack": "npm run build","test:service": "npm run build && node --test test/service.test.mjs",- "test:team-types": "npm run build && node --test --test-concurrency=1 test/team-types.test.mjs"+ "test:team-types": "npm run build && node --test --test-concurrency=1 test/team-types.test.mjs",+ "test:observability": "npm run build && node --test test/observability.test.mjs test/evaluation-export.test.mjs",+ "test:observability:integration": "npm run build && node --test test/observability-live.test.mjs"},"devDependencies": {"@types/node": "^22.0.0",diff --git a/harness/typescript-sdk/src/config.ts b/harness/typescript-sdk/src/config.tsindex c632d6f1..b62afdff 100644--- a/harness/typescript-sdk/src/config.ts+++ b/harness/typescript-sdk/src/config.ts@@ -139,6 +139,19 @@ export function hostConfig(options: ResolvedHarnessOptions): object {: (options.sandbox ?? "openshell"),scripted: options.scripted ?? false,telemetry: options.telemetry ?? null,+ ...(options.recording ? { recording: {+ evaluations: options.recording.evaluations ?? false,+ max_pending_evaluations:options.recording.maxPendingEvaluations??10000,+ evaluation_retention_days:options.recording.evaluationRetentionDays??30,+ evaluation_sinks:(options.recording.evaluationSinks??[]).map(s=>({name:s.name,url:s.url,authorization_env:s.authorizationEnv,include_explanation:s.includeExplanation??false})),+ evidence: options.recording.evidence ? {user_input:options.recording.evidence.userInput??false,assistant_output:options.recording.evidence.assistantOutput??false,tool_arguments:options.recording.evidence.toolArguments??false,tool_results:options.recording.evidence.toolResults??false,max_bytes:options.recording.evidence.maxBytes}:null,+ database: options.recording.database.type === "sqlite"+ ? {kind:"sqlite",path:resolve(options.recording.database.path)}+ : {kind:"postgres",connection_env:options.recording.database.connectionEnv,schema:options.recording.database.schema,ca_file:options.recording.database.caFile ?? null,loopback_plaintext:options.recording.database.loopbackPlaintext ?? false},+ deployment:options.recording.deployment ?? "sdk",+ retention_days:options.recording.retentionDays ?? 30,+ queue_capacity:options.recording.queueCapacity ?? 1024,+ }} : {}),};}export function agentWire(a: Agent, depth = 0): object {@@ -186,6 +199,8 @@ export function environment(options: HarnessOptions): NodeJS.ProcessEnv {const explicit = [...(options.inheritEnv ?? []), ...(options.complianceSinks ?? []).map(s=>s.credentialEnv)];if (options.storage?.type === "postgres")explicit.push(options.storage.connectionEnv);+ if(options.recording?.database.type === "postgres") explicit.push(options.recording.database.connectionEnv);+ for (const sink of options.recording?.evaluationSinks ?? []) explicit.push(sink.authorizationEnv);for (const header of options.telemetry?.headers ?? [])if ("value_env" in header) explicit.push(header.value_env);for (const key of explicit) {diff --git a/harness/typescript-sdk/src/evaluation-export.ts b/harness/typescript-sdk/src/evaluation-export.tsnew file mode 100644index 00000000..a3b5b13c--- /dev/null+++ b/harness/typescript-sdk/src/evaluation-export.ts@@ -0,0 +1,34 @@+/** Optional explicit export helpers. No background worker or network at import. */+import {createHash} from 'node:crypto';+import type {Evaluation} from './observability.js';+export interface ScoreExportOptions {url:string;authorizationEnv:string;includeExplanation?:boolean}+export function langfuseScore(value:Evaluation,includeExplanation=false):Record<string,unknown> {+ if(!value.target.trace_id)throw Error('Select a recorded span before exporting a Langfuse score');+ if(value.result.status!=='completed'||!value.result.score)throw Error('Skipped/error evaluations are not quality scores');+ const s=value.result.score;+ return {id:createHash('sha256').update(`${value.deployment}\0${value.result.id}`).digest('hex'),name:value.result.name,traceId:value.target.trace_id,observationId:value.target.span_id??undefined,dataType:s.type.toUpperCase(),value:s.type==='boolean'?(s.value?1:0):s.value,timestamp:new Date(value.result.created_ms).toISOString(),...(includeExplanation&&value.result.explanation?{comment:value.result.explanation}:{}),metadata:{'openworker.evaluation.id':value.result.id,evaluator:value.result.evaluator.name,evaluatorVersion:value.result.evaluator.version,evaluatorSource:value.result.evaluator.source,evaluatorModel:value.result.evaluator.model,evaluatorUsage:value.result.evaluator.usage,...(value.result.supersedes?{supersedes:value.result.supersedes}:{})}};+}+/** Sends one stable score. Does not retry unknown outcomes or re-export a trace. */+export async function exportLangfuseScore(value:Evaluation,options:ScoreExportOptions):Promise<{id:string}> {+ const url=new URL(options.url);+ if(url.username||url.password||url.search||url.hash||url.pathname!=='/'||(url.protocol!=='https:'&&!(url.protocol==='http:'&&['localhost','127.0.0.1','[::1]'].includes(url.hostname))))throw Error('Use a HTTPS origin or explicit loopback Langfuse origin');+ if(!/^[A-Z_][A-Z0-9_]{0,99}$/.test(options.authorizationEnv))throw Error('Invalid export credential reference');+ const auth=process.env[options.authorizationEnv];if(!auth)throw Error('Export credential unavailable');+ const body=langfuseScore(value,options.includeExplanation);let response;+ try {response=await fetch(new URL('/api/public/scores',url),{method:'POST',headers:{Authorization:auth,'Content-Type':'application/json'},body:JSON.stringify(body),redirect:'error',signal:AbortSignal.timeout(3000)});}+ catch {throw Error('Score export outcome unknown; reconcile the stable score ID before retrying');}+ if(response.status!==200){await response.body?.cancel();throw Error(`Score export not confirmed (HTTP ${response.status}); no automatic retry`);}+ const reader=response.body?.getReader();if(!reader)throw Error('Score export acknowledgement missing');+ let bytes=0;const chunks:Uint8Array[]=[];+ for(;;){const {value,done}=await reader.read();if(done)break;bytes+=value.byteLength;if(bytes>65536){await reader.cancel();throw Error('Score acknowledgement too large; outcome unknown');}chunks.push(value);}+ let ack;try{ack=JSON.parse(Buffer.concat(chunks).toString('utf8'));}catch{throw Error('Invalid score acknowledgement; outcome unknown');}+ if(ack?.id!==body.id)throw Error('Score identity mismatch; outcome unknown');+ return {id:String(body.id)};+}+/** Caller supplies authorized projections. Content is absent unless explicitly supplied. */+export function evidenceJsonl(input:{selection:{sessionId:string;turnId:string};watermark:number;timeline:unknown;summary:unknown;evaluations?:Evaluation[];content?:unknown}):string {+ const records=[{kind:'timeline',value:input.timeline},{kind:'summary',value:input.summary},...(input.evaluations??[]).map(value=>({kind:'evaluation',value})),...(input.content===undefined?[]:[{kind:'selected_content',value:input.content}])];+ const lines=records.map(r=>JSON.stringify(r)).join('\n')+'\n';+ if(Buffer.byteLength(lines)>8*1024*1024)throw Error('Evidence export exceeds 8 MiB; export smaller scoped pages');+ return JSON.stringify({kind:'manifest',version:1,selection:input.selection,watermark:input.watermark,contentIncluded:input.content!==undefined,coverage:'selected_pages_only',guarantee:'best_effort_traces_not_an_audit_archive',sha256:createHash('sha256').update(lines).digest('hex')})+'\n'+lines;+}diff --git a/harness/typescript-sdk/src/index.ts b/harness/typescript-sdk/src/index.tsindex ed0b808f..630a9f7c 100644--- a/harness/typescript-sdk/src/index.ts+++ b/harness/typescript-sdk/src/index.ts@@ -1,3 +1,5 @@+import { Observability } from "./observability.js";+export * from "./observability.js";import { normalizeSafety, type SafetyOptions, type SafetyHandler, type SafetyContext, type SafetyDecision } from "./safety.js";export * from "./safety.js";import { Service, type ServiceOptions } from "./service.js";@@ -101,6 +103,7 @@ function interruptionOf(value: unknown): Interruption {/** Node.js host owner. Agent logic, storage and permission enforcement remain in Rust. */export class Harness {+ get observability(): Observability { return new Observability((method,args)=>this.request(method,args as Json)); }static connect(options: ServiceOptions): Promise<Service> {return Service.connect(options);}diff --git a/harness/typescript-sdk/src/observability.ts b/harness/typescript-sdk/src/observability.tsnew file mode 100644index 00000000..7b3af046--- /dev/null+++ b/harness/typescript-sdk/src/observability.ts@@ -0,0 +1,120 @@+/** Optional trace queries. This module neither starts a server nor installs an exporter. */+import { HarnessError, type Telemetry } from "./types.js";+export interface TraceQuery { sessionId:string; turnId:string; after?:number; through?:number; limit?:number }+export interface TraceStep {+ version:number; deployment:string; role:string; trace_id:string; span_id:string;+ parent_trace_id:string|null; parent_span_id:string|null; session_hash:string|null; turn_hash?:string|null;+ operation:string; outcome:string; started_ms:number; ended_ms:number; duration_us:number|null;+ first_output_us:number|null; attempt:number|null; model:string|null; tool:string|null; purpose:string|null;+ output_bytes:number|null; usage:{input:number|null;output:number|null;cache_read:number|null;cache_write:number|null;reasoning:number|null}|null;+ links?:{trace_id:string;span_id:string}[];+ sandbox?:string|null;enforcement?:string|null;policy_digest?:string|null;+}+export interface Timeline {+ run:{session_id:string;turn_id:string};+ page:{steps:{cursor:number;step:TraceStep}[];through:number;next:number|null};+ coverage:{guarantee:"best_effort";order:"recording_cursor";snapshot_across_requests:false};+}+export interface Measure {total:number|null;known:number;unknown:number;overflow:boolean}+export interface Usage {input:Measure;output:Measure;cache_read:Measure;cache_write:Measure;reasoning:Measure}+export interface TraceGroup {+ name:string|null;purpose:string|null;calls:number;outcomes:Record<string,number>;+ accumulated_us:Measure;active_union_us:number|null;first_output_us:Measure;returned_bytes:Measure;usage:Usage|null;+}+export interface TraceCoverage {+ guarantee:string;scanned_steps:number;scan_truncated:boolean;group_truncated:boolean;+ expected_segments:number|null;recorded_segments:number;all_visible_segments_closed:boolean;pending_waits:number|null;+ requests_without_completion:number;completions_without_start:number;unknown_timing_steps:number;unknown_outcome_steps:number;+ clock_or_range_anomaly:boolean;hierarchy_incomplete:boolean;+}+export interface Summary {+ run:{session_id:string;turn_id:string};+ summary:{projection_version:number;through:number;coverage:TraceCoverage;wall_elapsed_us:number|null;observed_window_us:number|null;active_segments_accumulated_us:Measure;model_accumulated_us:Measure;tool_accumulated_us:Measure;exclusive_observed_us:Record<string,number>;usage:Usage;models:TraceGroup[];tools:TraceGroup[]};+}+export interface TurnPage {turns:{session_id:string;turn_id:string;status:string|null;agent:string;name:string}[];next_cursor:number|null;through:number}+export interface ChildLink {+ origin:{parent_session_id:string;parent_turn_id:string;parent_call_id:string|null};+ child_session_id:string;child_turn_id:string|null;agent_id:string;phase:string;+}+export interface SubtreeSummary {selection:{runs:{session_id:string;turn_id:string}[];selection_truncated:boolean;scope:"root_and_visible_spawned_initial_turns"};summary:Summary["summary"]}+export interface Children {children:ChildLink[];coverage:string}+export interface ObservabilityAdapter {+ subtreeSummary?(query:TraceQuery):Promise<SubtreeSummary>;+ children?(query:TraceQuery):Promise<Children>;++ timeline(query:TraceQuery):Promise<Timeline>;+ summary(query:TraceQuery):Promise<Summary>;+}+function checked(q:TraceQuery):void {+ if (![q.sessionId,q.turnId].every(s=>typeof s==="string"&&/^[a-zA-Z0-9_-]{1,80}$/.test(s)) ||+ ![q.after??0,q.through??0,q.limit??100].every(n=>Number.isSafeInteger(n)&&n>=0) ||+ (q.limit??100)<1 || (q.limit??100)>200 || (q.through!==undefined&&q.through<(q.after??0)))+ throw new HarnessError("InvalidInput","Invalid trace scope or page bounds");+}+function wire(q:TraceQuery):Record<string,unknown> {+ checked(q);+ return {session_id:q.sessionId,turn_id:q.turnId,after:q.after??0,through:q.through??null,limit:q.limit??100};+}+export class Observability implements ObservabilityAdapter {+ constructor(private readonly request:(method:string,args:Record<string,unknown>)=>Promise<unknown>){}+ async flush(timeoutMs=5000):Promise<Record<string,unknown>> {if(!Number.isInteger(timeoutMs)||timeoutMs<1||timeoutMs>5000)throw new HarnessError("InvalidInput","Flush budget must be 1..5000ms");return await this.request("observability/flush",{timeout_ms:timeoutMs}) as Record<string,unknown>;}+ async health():Promise<Record<string,unknown>> {return await this.request("observability/health",{}) as Record<string,unknown>;}+ async evidence(q:{sessionId:string;turnId:string;afterEventId?:number}):Promise<Record<string,unknown>> {+ checked({sessionId:q.sessionId,turnId:q.turnId});+ return await this.request("observability/evidence",{session_id:q.sessionId,turn_id:q.turnId,after_event_id:q.afterEventId??0}) as Record<string,unknown>;+ }+ async recordEvaluation(q:TraceQuery,input:EvaluationInput,spanId?:string):Promise<{result:Evaluation;duplicate:boolean}> {+ checked(q);+ return await this.request("observability/evaluation-record",{session_id:q.sessionId,turn_id:q.turnId,span_id:spanId??null,result:evaluationWire(input)}) as {result:Evaluation;duplicate:boolean};+ }+ async evaluations(q:TraceQuery):Promise<EvaluationPage> {+ return await this.request("observability/evaluation-list",wire({...q,limit:q.limit??25})) as EvaluationPage;+ }+ async listTurns(q:{sessionId:string;after?:number;limit?:number}):Promise<TurnPage> {+ if(!Number.isSafeInteger(q.after??0)||(q.after??0)<0||!Number.isSafeInteger(q.limit??25)||(q.limit??25)<1||(q.limit??25)>100)throw new HarnessError("InvalidInput","Invalid turn page");+ return await this.request("observability/turns",{session_id:q.sessionId,after:q.after??0,limit:q.limit??25}) as TurnPage;+ }+ async subtreeSummary(q:TraceQuery):Promise<SubtreeSummary>{return await this.request("observability/subtree-summary",wire(q)) as SubtreeSummary;}+ async children(q:TraceQuery):Promise<Children>{return await this.request("observability/children",wire(q)) as Children;}+ async timeline(q:TraceQuery):Promise<Timeline>{return await this.request("observability/timeline",wire(q)) as Timeline;}+ async summary(q:TraceQuery):Promise<Summary>{return await this.request("observability/summary",wire(q)) as Summary;}+}+/** Authenticated service APIs enforce access on every read, including descendants. */+export class ServiceObservability implements ObservabilityAdapter {+ constructor(private readonly get:(path:string)=>Promise<unknown>){}+ async health():Promise<Record<string,unknown>> {return await this.get("/v1/observability/health") as Record<string,unknown>;}+ async search(q:{q?:string;agent?:string;limit?:number;after?:string}={}):Promise<{runs:Record<string,unknown>[];next_cursor:string|null;as_of_us:number}> {+ const query=new URLSearchParams();for(const [k,v] of Object.entries(q))if(v!==undefined)query.set(k,String(v));+ return await this.get(`/v1/observability/runs?${query}`) as {runs:Record<string,unknown>[];next_cursor:string|null;as_of_us:number};+ }+ private path(q:TraceQuery,kind:string):string {+ checked(q);const args=new URLSearchParams();+ if(kind!=="children"&&q.through!==undefined)args.set("through",String(q.through));+ if(kind==="timeline"){if(q.after!==undefined)args.set("after",String(q.after));if(q.limit!==undefined)args.set("limit",String(q.limit));}+ return `/v1/sessions/${encodeURIComponent(q.sessionId)}/observability/${encodeURIComponent(q.turnId)}/${kind}${args.size ? `?${args}` : ""}`;+ }+ async subtreeSummary(q:TraceQuery):Promise<SubtreeSummary>{return await this.get(this.path(q,"subtree-summary")) as SubtreeSummary;}+ async children(q:TraceQuery):Promise<Children>{return await this.get(this.path(q,"children")) as Children;}+ async timeline(q:TraceQuery):Promise<Timeline>{return await this.get(this.path(q,"timeline")) as Timeline;}+ async summary(q:TraceQuery):Promise<Summary>{return await this.get(this.path(q,"summary")) as Summary;}+}+/** Direct Langfuse traces; auth is read in the trusted harness process. */+export function langfuseTelemetry(endpoint:string, authorizationEnv:string):Telemetry {+ return {endpoint,signals:["traces"],acknowledgement:"langfuse_v4",headers:[{name:"Authorization",value_env:authorizationEnv},{name:"x-langfuse-ingestion-version",value:"4"}],content:{capture:"metadata"}};+}++export interface EvaluationInput {+ id:string;name:string;evaluator:{name:string;version:string;source:"human"|"code"|"model";model?:string;usage?:TraceStep["usage"]};+ status:"completed"|"skipped"|"error";+ score?:{type:"numeric";value:number}|{type:"boolean";value:boolean}|{type:"categorical"|"text";value:string};+ createdMs:number;explanation?:string;datasetCase?:string;experiment?:string;supersedes?:string;+}+export interface Evaluation {+ version:1;deployment:string;+ target:{session_hash:string;turn_hash:string;trace_id:string|null;span_id:string|null};+ result:{id:string;name:string;evaluator:EvaluationInput["evaluator"];status:EvaluationInput["status"];score:EvaluationInput["score"]|null;created_ms:number;explanation:string|null;dataset_case:string|null;experiment:string|null;supersedes:string|null};+}+export interface EvaluationPage {results:{cursor:number;evaluation:Evaluation}[];next:number|null}+export function evaluationWire(input:EvaluationInput):Record<string,unknown> {+ return {id:input.id,name:input.name,evaluator:input.evaluator,status:input.status,score:input.score??null,created_ms:input.createdMs,explanation:input.explanation??null,dataset_case:input.datasetCase??null,experiment:input.experiment??null,supersedes:input.supersedes??null};+}diff --git a/harness/typescript-sdk/src/service.ts b/harness/typescript-sdk/src/service.tsindex 4ff91b74..0ed6fc6f 100644--- a/harness/typescript-sdk/src/service.ts+++ b/harness/typescript-sdk/src/service.ts@@ -1,3 +1,4 @@+import { ServiceObservability } from "./observability.js";/** Authenticated remote service handles. The service owns all execution and policy. */import { randomUUID } from "node:crypto";import {sandboxWire, serviceDelegationWire, type ServiceTeamDelegation} from "./team-placement.js";@@ -222,6 +223,7 @@ export async function* serviceFrames(}}export class Service {+ get observability():ServiceObservability {return new ServiceObservability(path=>this.json("GET",path));}private closed = false;private requests = new Set<AbortController>();private endpoint = 0;diff --git a/harness/typescript-sdk/src/types.ts b/harness/typescript-sdk/src/types.tsindex 8b34028d..2295394a 100644--- a/harness/typescript-sdk/src/types.ts+++ b/harness/typescript-sdk/src/types.ts@@ -167,6 +167,8 @@ export type Storage =loopbackPlaintext?: boolean;};export interface Telemetry {+ /** Receiver protocol; direct Langfuse v4 requires its explicit receipt dialect. */+ acknowledgement?: "otlp" | "langfuse_v4";endpoint: string;deployment?: string;headers?: (@@ -205,6 +207,7 @@ export interface HarnessOptions extends SafetyOptions {sandbox?: "openshell" | "off" | { provider: "openshell" };mcpServers?: McpServer[];telemetry?: Telemetry;+ recording?: RecordingOptions;restart?: { maxRestarts?: number; delayMs?: number };requestTimeoutMs?: number;shutdownTimeoutMs?: number;@@ -452,3 +455,17 @@ export interface ComplianceActivity {}export interface CompliancePage { activities:ComplianceActivity[]; nextAfter:number; hasMore:boolean }export interface ComplianceDeliveryHealth { sinks:{sink:string;cursor:number;pending:number;failures:number;nextAtMs:number;lastSuccessMs:number|null}[] }++/** Optional diagnostics storage, independent from execution storage. */+export interface RecordingOptions {+ maxPendingEvaluations?:number;+ evaluationRetentionDays?:number;+ evaluationSinks?: {name:string;url:string;authorizationEnv:string;includeExplanation?:boolean}[];+ evidence?: {userInput?:boolean;assistantOutput?:boolean;toolArguments?:boolean;toolResults?:boolean;maxBytes:number};+ /** Enable acknowledged evaluation-result storage (requires runtime capability). */+ evaluations?: boolean;+ database: { type: "sqlite"; path: string } | { type: "postgres"; connectionEnv: string; schema: string; caFile?: string; loopbackPlaintext?: boolean };+ deployment?: string;+ retentionDays?: number;+ queueCapacity?: number;+}diff --git a/harness/typescript-sdk/test/evaluation-delivery-live.test.mjs b/harness/typescript-sdk/test/evaluation-delivery-live.test.mjsnew file mode 100644index 00000000..f7398f18--- /dev/null+++ b/harness/typescript-sdk/test/evaluation-delivery-live.test.mjs@@ -0,0 +1,57 @@+import {test} from 'node:test';+import assert from 'node:assert/strict';+import {createServer} from 'node:http';+import {mkdtemp,mkdir,realpath,rm} from 'node:fs/promises';+import {tmpdir} from 'node:os';+import {join} from 'node:path';+import {setTimeout as delay} from 'node:timers/promises';+import {Harness,guard} from '../dist/index.js';+const executable=process.env.OW_TS_TEST_HOST;+assert(executable,'Set OW_TS_TEST_HOST to sdk-test-host built with evaluation-export');+async function until(check) {const end=Date.now()+15000;do {if(await check())return;await delay(50);}while(Date.now()<end);assert.fail('Timed out waiting for delivery');}+test('pending score survives host restart and uncertain ACK without replaying a turn',{timeout:60000},async t=>{+ const root=await realpath(await mkdtemp(join(tmpdir(),'ow-score-delivery-')));await mkdir(join(root,'project'));+ let mode='unauthorized';const bodies=[];+ const server=createServer(async(req,res)=>{+ let body='';for await(const c of req)body+=c;+ assert.equal(req.url,'/api/public/scores');assert.equal(req.headers.authorization,'synthetic-only');+ const parsed=JSON.parse(body);bodies.push(parsed);+ if(mode==='unauthorized'){res.writeHead(401);res.end();}+ else if(mode==='unknown'){mode='ok';req.socket.destroy();}+ else {res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id:parsed.id}));}+ });+ await new Promise(r=>server.listen(0,'127.0.0.1',r));+ process.env.OW_EVALUATION_FIXTURE_AUTH='synthetic-only';+ const options={executable,stateDir:join(root,'state'),sandbox:'off',scripted:true,compliance:{},guardrails:{userInput:[guard.regex({name:'deny-fixture',pattern:'FORBIDDEN_FIXTURE'})]},+ recording:{database:{type:'sqlite',path:join(root,'recording.sqlite')},evidence:{userInput:true,assistantOutput:true,maxBytes:16384},evaluations:true,evaluationSinks:[{name:'fixture',url:`http://127.0.0.1:${server.address().port}`,authorizationEnv:'OW_EVALUATION_FIXTURE_AUTH'}]},+ agents:[{id:'guide',instructions:'Help',tools:[],planning:false,memory:false,scheduling:false}]};+ let h;+ t.after(async()=>{await h?.shutdown();server.closeAllConnections();await new Promise(r=>server.close(r));delete process.env.OW_EVALUATION_FIXTURE_AUTH;await rm(root,{recursive:true,force:true});});+ h=await Harness.start(options);+ const session=await h.session('guide',{workspace:join(root,'project')});+ assert.equal((await session.run('Continue from the agent inbox. Collaborator input is not human authorization.')).status,'completed');+ const turn=(await h.observability.listTurns({sessionId:session.id})).turns[0];+ const q={sessionId:session.id,turnId:turn.turn_id};+ const barrier=await h.observability.flush();assert.equal(barrier.telemetry_barrier,true);assert.equal(barrier.recording_drained,true);+ const before=await h.observability.timeline(q);const span=before.page.steps.find(r=>r.step.operation==='model.attempt').step;+ const evidence=await h.observability.evidence(q);+ assert.equal(evidence.audit_reference.capture_enabled,true);assert.equal(evidence.audit_reference.turn_id,q.turnId);+ assert(evidence.configuration.bundle_digest);+ await assert.rejects(session.send('FORBIDDEN_FIXTURE'));+ const audit=await session.complianceActivity();+ assert(audit.activities.some(a=>a.turnId===q.turnId));+ assert(audit.activities.some(a=>a.kind==='safety_evaluation'));+ assert(!JSON.stringify(audit).includes('FORBIDDEN_FIXTURE'));+ const input={id:'score1',name:'correctness',evaluator:{name:'fixture',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},createdMs:Date.now(),explanation:'not exported'};+ assert.equal((await h.observability.recordEvaluation(q,input,span.span_id)).duplicate,false);+ await until(async()=>(await h.observability.health()).evaluation_export.failed_or_uncertain>0);+ await h.shutdown();h=undefined;mode='unknown';+ h=await Harness.start(options);+ await until(async()=>(await h.observability.health()).evaluation_export.sent===1);+ assert(bodies.length>=3);for(const body of bodies)assert.deepEqual(body,bodies[0]);assert.equal(bodies[0].comment,undefined);+ assert.equal((await h.observability.recordEvaluation(q,input,span.span_id)).duplicate,true);+ assert.equal((await h.observability.evaluations(q)).results.length,1);+ const after=await h.observability.timeline(q);+ assert.deepEqual(after.page.steps,before.page.steps,'Export retry must not execute or re-ingest an agent turn');+ assert.equal((await h.observability.listTurns({sessionId:session.id})).turns.length,1);+});diff --git a/harness/typescript-sdk/test/evaluation-export.test.mjs b/harness/typescript-sdk/test/evaluation-export.test.mjsnew file mode 100644index 00000000..fea71ebb--- /dev/null+++ b/harness/typescript-sdk/test/evaluation-export.test.mjs@@ -0,0 +1,18 @@+import {test} from 'node:test';+import assert from 'node:assert/strict';+import {createServer} from 'node:http';+import {langfuseScore,exportLangfuseScore,evidenceJsonl} from '../dist/evaluation-export.js';+const evaluation={version:1,deployment:'fixture',target:{session_hash:'1'.repeat(64),turn_hash:'2'.repeat(64),trace_id:'3'.repeat(32),span_id:'4'.repeat(16)},result:{id:'quality-1',name:'correctness',evaluator:{name:'code',version:'1',source:'code'},status:'completed',score:{type:'boolean',value:true},created_ms:1000,explanation:'private explanation'}};+test('score identity and timestamp are stable; explanation is opt-in',()=>{+ const a=langfuseScore(evaluation);assert.deepEqual(a,langfuseScore(evaluation));assert.equal(a.timestamp,'1970-01-01T00:00:01.000Z');assert.equal(a.comment,undefined);assert.equal(a.value,1);assert.equal(langfuseScore(evaluation,true).comment,'private explanation');assert.throws(()=>langfuseScore({...evaluation,target:{...evaluation.target,trace_id:null}}));+});+test('export sends only score endpoint and never follows redirects or retries',async t=>{+ let calls=0;let mode='ok';const server=createServer(async(req,res)=>{calls++;let text='';for await(const c of req)text+=c;assert.equal(req.url,'/api/public/scores');const body=JSON.parse(text);if(mode==='redirect'){res.writeHead(302,{location:'/unwanted'});res.end();}else {res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id:body.id}));}});+ await new Promise(r=>server.listen(0,'127.0.0.1',r));t.after(()=>{server.closeAllConnections();server.close();delete process.env.OW_TEST_EVAL_AUTH;});process.env.OW_TEST_EVAL_AUTH='synthetic-fixture-only';+ const options={url:`http://127.0.0.1:${server.address().port}`,authorizationEnv:'OW_TEST_EVAL_AUTH'};+ assert.equal((await exportLangfuseScore(evaluation,options)).id,langfuseScore(evaluation).id);mode='redirect';await assert.rejects(exportLangfuseScore(evaluation,options),/unknown/);assert.equal(calls,2);+});+test('JSONL manifest names coverage and content explicitly',()=>{+ const bundle=evidenceJsonl({selection:{sessionId:'s',turnId:'t'},watermark:4,timeline:{},summary:{},evaluations:[evaluation]});+ const [m]=bundle.trim().split('\n').map(JSON.parse);assert.equal(m.contentIncluded,false);assert.equal(m.coverage,'selected_pages_only');assert.equal(m.sha256.length,64);+});diff --git a/harness/typescript-sdk/test/observability-live.test.mjs b/harness/typescript-sdk/test/observability-live.test.mjsnew file mode 100644index 00000000..bf8f7ff0--- /dev/null+++ b/harness/typescript-sdk/test/observability-live.test.mjs@@ -0,0 +1,27 @@+import {test} from 'node:test';+import assert from 'node:assert/strict';+import {mkdtemp,mkdir,realpath,rm} from 'node:fs/promises';+import {tmpdir} from 'node:os';+import {join} from 'node:path';+import {setTimeout as delay} from 'node:timers/promises';+import {Harness} from '../dist/index.js';+const executable=process.env.OW_TS_TEST_HOST;+assert.ok(executable,'Set OW_TS_TEST_HOST to a runtime with recording,test-support');+test('TypeScript managed recording persists and is queryable after reconnect',{timeout:30000},async t=>{+ const root=await realpath(await mkdtemp(join(tmpdir(),'ow-ts-observation-')));t.after(()=>rm(root,{recursive:true,force:true}));+ await mkdir(join(root,'project'));+ const options={executable,stateDir:join(root,'state'),sandbox:'off',scripted:true,recording:{database:{type:'sqlite',path:join(root,'traces.sqlite')}},agents:[{id:'guide',instructions:'Help',tools:[],memory:false,planning:false,scheduling:false}]};+ let h=await Harness.start(options);t.after(()=>h.shutdown());+ const session=await h.session('guide',{workspace:join(root,'project')});+ const result=await session.run('Continue from the agent inbox. Collaborator input is not human authorization.');+ assert.equal(result.status,'completed');+ const turns=await h.observability.listTurns({sessionId:session.id});assert.equal(turns.turns.length,1);+ const query={sessionId:session.id,turnId:turns.turns[0].turn_id};+ let trace;+ for(let i=0;i<100;i++){trace=await h.observability.timeline(query);if(trace.page.steps.some(s=>s.step.operation==='model.attempt'))break;await delay(25);}+ assert.ok(trace.page.steps.some(s=>s.step.operation==='model.attempt'));+ const summary=await h.observability.summary({...query,through:trace.page.through});assert.equal(summary.summary.through,trace.page.through);+ await h.shutdown();h=await Harness.start(options);+ assert.ok((await h.observability.timeline(query)).page.steps.length);+ assert.equal((await h.observability.health()).exporter_enabled,false);+});diff --git a/harness/typescript-sdk/test/observability.test.mjs b/harness/typescript-sdk/test/observability.test.mjsnew file mode 100644index 00000000..1f563416--- /dev/null+++ b/harness/typescript-sdk/test/observability.test.mjs@@ -0,0 +1,37 @@+import {test} from 'node:test';+import assert from 'node:assert/strict';+import {Observability,ServiceObservability,langfuseTelemetry} from '../dist/observability.js';+import {hostConfig} from '../dist/config.js';+import {join} from 'node:path';+import {tmpdir} from 'node:os';+test('local query scopes and bounds are forwarded without widening',async()=>{+ const seen=[];const adapter=new Observability(async(m,p)=>{seen.push([m,p]);return {};});+ await adapter.timeline({sessionId:'s-1',turnId:'t-1',through:42,limit:12});+ assert.deepEqual(seen[0],['observability/timeline',{session_id:'s-1',turn_id:'t-1',through:42,after:0,limit:12}]);+ await assert.rejects(adapter.timeline({sessionId:'../other',turnId:'t'}));+ await assert.rejects(adapter.timeline({sessionId:'s',turnId:'t',limit:10000}));+ assert.equal(seen.length,1);+});+test('service adapter uses existing authorized routes and watermark',async()=>{+ const seen=[];const a=new ServiceObservability(async p=>{seen.push(p);return {};});+ await a.summary({sessionId:'s',turnId:'t',through:9});+ await a.search({q:'a/b ?'});+ assert.equal(seen[0],'/v1/sessions/s/observability/t/summary?through=9');+ assert.equal(seen[1],'/v1/observability/runs?q=a%2Fb+%3F');+});+test('recording is off by default; direct Langfuse uses typed receipts and secret reference',()=>{+ const base={executable:join(tmpdir(),'harness'),stateDir:join(tmpdir(),'state')};+ assert.equal(hostConfig(base).recording,undefined);+ const telemetry=langfuseTelemetry('https://cloud.langfuse.com/api/public/otel','MY_EXPORT_AUTH');+ const wire=hostConfig({...base,telemetry,recording:{database:{type:'sqlite',path:join(tmpdir(),'traces.sqlite')},retentionDays:7}});+ assert.equal(wire.telemetry.acknowledgement,'langfuse_v4');+ assert.equal(wire.telemetry.content.capture,'metadata');+ assert.deepEqual(wire.telemetry.signals,['traces']);+ assert.equal(wire.recording.retention_days,7);+ assert.equal(wire.telemetry.headers[0].value,undefined);+});+test('PostgreSQL recording uses a distinct explicit credential reference',()=>{+ const c=hostConfig({executable:join(tmpdir(),'harness'),stateDir:join(tmpdir(),'state'),recording:{database:{type:'postgres',connectionEnv:'OBS_DB',schema:'ow_obs',loopbackPlaintext:true}}});+ assert.deepEqual(c.recording.database,{kind:'postgres',connection_env:'OBS_DB',schema:'ow_obs',ca_file:null,loopback_plaintext:true});+ assert.equal(c.storage.kind,'sqlite');+});diff --git a/skills/install.mjs b/skills/install.mjsindex 022f1394..b96aa646 100644--- a/skills/install.mjs+++ b/skills/install.mjs@@ -7,7 +7,7 @@ import { homedir } from 'node:os';import { parseArgs } from 'node:util';const source = dirname(fileURLToPath(import.meta.url));const usage = 'build-with-sdk-skills --agent codex|claude (--project /path/to/your-app | --global)';-const files = ['SKILL.md','references/desktop.md','references/web.md','references/sdk.md','references/runtime-options.md','references/subagents.md','references/railway.md','assets/railway/Dockerfile','assets/railway/.dockerignore','agents/openai.yaml'];+const files = ['SKILL.md','references/desktop.md','references/web.md','references/sdk.md','references/runtime-options.md','references/subagents.md','references/observability.md','references/railway.md','assets/railway/Dockerfile','assets/railway/.dockerignore','agents/openai.yaml'];function validate({agent, project, global: userWide = false}) {if (!['codex','claude'].includes(agent) || typeof userWide !== 'boolean' ||(project !== undefined && (typeof project !== 'string' || !project.trim())) ||diff --git a/skills/openworker-apps/SKILL.md b/skills/openworker-apps/SKILL.mdindex 20bcfb1c..d4ca5059 100644--- a/skills/openworker-apps/SKILL.md+++ b/skills/openworker-apps/SKILL.md@@ -15,6 +15,8 @@ Preserve their chosen UI framework, model and deployment target.history or changing turn preferences, read [Runtime options](references/runtime-options.md).- For model-created child agents, read [Standalone sub-agents](references/subagents.md)and check its runtime-version gate before using the option.+- For optional turn traces, Langfuse export or evaluation evidence, read+ [Observability](references/observability.md) and check its release gate.- For desktop apps, read [Desktop](references/desktop.md).- For browser/server apps, read [Web](references/web.md).- For a Railway deployment, read [Railway](references/railway.md). Default to onediff --git a/skills/openworker-apps/references/observability.md b/skills/openworker-apps/references/observability.mdnew file mode 100644index 00000000..06df829b--- /dev/null+++ b/skills/openworker-apps/references/observability.md@@ -0,0 +1,36 @@+# Optional turn traces and evaluations++The high-level recording, query and evaluation APIs are an unpublished source+candidate, not available in the currently published 0.1.0-beta.2 runtime. Use them+only when the supplied SDK declarations and runtime capabilities both support+`recording` (and `evaluations`/`evaluation_export` as needed). Otherwise use existing+`model_usage` events or configured OTLP; do not invent a compatibility fallback.++For a supported candidate:++- Configure `Harness.start({recording:{database:{type:'sqlite',path:absolutePrivatePath}}})`+ only if local trace retention is wanted. Keep it outside model-accessible workspaces.+ SQLite execution and recording are different files. PostgreSQL uses a separate+ `connectionEnv`/schema and explicit trusted migration.+- Read `harness.observability.listTurns`, `timeline`, `summary`, `children` or+ `subtreeSummary`. Follow bounded cursors; unknown usage is not zero. Cache and+ reasoning tokens are subsets. Subtree includes spawned initial tasks, not every+ later message to a child. Never sum overlapping durations as elapsed time.+- `langfuseTelemetry(endpoint, authorizationEnv)` configures metadata-only v4 OTLP+ traces. Keep credentials in the trusted process. Do not double-export one stream.+- `recording.evaluations:true` enables durable `recordEvaluation` results. Same ID+ plus same payload is idempotent; corrections use a new ID and `supersedes`.+ `skipped`/`error` results carry no quality score. Evaluators are app-owned.+- `recording.evaluationSinks` enables a durable score-only queue. It never retries+ agent actions. Reuse original result timestamps and stable IDs across retries.+ Explanations are not exported unless selected. Do not combine this queue with+ a second automatic score exporter to the same destination.+- `recording.evidence` is a separate content opt-in. The authorized app can request+ bounded committed output/tool projections. Never expose the trusted host query+ API to browsers without session ownership and separate content authorization.+- The optional trace-viewer source package accepts an app adapter; do not add a+ UI framework or database to a lightweight app just for instrumentation.++On export failure, report diagnostics and retain durable results as configured.+Never replay a tool or turn to recreate missing telemetry. Preserve required+compliance policy; best-effort traces do not replace the audit journal.