Skip to content

The open-source app everyone uses to manage agents at work

8.6 KB
Raw Blame
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import { open } from "node:fs/promises";
import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";
import {
  applyProviderObservabilityEnvironment,
  writeManagedCodexObservability,
  type ProviderObservabilityLaunch,
} from "../drivers/agent-observability.js";
import type { EvalSessionRequest } from "./eval-session-contract.js";
import type { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";

type Gateway = Awaited<ReturnType<typeof prepareEvalGatewayRuntime>>;
const invalid = () => new Error("Invalid controlled eval observability configuration");
function object(value: unknown, names: string[]): Record<string, unknown> {
  if (!value || typeof value !== "object" || Array.isArray(value)
      || Object.keys(value).length !== names.length || Object.keys(value).some(key => !names.includes(key))) throw invalid();
  return value as Record<string, unknown>;
}
function text(value: unknown): string {
  if (typeof value !== "string" || !value || value.length > 4096 || /[\r\n\0]/.test(value)) throw invalid();
  return value;
}
function decodedMap(value: unknown): Record<string, string> {
  const result: Record<string, string> = Object.create(null);
  for (const item of text(value).split(",")) {
    const index = item.indexOf("=");
    if (index < 1) throw invalid();
    const key = item.slice(0, index);
    if (Object.hasOwn(result, key)) throw invalid();
    result[key] = decodeURIComponent(item.slice(index + 1));
  }
  return result;
}
function same(value: unknown, expected: Record<string, unknown>) {
  const actual = object(value, Object.keys(expected));
  if (Object.entries(expected).some(([key, item]) => actual[key] !== item)) throw invalid();
}
function parentHeader(value: unknown, projectId: string, traceId: string, spanId: string): string {
  const headers = decodedMap(value), parent = text(headers["x-bt-parent"]);
  object(headers, ["x-bt-parent"]);
  const bytes = Buffer.from(parent, "base64");
  if (bytes.toString("base64") !== parent || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw invalid();
  same(JSON.parse(bytes.subarray(3).toString("utf8")), { object_id: projectId, row_id: spanId, span_id: spanId, root_span_id: traceId });
  return parent;
}

export async function prepareEvalObservabilityRuntime(
  file: string,
  request: EvalSessionRequest,
  gateway: Gateway,
  environment: NodeJS.ProcessEnv = process.env,
) {
  let raw: Buffer;
  const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
  try {
    const stat = await handle.stat();
    if (!stat.isFile() || stat.size > 131_072) throw invalid();
    raw = Buffer.alloc(stat.size + 1);
    const { bytesRead } = await handle.read(raw, 0, raw.length, 0);
    if (bytesRead !== stat.size) throw invalid();
    raw = raw.subarray(0, bytesRead);
  } finally { await handle.close(); }
  let launch: ProviderObservabilityLaunch;
  let projectId: string, traceId: string, spanId: string;
  try {
    const config = object(JSON.parse(raw.toString("utf8")), ["schema", "campaignId", "executionId", "attemptId", "projectId", "traceId", "spanId", "baseUrl", "launch"]);
    projectId = text(config.projectId); traceId = text(config.traceId); spanId = text(config.spanId);
    if (config.schema !== "paperclip.runner-eval.observability/v1" || config.attemptId !== request.attemptId
        || config.campaignId !== gateway.evidence.campaignId || config.executionId !== gateway.evidence.executionId
        || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(projectId)
        || !/^[a-f0-9]{32}$/.test(traceId) || /^0+$/.test(traceId) || !/^[a-f0-9]{16}$/.test(spanId) || /^0+$/.test(spanId)
        || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(text(config.baseUrl))
        || Number(new URL(config.baseUrl as string).port) > 65535) throw invalid();
    const kind = request.provider === "codex" ? "codex" : request.provider === "acpx" && request.acpxAgent === "claude" ? "claude" : null;
    if (!kind) throw invalid();
    const source = object(config.launch, ["kind", "compatibilityKey", "environment", ...(kind === "codex" ? ["otelConfig"] : [])]);
    if (source.kind !== kind || !/^[A-Za-z0-9_-]{1,128}$/.test(text(source.compatibilityKey))) throw invalid();
    const state = new CapabilityMockControlPlaneAdapter(request.session.seed).snapshot();
    const attributes = {
      "paperclip.company.id": request.session.companyId ?? state.company.id,
      "paperclip.agent.id": request.session.actorId ?? state.actors[0]!.id,
      "paperclip.observability.revision": source.compatibilityKey,
    };
    const common = ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"];
    const env = object(source.environment, kind === "codex" ? common : [...common,
      "CLAUDE_CODE_ENABLE_TELEMETRY", "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "OTEL_LOGS_EXPORTER", "OTEL_TRACES_EXPORTER", "OTEL_METRICS_EXPORTER",
      "OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT", "OTEL_LOG_RAW_API_BODIES",
      "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES",
    ]);
    for (const value of Object.values(env)) text(value);
    for (const signal of ["LOGS", "TRACES"]) {
      if (!/^[1-9][0-9]{0,3}$/.test(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`] as string) || Number(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`]) > 3000) throw invalid();
      parentHeader(env[`OTEL_EXPORTER_OTLP_${signal}_HEADERS`], projectId, traceId, spanId);
    }
    if (kind === "codex") {
      const otel = object(source.otelConfig, ["exporter", "trace_exporter", "metrics_exporter", "log_user_prompt", "log_agent_responses", "log_guardian_assessments", "tool_result", "span_attributes"]);
      for (const [key, signal] of [["exporter", "logs"], ["trace_exporter", "traces"]]) {
        const exporter = object(otel[key!], ["otlp-http"]);
        const http = object(exporter["otlp-http"], ["endpoint", "protocol", "headers"]);
        if (http.endpoint !== `${config.baseUrl}/v1/${signal}` || http.protocol !== "binary") throw invalid();
        object(http.headers, []);
      }
      const tool = object(otel.tool_result, ["max_bytes"]);
      if (otel.metrics_exporter !== "none" || otel.log_guardian_assessments !== false
          || typeof otel.log_user_prompt !== "boolean" || typeof otel.log_agent_responses !== "boolean"
          || ![0, 65536].includes(tool.max_bytes as number)) throw invalid();
      same(otel.span_attributes, attributes);
    } else {
      const fixed: Record<string, string> = { CLAUDE_CODE_ENABLE_TELEMETRY: "1", CLAUDE_CODE_ENHANCED_TELEMETRY_BETA: "1",
        OTEL_LOGS_EXPORTER: "otlp", OTEL_TRACES_EXPORTER: "otlp", OTEL_METRICS_EXPORTER: "none", OTEL_LOG_RAW_API_BODIES: "0",
        OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: "http/protobuf", OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf",
        OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: `${config.baseUrl}/v1/logs`, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${config.baseUrl}/v1/traces` };
      if (Object.entries(fixed).some(([key, value]) => env[key] !== value)
          || ["OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT"].some(key => !["0", "1"].includes(env[key] as string))) throw invalid();
      same(decodedMap(env.OTEL_RESOURCE_ATTRIBUTES), attributes);
    }
    launch = structuredClone(source) as unknown as ProviderObservabilityLaunch;
  } catch { throw invalid(); }
  const token = environment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN;
  if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controlled eval OTLP token");
  const launchEnvironment = { ...launch.environment };
  for (const signal of ["LOGS", "TRACES"]) {
    const key = `OTEL_EXPORTER_OTLP_${signal}_HEADERS`;
    launchEnvironment[key] += `,authorization=${encodeURIComponent(`Bearer ${token}`)}`;
  }
  launch = { ...launch, environment: launchEnvironment };
  if (launch.kind === "codex") writeManagedCodexObservability(gateway.transportOptions.environment.CODEX_HOME!, launch.otelConfig!);
  return {
    transportOptions: { ...gateway.transportOptions, observability: launch,
      environment: applyProviderObservabilityEnvironment(gateway.transportOptions.environment, launch) },
    evidence: { mode: "controller_otlp" as const, configured: true, verification: "unverified" as const,
      projectId, traceId, spanId, descriptorSha256: createHash("sha256").update(raw).digest("hex") },
  };
}