Skip to content

The Open Standard for Generative UI

Add sampled Lang Core system prompt telemetry (#930)

Zahle Khandbad744d93d9
diff --git a/packages/lang-core/README.md b/packages/lang-core/README.md
index e41565643..a62851d77 100644
--- a/packages/lang-core/README.md
+++ b/packages/lang-core/README.md
@@ -105,6 +105,15 @@ const merged = mergeStatements(original, patch);
**`ToolSpec`** describes a tool for prompt generation (name, description, inputSchema, outputSchema). Shape inspired by MCP's tool schema.
+## Telemetry
+
+Lang Core collects limited usage telemetry from 10% of successful server-side
+`generateSystemPrompt()` calls to help improve OpenUI. It does not collect
+application-user data, prompts, messages, generated output, credentials, or raw
+component and tool definitions. Project and prompt-configuration identifiers
+are hashed locally before being sent, and browsers do not send telemetry. Set
+`OPENUI_TELEMETRY_DISABLED=1` or `DO_NOT_TRACK=1` to disable it.
+
### Runtime
| Export | Description |
diff --git a/packages/lang-core/src/parser/prompt.ts b/packages/lang-core/src/parser/prompt.ts
index 42cd20c90..5c3e5da2e 100644
--- a/packages/lang-core/src/parser/prompt.ts
+++ b/packages/lang-core/src/parser/prompt.ts
@@ -1,3 +1,4 @@
+import { recordSystemPromptGeneration } from "../telemetry";
import { BUILTINS, LAZY_BUILTIN_DEFS } from "./builtins";
import type { LibraryJSONSchema } from "./types";
@@ -709,10 +710,14 @@ export function generateSystemPrompt(spec: SystemPromptSpec): string;
export function generateSystemPrompt(spec: PromptSpec): string;
export function generateSystemPrompt(spec: SystemPromptSpec | PromptSpec): string {
if (!isSystemPromptSpec(spec)) {
- return generatePrompt(spec);
+ const prompt = generatePrompt(spec);
+ recordSystemPromptGeneration(spec, "legacy_prompt_spec");
+ return prompt;
}
const merged: PromptSpec = { ...spec.library, ...spec.promptOptions };
- return generatePrompt(merged);
+ const prompt = generatePrompt(merged);
+ recordSystemPromptGeneration(merged, "library_spec");
+ return prompt;
}
// use `library` to discriminate SystemPromptSpec from the deprecated base-PromptSpec
diff --git a/packages/lang-core/src/telemetry.ts b/packages/lang-core/src/telemetry.ts
new file mode 100644
index 000000000..b27be1982
--- /dev/null
+++ b/packages/lang-core/src/telemetry.ts
@@ -0,0 +1,420 @@
+import type { PromptSpec } from "./parser/prompt";
+
+/**
+ * Telemetry disclosure
+ *
+ * Sent for sampled server-side generations:
+ * - Event timestamp, SDK and runtime versions, environment and CI status
+ * - API input shape, component count, tool count, and schema/sample versions
+ * - Random event and runtime identifiers
+ * - A locally computed SHA-256 prompt-configuration hash
+ * - A locally computed SHA-256 project hash when a project identifier is available
+ *
+ * The configuration hash covers the root component, component names, signatures and
+ * descriptions, component groups and notes, and generation-mode flags. The project hash
+ * uses the normalized Git origin, REPOSITORY_URL, or working directory. Only the hashes
+ * are sent.
+ *
+ * Not sent:
+ * - Prompts, preambles, examples, additional rules, or generated output
+ * - Raw component definitions, tool definitions, or tool examples
+ * - Git origins, working-directory paths, credentials, user identifiers, or chat data
+ *
+ * Browser and browser-worker environments never send this telemetry.
+ */
+declare const __OPENUI_LANG_CORE_VERSION__: string;
+
+const EVENT_NAME = "lang_core_system_prompt_generation_used";
+const SAMPLE_RATE = 0.1;
+const REQUEST_TIMEOUT_MS = 2_000;
+const SDK_VERSION =
+ typeof __OPENUI_LANG_CORE_VERSION__ === "string"
+ ? __OPENUI_LANG_CORE_VERSION__
+ : "0.0.0-development";
+const CAPTURE_URL = "https://dgoeivjus9jfp.cloudfront.net/capture/";
+const POSTHOG_KEY = "phc_3OLW53x09ZTVZSV6BEpj5uycj3ooqR6KOemOjx04e3D";
+
+type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
+type InputShape = "library_spec" | "legacy_prompt_spec";
+type Environment = "production" | "development" | "test" | "unknown";
+type RuntimeName = "node" | "bun" | "deno" | "edge";
+
+interface ProcessLike {
+ cwd?: () => string;
+ env?: Record<string, string | undefined>;
+ getBuiltinModule?: (specifier: string) => unknown;
+ release?: { name?: string };
+ versions?: Record<string, string | undefined>;
+}
+
+interface RuntimeInfo {
+ name: RuntimeName;
+ version?: string;
+ env?: Record<string, string | undefined>;
+ process?: ProcessLike;
+}
+
+interface TelemetryState {
+ projectHash?: Promise<string | undefined>;
+ runtimeId: string;
+}
+
+interface CaptureProperties {
+ distinct_id: string;
+ $process_person_profile: false;
+ event_id: string;
+ telemetry_schema_version: 1;
+ system_prompt_config_hash_version: 1;
+ system_prompt_config_hash: string;
+ project_hash_version?: 1;
+ project_hash?: string;
+ component_count: number;
+ tool_count: number;
+ sdk_name: "@openuidev/lang-core";
+ sdk_version: string;
+ api_surface: "generate_system_prompt";
+ input_shape: InputShape;
+ runtime: RuntimeName;
+ runtime_version?: string;
+ environment: Environment;
+ ci: boolean;
+ sample_rate: 0.1;
+}
+
+const STATE_KEY = Symbol.for("@openuidev/lang-core/telemetry/v1");
+
+export function buildSystemPromptConfigProjection(spec: PromptSpec): Json {
+ const hasTools = (spec.tools?.length ?? 0) > 0;
+ const toolCalls = spec.toolCalls ?? hasTools;
+ const bindings = spec.bindings ?? toolCalls;
+
+ const components = Object.entries(spec.components)
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([name, component]) => ({
+ name,
+ signature: component.signature,
+ description: component.description ?? null,
+ }));
+
+ const componentGroups = (spec.componentGroups ?? [])
+ .map((group) => ({
+ name: group.name,
+ components: [...group.components].sort(),
+ notes: [...(group.notes ?? [])],
+ }))
+ .sort((left, right) => left.name.localeCompare(right.name));
+
+ return {
+ root: spec.root ?? "Root",
+ components,
+ componentGroups,
+ modes: {
+ toolCalls,
+ bindings,
+ editMode: spec.editMode === true,
+ inlineMode: spec.inlineMode === true,
+ },
+ };
+}
+
+async function sha256(value: string): Promise<string> {
+ const subtle = globalThis.crypto?.subtle;
+ if (!subtle) throw new Error("Web Crypto is unavailable");
+
+ const bytes = new TextEncoder().encode(value);
+ const digest = await subtle.digest("SHA-256", bytes);
+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+export function calculateSystemPromptConfigHash(spec: PromptSpec): Promise<string> {
+ // The projection constructs object keys deterministically and sorts unordered collections.
+ const canonicalJson = JSON.stringify(buildSystemPromptConfigProjection(spec));
+ return sha256(canonicalJson);
+}
+
+function getProcess(): ProcessLike | undefined {
+ const candidate = (globalThis as typeof globalThis & { process?: unknown }).process;
+ if (!candidate || typeof candidate !== "object") return undefined;
+ return candidate as ProcessLike;
+}
+
+function hasBrowserWindow(): boolean {
+ const target = globalThis as typeof globalThis & {
+ document?: unknown;
+ navigator?: { product?: string };
+ window?: unknown;
+ };
+ return target.navigator?.product === "ReactNative" || Boolean(target.window && target.document);
+}
+
+function isBrowserWorker(): boolean {
+ const target = globalThis as typeof globalThis & {
+ WorkerGlobalScope?: { prototype: object };
+ };
+ const workerGlobalScope = target.WorkerGlobalScope;
+ return Boolean(workerGlobalScope && workerGlobalScope.prototype.isPrototypeOf(globalThis));
+}
+
+function detectRuntime(): RuntimeInfo | undefined {
+ if (hasBrowserWindow()) return undefined;
+
+ const target = globalThis as typeof globalThis & {
+ Bun?: { version?: string };
+ Deno?: {
+ version?: { deno?: string };
+ };
+ EdgeRuntime?: unknown;
+ WebSocketPair?: unknown;
+ };
+ const processLike = getProcess();
+
+ if (target.Bun && typeof target.Bun === "object") {
+ return {
+ name: "bun",
+ version: target.Bun.version,
+ env: processLike?.env,
+ process: processLike,
+ };
+ }
+
+ if (target.Deno && typeof target.Deno === "object") {
+ return { name: "deno", version: target.Deno.version?.deno };
+ }
+
+ if (typeof target.EdgeRuntime === "string") {
+ return { name: "edge", version: target.EdgeRuntime };
+ }
+
+ if (typeof target.WebSocketPair === "function") {
+ return { name: "edge" };
+ }
+
+ if (isBrowserWorker()) return undefined;
+
+ if (
+ processLike?.release?.name === "node" ||
+ (typeof processLike?.versions?.node === "string" && processLike.versions.node.length > 0)
+ ) {
+ return {
+ name: "node",
+ version: processLike.versions?.node,
+ env: processLike.env,
+ process: processLike,
+ };
+ }
+
+ return undefined;
+}
+
+function getEnvironment(env: Record<string, string | undefined> | undefined): Environment {
+ const value = env?.NODE_ENV;
+ if (value === "production" || value === "development" || value === "test") return value;
+ return "unknown";
+}
+
+function isTruthyOptOut(value: string | undefined): boolean {
+ return value === "1" || value?.toLowerCase() === "true";
+}
+
+function isCi(env: Record<string, string | undefined> | undefined): boolean {
+ if (!env) return false;
+ return [
+ "CI",
+ "CONTINUOUS_INTEGRATION",
+ "BUILD_NUMBER",
+ "GITHUB_ACTIONS",
+ "GITLAB_CI",
+ "BUILDKITE",
+ "CIRCLECI",
+ "VERCEL",
+ "NETLIFY",
+ ].some((name) => {
+ const value = env[name];
+ return Boolean(value && value !== "0" && value.toLowerCase() !== "false");
+ });
+}
+
+function getState(): TelemetryState {
+ const registry = globalThis as typeof globalThis & { [STATE_KEY]?: TelemetryState };
+ registry[STATE_KEY] ??= {
+ runtimeId: globalThis.crypto.randomUUID(),
+ };
+ return registry[STATE_KEY];
+}
+
+function normalizeRepositoryIdentifier(rawValue: string): string | undefined {
+ const value = rawValue.trim();
+ if (!value) return undefined;
+
+ try {
+ const url = new URL(value);
+ const pathname = decodeURIComponent(url.pathname)
+ .replace(/^\/+|\/+$/g, "")
+ .replace(/\.git$/i, "");
+ if (url.hostname && pathname) return `${url.hostname.toLowerCase()}/${pathname}`;
+ } catch {
+ // SCP-style Git origins and local paths are handled below.
+ }
+
+ if (/^[A-Za-z]:[\\/]/.test(value)) {
+ return value.replace(/\\/g, "/").replace(/\/+$/g, "");
+ }
+
+ const scpMatch = /^(?:[^@/\s]+@)?([^:/\s]+):(.+)$/.exec(value);
+ if (scpMatch) {
+ const host = scpMatch[1]?.toLowerCase();
+ const pathname = scpMatch[2]?.replace(/^\/+|\/+$/g, "").replace(/\.git$/i, "");
+ if (host && pathname) return `${host}/${pathname}`;
+ }
+
+ // Local paths are still useful as a last-resort, deployment-local identifier.
+ return value.replace(/\\/g, "/").replace(/\/+$/g, "");
+}
+
+function readGitOrigin(processLike: ProcessLike): Promise<string | undefined> {
+ const childProcess = processLike.getBuiltinModule?.("node:child_process") as
+ | {
+ execFile?: (
+ file: string,
+ args: string[],
+ options: Record<string, unknown>,
+ callback: (error: unknown, stdout?: string) => void,
+ ) => unknown;
+ }
+ | undefined;
+ const execFile = childProcess?.execFile;
+ if (!execFile) return Promise.resolve(undefined);
+
+ // Repository discovery happens at most once and must never block the event loop.
+ return new Promise((resolve) => {
+ try {
+ execFile(
+ "git",
+ ["config", "--local", "--get", "remote.origin.url"],
+ {
+ encoding: "utf8",
+ timeout: 1_000,
+ windowsHide: true,
+ },
+ (error, stdout) => {
+ const origin = stdout?.trim();
+ resolve(error || !origin ? undefined : origin);
+ },
+ );
+ } catch {
+ resolve(undefined);
+ }
+ });
+}
+
+async function getRepositoryIdentifier(
+ processLike: ProcessLike | undefined,
+): Promise<string | undefined> {
+ if (!processLike) return undefined;
+
+ const rawValue =
+ (await readGitOrigin(processLike)) || processLike.env?.REPOSITORY_URL || processLike.cwd?.();
+ return rawValue ? normalizeRepositoryIdentifier(rawValue) : undefined;
+}
+
+export function calculateProjectHash(repositoryIdentifier: string): Promise<string> {
+ const normalized = normalizeRepositoryIdentifier(repositoryIdentifier);
+ if (!normalized) return Promise.reject(new TypeError("Repository identifier is empty"));
+ return sha256(normalized);
+}
+
+function getProjectHash(state: TelemetryState, runtime: RuntimeInfo): Promise<string | undefined> {
+ if (runtime.name !== "node") return Promise.resolve(undefined);
+
+ state.projectHash ??= Promise.resolve()
+ .then(() => getRepositoryIdentifier(runtime.process))
+ .then((identifier) => (identifier ? calculateProjectHash(identifier) : undefined))
+ .catch(() => undefined);
+ return state.projectHash;
+}
+
+async function sendCapture(
+ state: TelemetryState,
+ spec: PromptSpec,
+ configHash: Promise<string>,
+ inputShape: InputShape,
+ runtime: RuntimeInfo,
+ environment: Environment,
+): Promise<void> {
+ const [systemPromptConfigHash, projectHash] = await Promise.all([
+ configHash,
+ getProjectHash(state, runtime),
+ ]);
+
+ const properties: CaptureProperties = {
+ distinct_id: state.runtimeId,
+ $process_person_profile: false,
+ event_id: globalThis.crypto.randomUUID(),
+ telemetry_schema_version: 1,
+ system_prompt_config_hash_version: 1,
+ system_prompt_config_hash: systemPromptConfigHash,
+ component_count: Object.keys(spec.components).length,
+ tool_count: spec.tools?.length ?? 0,
+ sdk_name: "@openuidev/lang-core",
+ sdk_version: SDK_VERSION,
+ api_surface: "generate_system_prompt",
+ input_shape: inputShape,
+ runtime: runtime.name,
+ runtime_version: runtime.version,
+ environment,
+ ci: isCi(runtime.env),
+ sample_rate: SAMPLE_RATE,
+ ...(projectHash
+ ? {
+ project_hash_version: 1 as const,
+ project_hash: projectHash,
+ }
+ : {}),
+ };
+
+ await globalThis.fetch(CAPTURE_URL, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ api_key: POSTHOG_KEY,
+ event: EVENT_NAME,
+ timestamp: new Date().toISOString(),
+ properties,
+ }),
+ keepalive: true,
+ signal: globalThis.AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+}
+
+export function recordSystemPromptGeneration(spec: PromptSpec, inputShape: InputShape): void {
+ try {
+ const runtime = detectRuntime();
+ if (
+ !runtime ||
+ typeof globalThis.fetch !== "function" ||
+ !globalThis.crypto?.subtle ||
+ typeof globalThis.crypto.randomUUID !== "function" ||
+ typeof globalThis.AbortSignal?.timeout !== "function"
+ ) {
+ return;
+ }
+
+ const env = runtime.env;
+ const environment = getEnvironment(env);
+ if (isTruthyOptOut(env?.DO_NOT_TRACK) || isTruthyOptOut(env?.OPENUI_TELEMETRY_DISABLED)) {
+ return;
+ }
+
+ // Reject 90% of calls before projection, hashing, repository lookup, or payload allocation.
+ if (Math.random() >= SAMPLE_RATE) return;
+
+ const state = getState();
+ const configHash = calculateSystemPromptConfigHash(spec);
+
+ void Promise.resolve()
+ .then(() => sendCapture(state, spec, configHash, inputShape, runtime, environment))
+ .catch(() => undefined);
+ } catch {
+ // Telemetry must never affect prompt generation.
+ }
+}
diff --git a/packages/lang-core/tsdown.config.ts b/packages/lang-core/tsdown.config.ts
index bd5f1a3e1..74c52dbb9 100644
--- a/packages/lang-core/tsdown.config.ts
+++ b/packages/lang-core/tsdown.config.ts
@@ -1,4 +1,5 @@
import { defineConfig } from "tsdown";
+import packageJson from "./package.json" with { type: "json" };
export default defineConfig({
entry: ["src/index.ts"],
@@ -8,6 +9,9 @@ export default defineConfig({
target: "es2022",
outDir: "dist",
clean: true,
+ define: {
+ __OPENUI_LANG_CORE_VERSION__: JSON.stringify(packageJson.version),
+ },
deps: {
neverBundle: [/^(?![./]|[A-Za-z]:[/\\])/],
},