Skip to content

The Open Standard for Generative UI

Instrument parser.parse runtime telemetry (#995)

Zahle Khanea2115d0cb41
diff --git a/packages/lang-core/README.md b/packages/lang-core/README.md
index fec6ee45f..453222ec7 100644
--- a/packages/lang-core/README.md
+++ b/packages/lang-core/README.md
@@ -109,10 +109,17 @@ const merged = mergeStatements(original, patch);
Lang Core sends pseudonymous installation telemetry during `postinstall`.
Runtime usage telemetry is opt-in: set `OPENUI_RUNTIME_TELEMETRY_ENABLED=1` to allow
-limited telemetry from 10% of successful server-side `generateSystemPrompt()`
-calls. It does not collect application-user data, prompts, messages, generated
-output, credentials, or raw component and tool definitions. Project identifiers
-are hashed locally before being sent, and browsers do not send telemetry.
+limited telemetry from 10% of eligible server-side `generateSystemPrompt()` and
+`createParser().parse()` calls. Parser events include only the outcome, incomplete/root
+booleans, structural counts, fixed validation error-code counts, runtime metadata, and
+an optional locally hashed project identifier. They do not include OpenUI Lang source,
+schemas, component/prop/statement names, validation messages or paths, unresolved or
+orphaned names, exception content, credentials, application-user identifiers, or chat
+data. Browser and worker runtimes do not send runtime telemetry. Direct low-level
+`parse()` and streaming-parser calls are not included in the parser event.
+
+Capture servers can observe standard network metadata such as source IP even though
+it is not included in event properties.
Set `OPENUI_TELEMETRY_DISABLED=1` or `DO_NOT_TRACK=1` to disable telemetry
(these also override `OPENUI_RUNTIME_TELEMETRY_ENABLED`). Set
diff --git a/packages/lang-core/src/parser/parser.ts b/packages/lang-core/src/parser/parser.ts
index 41bd2e7bc..2d201d559 100644
--- a/packages/lang-core/src/parser/parser.ts
+++ b/packages/lang-core/src/parser/parser.ts
@@ -1,3 +1,8 @@
+import {
+ captureParserParseException,
+ captureParserParseResult,
+ prepareParserParseTelemetry,
+} from "../telemetry/runtime";
import type { ASTNode, Statement } from "./ast";
import { isASTNode, walkAST } from "./ast";
import { isBuiltin, RESERVED_CALLS } from "./builtins";
@@ -665,7 +670,15 @@ export function createParser(schema: LibraryJSONSchema, rootName?: string): Pars
const paramMap = compileSchema(schema);
return {
parse(input: string): ParseResult {
- return parse(input, paramMap, rootName);
+ const telemetry = prepareParserParseTelemetry();
+ try {
+ const result = parse(input, paramMap, rootName);
+ captureParserParseResult(telemetry, result);
+ return result;
+ } catch (error) {
+ captureParserParseException(telemetry);
+ throw error;
+ }
},
};
}
diff --git a/packages/lang-core/src/telemetry/runtime.ts b/packages/lang-core/src/telemetry/runtime.ts
index 4b914e3d3..0eadf4abb 100644
--- a/packages/lang-core/src/telemetry/runtime.ts
+++ b/packages/lang-core/src/telemetry/runtime.ts
@@ -1,4 +1,5 @@
import type { PromptSpec } from "../parser/prompt";
+import type { ParseResult, ValidationErrorCode } from "../parser/types";
import {
getPostHogConfig,
isRuntimeTelemetryEnabled,
@@ -12,29 +13,36 @@ import {
*
* Opt-in: nothing is sent unless OPENUI_RUNTIME_TELEMETRY_ENABLED=1 is set.
*
- * Sent for sampled server-side generations:
+ * Sent for sampled server-side system-prompt generations:
* - Event timestamp, SDK and runtime versions, environment, CI status, and CI provider category
* - API input shape, component count, tool count, and schema/sample versions
* - Random event and runtime identifiers
* - A locally computed SHA-256 prompt-configuration hash
* - A locally computed SHA-256 project hash when a project identifier is available
*
+ * Sent for sampled server-side createParser().parse() calls:
+ * - The same event/runtime metadata and optional locally computed project hash
+ * - Outcome, incomplete/root booleans, structural counts, and fixed validation-code counts
+ *
* The configuration hash covers the root component, component names, signatures and
* descriptions, component groups and notes, and generation-mode flags. The project hash
* uses the normalized Git origin, REPOSITORY_URL, or working directory. Only the hashes
* are sent.
*
* Not sent:
- * - Prompts, preambles, examples, additional rules, or generated output
+ * - Prompts, preambles, examples, additional rules, OpenUI Lang source, or generated output
* - Raw component definitions, tool definitions, or tool examples
+ * - Component/prop/statement names, error messages/paths, unresolved/orphaned names, or exceptions
* - Git origins, working-directory paths, credentials, user identifiers, or chat data
*
* Browser and browser-worker environments never send this telemetry.
*/
declare const __OPENUI_LANG_CORE_VERSION__: string;
-const EVENT_NAME = "lang_core_system_prompt_generation_used";
-const SAMPLE_RATE = 0.1;
+const SYSTEM_PROMPT_EVENT_NAME = "lang_core_system_prompt_generation_used";
+const PARSER_PARSE_EVENT_NAME = "lang_core_parser_parse_used";
+const SYSTEM_PROMPT_SAMPLE_RATE = 0.1;
+const PARSER_PARSE_SAMPLE_RATE = 0.1;
const SDK_VERSION =
typeof __OPENUI_LANG_CORE_VERSION__ === "string"
? __OPENUI_LANG_CORE_VERSION__
@@ -65,7 +73,7 @@ interface TelemetryState {
runtimeId: string;
}
-interface CaptureProperties {
+interface SystemPromptCaptureProperties {
distinct_id: string;
$process_person_profile: false;
event_id: string;
@@ -85,7 +93,54 @@ interface CaptureProperties {
environment: Environment;
ci: boolean;
ci_name?: string;
- sample_rate: 0.1;
+ sample_rate: typeof SYSTEM_PROMPT_SAMPLE_RATE;
+}
+
+type ParserParseOutcome = "valid" | "invalid" | "no_renderable_root" | "threw";
+
+interface ParserParseResultProperties {
+ outcome: Exclude<ParserParseOutcome, "threw">;
+ incomplete: boolean;
+ has_renderable_root: boolean;
+ statement_count: number;
+ unresolved_count: number;
+ orphaned_count: number;
+ validation_error_count: number;
+ unknown_component_count: number;
+ missing_required_count: number;
+ null_required_count: number;
+ inline_reserved_count: number;
+ excess_args_count: number;
+}
+
+interface ParserParseThrownProperties {
+ outcome: "threw";
+}
+
+type ParserParseOutcomeProperties = ParserParseResultProperties | ParserParseThrownProperties;
+
+interface ParserParseCaptureProperties {
+ distinct_id: string;
+ $process_person_profile: false;
+ event_id: string;
+ telemetry_schema_version: typeof TELEMETRY_SCHEMA_VERSION;
+ project_hash_version?: 1;
+ project_hash?: string;
+ sdk_name: "@openuidev/lang-core";
+ sdk_version: string;
+ api_surface: "parser.parse";
+ runtime: RuntimeName;
+ runtime_version?: string;
+ environment: Environment;
+ ci: boolean;
+ ci_name?: string;
+ sample_rate: typeof PARSER_PARSE_SAMPLE_RATE;
+}
+
+export interface ParserParseCaptureContext {
+ state: TelemetryState;
+ runtime: RuntimeInfo;
+ environment: Environment;
}
interface CiInfoSnapshot {
@@ -305,7 +360,28 @@ async function getCIInfo(runtime: RuntimeInfo): Promise<CiInfoSnapshot> {
}
}
-async function sendCapture(
+async function postCapture(
+ event: string,
+ properties:
+ SystemPromptCaptureProperties | (ParserParseCaptureProperties & ParserParseOutcomeProperties),
+ runtime: RuntimeInfo,
+): Promise<void> {
+ const postHog = getPostHogConfig(runtime.env);
+ await globalThis.fetch(postHog.captureUrl, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ api_key: postHog.apiKey,
+ event,
+ timestamp: new Date().toISOString(),
+ properties,
+ }),
+ keepalive: true,
+ signal: globalThis.AbortSignal.timeout(TELEMETRY_REQUEST_TIMEOUT_MS),
+ });
+}
+
+async function sendSystemPromptCapture(
state: TelemetryState,
spec: PromptSpec,
configHash: Promise<string>,
@@ -319,8 +395,7 @@ async function sendCapture(
getCIInfo(runtime),
]);
- const postHog = getPostHogConfig(runtime.env);
- const properties: CaptureProperties = {
+ const properties: SystemPromptCaptureProperties = {
distinct_id: state.runtimeId,
$process_person_profile: false,
event_id: globalThis.crypto.randomUUID(),
@@ -338,7 +413,7 @@ async function sendCapture(
environment,
ci: ciInfo.isCI,
...(ciInfo.id ? { ci_name: ciInfo.id } : {}),
- sample_rate: SAMPLE_RATE,
+ sample_rate: SYSTEM_PROMPT_SAMPLE_RATE,
...(projectHash
? {
project_hash_version: 1 as const,
@@ -347,18 +422,7 @@ async function sendCapture(
: {}),
};
- await globalThis.fetch(postHog.captureUrl, {
- method: "POST",
- headers: { "content-type": "application/json" },
- body: JSON.stringify({
- api_key: postHog.apiKey,
- event: EVENT_NAME,
- timestamp: new Date().toISOString(),
- properties,
- }),
- keepalive: true,
- signal: globalThis.AbortSignal.timeout(TELEMETRY_REQUEST_TIMEOUT_MS),
- });
+ await postCapture(SYSTEM_PROMPT_EVENT_NAME, properties, runtime);
}
export function recordSystemPromptGeneration(spec: PromptSpec, inputShape: InputShape): void {
@@ -379,15 +443,154 @@ export function recordSystemPromptGeneration(spec: PromptSpec, inputShape: Input
if (!isRuntimeTelemetryEnabled(env)) return;
// Reject 90% of calls before projection, hashing, repository lookup, or payload allocation.
- if (Math.random() >= SAMPLE_RATE) return;
+ if (Math.random() >= SYSTEM_PROMPT_SAMPLE_RATE) return;
const state = getState();
const configHash = calculateSystemPromptConfigHash(spec);
void Promise.resolve()
- .then(() => sendCapture(state, spec, configHash, inputShape, runtime, environment))
+ .then(() =>
+ sendSystemPromptCapture(state, spec, configHash, inputShape, runtime, environment),
+ )
.catch(() => undefined);
} catch {
// Telemetry must never affect prompt generation.
}
}
+
+function getValidationErrorCounts(
+ result: ParseResult,
+): Omit<
+ ParserParseResultProperties,
+ | "outcome"
+ | "incomplete"
+ | "has_renderable_root"
+ | "statement_count"
+ | "unresolved_count"
+ | "orphaned_count"
+ | "validation_error_count"
+> {
+ const counts: Record<ValidationErrorCode, number> = {
+ "unknown-component": 0,
+ "missing-required": 0,
+ "null-required": 0,
+ "inline-reserved": 0,
+ "excess-args": 0,
+ };
+
+ for (const error of result.meta.errors) counts[error.code] += 1;
+
+ return {
+ unknown_component_count: counts["unknown-component"],
+ missing_required_count: counts["missing-required"],
+ null_required_count: counts["null-required"],
+ inline_reserved_count: counts["inline-reserved"],
+ excess_args_count: counts["excess-args"],
+ };
+}
+
+function summarizeParserParse(result: ParseResult): ParserParseResultProperties {
+ return {
+ outcome:
+ result.meta.errors.length > 0 ? "invalid" : result.root ? "valid" : "no_renderable_root",
+ incomplete: result.meta.incomplete,
+ has_renderable_root: result.root !== null,
+ statement_count: result.meta.statementCount,
+ unresolved_count: result.meta.unresolved.length,
+ orphaned_count: result.meta.orphaned.length,
+ validation_error_count: result.meta.errors.length,
+ ...getValidationErrorCounts(result),
+ };
+}
+
+async function sendParserParseCapture(
+ context: ParserParseCaptureContext,
+ outcome: ParserParseOutcomeProperties,
+): Promise<void> {
+ const { state, runtime, environment } = context;
+ const [projectHash, ciInfo] = await Promise.all([
+ getProjectHash(state, runtime),
+ getCIInfo(runtime),
+ ]);
+
+ const properties: ParserParseCaptureProperties & ParserParseOutcomeProperties = {
+ distinct_id: state.runtimeId,
+ $process_person_profile: false,
+ event_id: globalThis.crypto.randomUUID(),
+ telemetry_schema_version: TELEMETRY_SCHEMA_VERSION,
+ sdk_name: "@openuidev/lang-core",
+ sdk_version: SDK_VERSION,
+ api_surface: "parser.parse",
+ runtime: runtime.name,
+ runtime_version: runtime.version,
+ environment,
+ ci: ciInfo.isCI,
+ ...(ciInfo.id ? { ci_name: ciInfo.id } : {}),
+ sample_rate: PARSER_PARSE_SAMPLE_RATE,
+ ...outcome,
+ ...(projectHash
+ ? {
+ project_hash_version: 1 as const,
+ project_hash: projectHash,
+ }
+ : {}),
+ };
+
+ await postCapture(PARSER_PARSE_EVENT_NAME, properties, runtime);
+}
+
+export function prepareParserParseTelemetry(): ParserParseCaptureContext | undefined {
+ try {
+ const runtime = detectRuntime();
+ if (
+ !runtime ||
+ typeof globalThis.fetch !== "function" ||
+ typeof globalThis.crypto?.randomUUID !== "function" ||
+ typeof globalThis.AbortSignal?.timeout !== "function"
+ ) {
+ return undefined;
+ }
+
+ const env = runtime.env;
+ if (!isRuntimeTelemetryEnabled(env)) return undefined;
+
+ // Reject 90% of calls before telemetry state, project lookup, or payload allocation.
+ if (Math.random() >= PARSER_PARSE_SAMPLE_RATE) return undefined;
+
+ return {
+ state: getState(),
+ runtime,
+ environment: getEnvironment(env),
+ };
+ } catch {
+ return undefined;
+ }
+}
+
+export function captureParserParseResult(
+ context: ParserParseCaptureContext | undefined,
+ result: ParseResult,
+): void {
+ if (!context) return;
+
+ try {
+ const outcome = summarizeParserParse(result);
+ void Promise.resolve()
+ .then(() => sendParserParseCapture(context, outcome))
+ .catch(() => undefined);
+ } catch {
+ // Telemetry must never affect parsing.
+ }
+}
+
+export function captureParserParseException(context: ParserParseCaptureContext | undefined): void {
+ if (!context) return;
+
+ try {
+ void Promise.resolve()
+ .then(() => sendParserParseCapture(context, { outcome: "threw" }))
+ .catch(() => undefined);
+ } catch {
+ // Telemetry must never affect parsing or replace the original thrown value.
+ }
+}