πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// Client-side metadata structure analysis feeding the bar's `metadata.<key>`
// key suggestions. The API does not enumerate metadata keys (see
// observed-options.ts) and backend structure analysis is deferred, so the map
// of observed keys is built OPPORTUNISTICALLY from rows the user has already
// loaded: each fetch samples the visible rows, records their TOP-LEVEL
// metadata keys with the observed JSON value type, and unions the result into
// the persisted per-project store (stores/observedMetadataStore.ts). Accepted
// caveat: metadata the user has never loaded is never suggested.
//
// Top-level keys ONLY β deliberately not flattened into nested dot-paths.
// Metadata is stored as a flat Map(String, String): nested object values are
// JSON-encoded strings under their top-level key, and the stringObject filter
// matches the LITERAL top-level key ("we can only filter on the first level",
// StringObjectFilter in clickhouse-filter.ts; the metadata view's filter
// shortcut resolves the top-level key for the same reason β ValueCell.tsx).
// A flattened `metadata.scope.name` suggestion would lower to key
// "scope.name" and match nothing. Dot-paths still appear whenever producers
// use dotted top-level keys (the OTel-attribute shape, e.g.
// `gen_ai.request.model`) β those ARE literal keys and filter correctly.
// Object-valued keys are suggested with type "object"; their nested content
// is matched as a substring of the JSON-encoded branch (`metadata.scope:*v*`).
//
// Types are DISPLAY-ONLY hints: metadata filters always lower to
// `stringObject` (see fields.ts operatorIssue β numeric metadata comparisons
// are rejected), so a key observed with more than one type simply drops its
// hint rather than changing any filter behavior.
type MetadataLeafType = "string" | "number" | "boolean" | "array" | "object";
/**
* Stored per-key type: one observed value type, `"mixed"` once more than one
* distinct type has been seen, or `""` while only null has been observed
* (null registers the key but does not count as a type).
*/
export type StoredPathType = MetadataLeafType | "mixed" | "";
/** Persisted per-key record: display type + a few observed scalar values. */
export type StoredKeyInfo = {
type: StoredPathType;
/**
* First-observed distinct SCALAR values (string/number/boolean, stringified)
* feeding the value-stage dropdown. Object/array/null values and values
* longer than MAX_VALUE_LENGTH are never collected β their stored form
* could not round-trip into a matching `=` filter. Absent when none.
*/
values?: string[];
};
// Bounds β metadata is user-shaped and unbounded, so every axis is capped:
// sampled rows per fetch, key string length, keys kept per project, values
// kept per key, and values kept per project in total (the store enforces the
// per-project caps on merge; entries beyond a cap are dropped, first-observed
// wins).
export const METADATA_SAMPLE_ROWS = 30;
export const MAX_PATHS_PER_PROJECT = 200;
export const MAX_VALUES_PER_KEY = 5;
// Hard per-project total. With today's caps the effective bound is the
// product 200 Γ 5 = 1000; this backstop binds only against drifted persisted
// state (older schema, tampered localStorage) or a future per-key cap raise.
export const MAX_VALUES_PER_PROJECT = 1024;
const MAX_PATH_LENGTH = 100;
// Values longer than this are SKIPPED, never truncated β a truncated value
// would insert a filter that confidently matches nothing.
const MAX_VALUE_LENGTH = 60;
// Skip parsing giant metadata blobs so the per-fetch analysis stays cheap.
const MAX_METADATA_JSON_LENGTH = 200_000;
function safeJsonParse(s: string): unknown {
try {
return JSON.parse(s);
} catch {
return undefined;
}
}
function valueType(v: unknown): MetadataLeafType | null {
if (Array.isArray(v)) return "array";
const t = typeof v;
if (t === "string" || t === "number" || t === "boolean") return t;
if (t === "object" && v !== null) return "object";
return null;
}
/**
* Merge an already-stored key type with a newly observed one. `"mixed"` is
* absorbing; null-only observations (`""`) never demote a known type.
*/
export function mergePathType(
prev: StoredPathType | undefined,
next: StoredPathType | null,
): StoredPathType {
const observed = next ?? "";
if (prev === undefined || prev === "") return observed;
if (observed === "" || observed === prev) return prev;
return "mixed";
}
/** Suggestible scalar value, stringified β or null when not suggestible. */
function scalarValue(v: unknown): string | null {
const t = typeof v;
if (t !== "string" && t !== "number" && t !== "boolean") return null;
const s = String(v);
return s.length > 0 && s.length <= MAX_VALUE_LENGTH ? s : null;
}
/**
* Record the observed top-level metadata keys of sampled rows (JSON-encoded
* strings per `MetadataDomainClient`, or already-parsed objects) with their
* value types, merged across rows (`a:1` + `a:"x"` β `"mixed"`), plus the
* first few distinct scalar values per key for the value-stage dropdown.
*/
export function collectMetadataPathTypes(
sampleMetadata: readonly unknown[],
): Map<string, StoredKeyInfo> {
const out = new Map<string, StoredKeyInfo>();
for (const md of sampleMetadata) {
if (typeof md === "string" && md.length > MAX_METADATA_JSON_LENGTH)
continue;
const parsed = typeof md === "string" ? safeJsonParse(md) : md;
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
continue;
for (const [key, v] of Object.entries(parsed as Record<string, unknown>)) {
// Empty keys are unfilterable; "__proto__" keys would turn the store's
// plain-object key map into a prototype write.
if (key.length === 0 || key === "__proto__") continue;
if (key.length > MAX_PATH_LENGTH) continue;
const existing = out.get(key);
if (existing === undefined && out.size >= MAX_PATHS_PER_PROJECT) continue;
const type = mergePathType(existing?.type, valueType(v));
const value = scalarValue(v);
let values = existing?.values;
if (
value !== null &&
(values?.length ?? 0) < MAX_VALUES_PER_KEY &&
!values?.includes(value)
) {
values = [...(values ?? []), value];
}
out.set(key, values === undefined ? { type } : { type, values });
}
}
return out;
}
/** One offered metadata key or value; `type` is the display-only hint. */
export type MetadataSuggestion = { value: string; type?: MetadataLeafType };
/**
* Project the persisted per-project key map into option-map entries, keyed the
* way both suggestion surfaces look them up: observed keys under `metadata`,
* each key's observed values under `metadata.<key>`. Keys are sorted so
* related dotted keys group together; `"mixed"`/null-only keys carry no type
* hint. Empty when nothing has been observed, so callers can skip the merge.
*/
export function observedMetadataOptions(
paths: Record<string, StoredKeyInfo> | undefined,
): Record<string, MetadataSuggestion[]> {
const entries = Object.entries(paths ?? {});
if (entries.length === 0) return {};
const out: Record<string, MetadataSuggestion[]> = {
metadata: [...entries]
.sort(([a], [b]) => a.localeCompare(b))
.map(([path, { type }]) =>
type === "" || type === "mixed"
? { value: path }
: { value: path, type },
),
};
for (const [path, { values }] of entries) {
if (values !== undefined && values.length > 0) {
out[`metadata.${path}`] = values.map((v) => ({ value: v }));
}
}
return out;
}