🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(seeder): add a custom-models scenario (#16038)
Nikita Kabardin14b61fbd35df
diff --git a/.agents/skills/seed-test-data/SKILL.md b/.agents/skills/seed-test-data/SKILL.mdindex ea9ce4ba2e..8633ce6d26 100644--- a/.agents/skills/seed-test-data/SKILL.md+++ b/.agents/skills/seed-test-data/SKILL.md@@ -41,6 +41,7 @@ this.| Many traces for list/filter performance | `pnpm run seed -- many-traces --count 100000 --days 14` || Long-window v4 traffic with cost/latency/token OUTLIERS (outlier chart strip, LFE-14451) | `pnpm run seed -- outlier-traffic --days 90` (diurnal base load + deterministic spikes + hour-long ×8-latency incidents; root AGENT + GENERATION carrying cost + TOOL per trace; v4 on by default) || Scores with spaces in the name (filter/grammar testing) | `pnpm run seed -- scored-traces --traces 24 --v4` |+| Custom model definitions reachable from a trace (price editor entry points) | `pnpm run seed -- custom-models --v4` (a tiered model with a condition-gated second tier and one usage type priced at 0, a single-tier model, and a generation whose model matches no definition so its badge opens the create dialog) || Lots of scores on every node (dense score badges, tree-row overflow testing) | add `--scores-per-node 12` to `trace-tree` (N distinct scores per observation; try `--depth 2 --breadth 44` for many tall sibling rows) || Extra trace tags, incl. mixed case/accents (tag filter ordering) | add `--tags "Zebra,apple,Ärger"` to `trace-tree` (comma-separated, appended to the scenario's own tags) || Varied human-annotation queues (annotate UI / keyboard testing) | `pnpm run seed -- annotation-queue --core-items 12` (creates a "core types" queue covering every score-field render path + an "edge cases" queue with archived/stale/partial scores and observation/session/deleted/completed items) |diff --git a/packages/shared/scripts/seeder/AGENTS.md b/packages/shared/scripts/seeder/AGENTS.mdindex 32999ab1fb..6a5ee210b2 100644--- a/packages/shared/scripts/seeder/AGENTS.md+++ b/packages/shared/scripts/seeder/AGENTS.md@@ -20,6 +20,7 @@ pnpm run seed -- session-shapes --shape media # messages carrying @@@langfupnpm run seed -- many-traces --count 100000 --days 14pnpm run seed -- outlier-traffic --days 90 # diurnal v4 traffic w/ cost/latency/token outliers (outlier chart strip)pnpm run seed -- scored-traces --traces 24 --v4 # scores w/ spaces in the name+pnpm run seed -- custom-models --v4 # project-level model definitions (tiered + single-tier, one price at 0) + a trace whose generations link to them, plus one unpriced model```The last stdout line of a run is a JSON summary with `traceIds`,diff --git a/packages/shared/scripts/seeder/scenarios/custom-models.ts b/packages/shared/scripts/seeder/scenarios/custom-models.tsnew file mode 100644index 0000000000..2771f75b8a--- /dev/null+++ b/packages/shared/scripts/seeder/scenarios/custom-models.ts@@ -0,0 +1,469 @@+import { prisma } from "../../../src/db";+import {+ createObservation,+ createObservationsCh,+ createTrace,+ createTracesCh,+ createEventsCh,+ EventRecordInsertType,+ ObservationRecordInsertType,+ TraceRecordInsertType,+} from "../../../src/server";+import { observationToEvent, traceToEvent } from "./event-mirror";+import { jitter, utcDayStartMs } from "./rng";+import {+ chunk,+ ScenarioContext,+ ScenarioDefinition,+ SeedError,+ SeedSummary,+} from "./types";+import { countRows, traceLink } from "./verify";++// ---------------------------------------------------------------------------+// Project-level model definitions plus generations that reference them, so the+// model editor can be reached the way a user reaches it: from a generation in+// a trace. Three entry points are covered.+//+// - LINKED, tiered: a realtime-style model priced per usage type with a+// second, condition-gated tier. The generation's model badge links to the+// model page, where Edit opens the tiered (accordion) editor. One usage+// type is priced at exactly 0 — that price must survive a round trip.+// - LINKED, single tier: an embedding model, one usage type. Edit opens the+// simple (single-tier) editor.+// - UNLINKED: a generation whose model name matches no definition. Its badge+// opens the CREATE dialog prefilled from the generation's usage details,+// which is the path where submitting navigates away to the new model.+// ---------------------------------------------------------------------------++type TierDef = {+ name: string;+ isDefault: boolean;+ priority: number;+ conditions: {+ usageDetailPattern: string;+ operator: "gt" | "gte" | "lt" | "lte" | "eq" | "neq";+ value: number;+ caseSensitive: boolean;+ }[];+ prices: Record<string, number>;+};++type ModelDef = {+ key: string;+ modelName: string;+ tokenizerId: string | null;+ tiers: TierDef[];+ /** usage_details written on the generations that link to this model */+ usage: Record<string, number>;+};++const REALTIME_STANDARD_PRICES: Record<string, number> = {+ text_input: 0.000004,+ text_output: 0.000016,+ audio_input: 0.00004,+ audio_output: 0.00008,+ // Deliberately free: a zero price is a real price and must round-trip.+ text_input_cached: 0,+};++const MODELS: ModelDef[] = [+ {+ key: "realtime",+ modelName: "seed-realtime-2.1",+ tokenizerId: null,+ tiers: [+ {+ name: "Standard",+ isDefault: true,+ priority: 0,+ conditions: [],+ prices: REALTIME_STANDARD_PRICES,+ },+ {+ name: "Long context",+ isDefault: false,+ priority: 1,+ conditions: [+ {+ usageDetailPattern: "^text_input",+ operator: "gt",+ value: 128000,+ caseSensitive: false,+ },+ ],+ // Same usage types at double the price: renaming a usage type in the+ // default tier must not zero these out.+ prices: Object.fromEntries(+ Object.entries(REALTIME_STANDARD_PRICES).map(([type, price]) => [+ type,+ price * 2,+ ]),+ ),+ },+ ],+ usage: {+ text_input: 1840,+ text_output: 320,+ audio_input: 4400,+ audio_output: 1200,+ text_input_cached: 900,+ total: 8660,+ },+ },+ {+ key: "embed",+ modelName: "seed-embed-3-small",+ tokenizerId: null,+ tiers: [+ {+ name: "Standard",+ isDefault: true,+ priority: 0,+ conditions: [],+ prices: { input: 0.00000002 },+ },+ ],+ usage: { input: 512, total: 512 },+ },+];++/** Matches no definition, so its badge opens the create dialog. */+const UNPRICED_MODEL_NAME = "seed-unpriced-vision-1";+const UNPRICED_USAGE: Record<string, number> = {+ input: 900,+ image_input: 3,+ output: 210,+ total: 1113,+};++/**+ * A model's identity in Postgres is `(projectId, modelName, startDate, unit)`,+ * not an id we derive — so the derived id carries the project rather than+ * `--id-prefix`. With the prefix in it, seeding a second project would have+ * reassigned the first project's model row (orphaning its generations), and a+ * second prefix in one project would have hit the unique constraint instead of+ * resetting. Traces and observations still key off the prefix.+ */+const derivedModelId = (ctx: ScenarioContext, key: string) =>+ `model-${ctx.projectId}-${key}`;++const costFor = (+ usage: Record<string, number>,+ prices: Record<string, number>,+): Record<string, number> => {+ const costs = Object.fromEntries(+ Object.entries(usage)+ .filter(([type]) => type !== "total" && prices[type] !== undefined)+ .map(([type, units]) => [type, units * prices[type]]),+ );+ return {+ ...costs,+ total: Object.values(costs).reduce((sum, cost) => sum + cost, 0),+ };+};++/**+ * Re-seeding resets a model: tiers (and the prices that cascade from them) are+ * replaced wholesale, in one transaction so an interrupted run cannot leave a+ * model priced by nothing. Returns the id the row actually has.+ */+const writeModel = async (+ ctx: ScenarioContext,+ model: ModelDef,+): Promise<string> => {+ const modelRow = {+ projectId: ctx.projectId,+ modelName: model.modelName,+ matchPattern: `(?i)^(${model.modelName})$`,+ tokenizerId: model.tokenizerId,+ // The UI writes both so the (projectId, modelName, startDate, unit)+ // uniqueness constraint bites; match it or a UI save creates a twin.+ startDate: new Date("2010-01-01"),+ unit: "TOKENS",+ };+ // Whatever row already owns this name in this project wins, so a re-run+ // resets it instead of colliding with the constraint.+ const existing = await prisma.model.findFirst({+ where: { projectId: ctx.projectId, modelName: model.modelName },+ select: { id: true },+ });+ const id = existing?.id ?? derivedModelId(ctx, model.key);++ return prisma.$transaction(async (tx) => {+ await tx.model.upsert({+ where: { id },+ create: { id, ...modelRow },+ update: modelRow,+ });+ // Prices cascade from their tier, so deleting tiers clears both.+ await tx.pricingTier.deleteMany({ where: { modelId: id } });++ for (const tier of model.tiers) {+ const tierId = `tier-${id}-${tier.priority}`;+ await tx.pricingTier.create({+ data: {+ id: tierId,+ modelId: id,+ name: tier.name,+ isDefault: tier.isDefault,+ priority: tier.priority,+ conditions: tier.conditions,+ },+ });+ await tx.price.createMany({+ data: Object.entries(tier.prices).map(([usageType, price]) => ({+ id: `price-${tierId}-${usageType}`,+ modelId: id,+ projectId: ctx.projectId,+ pricingTierId: tierId,+ usageType,+ price,+ })),+ });+ }+ return id;+ });+};++const run = async (+ ctx: ScenarioContext,+ params: Record<string, string | number | boolean>,+): Promise<SeedSummary> => {+ const startedAt = Date.now();+ const withV4 = params.v4 === true;+ const traceId = `${ctx.idPrefix}-t0`;+ // Anchored on today's UTC midnight, never the wall clock: these land in+ // ClickHouse ORDER BY keys and a re-run must overwrite in place.+ const traceTimestamp = utcDayStartMs() - 30 * 60 * 1000;++ const generationPlan = [+ ...MODELS.flatMap((model, index) => [+ { model, suffix: `${index}a` },+ { model, suffix: `${index}b` },+ ]),+ { model: null, suffix: "unpriced" },+ ];++ const modelsSettingsLink = `${ctx.baseUrl}/project/${ctx.projectId}/settings/models`;+ const linksFor = (ids: string[]) => [+ traceLink(ctx, traceId, traceTimestamp),+ ...ids.map(+ (id) => `${ctx.baseUrl}/project/${ctx.projectId}/settings/models/${id}`,+ ),+ modelsSettingsLink,+ ];++ if (ctx.dryRun) {+ return {+ scenario: "custom-models",+ target: "clickhouse",+ params,+ projectId: ctx.projectId,+ environment: ctx.environment,+ traceIds: [traceId],+ sessionIds: [],+ counts: {+ models: MODELS.length,+ pricingTiers: MODELS.reduce((sum, m) => sum + m.tiers.length, 0),+ prices: MODELS.reduce(+ (sum, m) =>+ sum + m.tiers.reduce((n, t) => n + Object.keys(t.prices).length, 0),+ 0,+ ),+ traces: 1,+ observations: generationPlan.length,+ events: withV4 ? generationPlan.length + 1 : 0,+ },+ verified: {},+ links: linksFor(MODELS.map((model) => derivedModelId(ctx, model.key))),+ dryRun: true,+ durationMs: Date.now() - startedAt,+ };+ }++ ctx.log(`writing ${MODELS.length} model definitions to postgres`);+ const writtenModelIds = new Map<string, string>();+ for (const model of MODELS) {+ writtenModelIds.set(model.key, await writeModel(ctx, model));+ }++ const trace: TraceRecordInsertType = createTrace({+ id: traceId,+ project_id: ctx.projectId,+ environment: ctx.environment,+ session_id: null,+ timestamp: traceTimestamp,+ name: "realtime-voice-agent",+ user_id: `user-${ctx.idPrefix}`,+ tags: ["seed", "custom-models"],+ public: false,+ bookmarked: false,+ metadata: { scenario: "custom-models" },+ input: JSON.stringify({ request: "Summarise the call and embed it." }),+ output: "Summary stored.",+ created_at: Date.now(),+ updated_at: Date.now(),+ event_ts: Date.now(),+ });++ const observations: ObservationRecordInsertType[] = generationPlan.map(+ ({ model, suffix }, index) => {+ const startTime =+ traceTimestamp + index * 1200 + jitter(ctx.seed, index, 200);+ const usage = model ? model.usage : UNPRICED_USAGE;+ const defaultPrices = model?.tiers.find((tier) => tier.isDefault)?.prices;+ const costs = defaultPrices ? costFor(usage, defaultPrices) : null;++ return createObservation({+ id: `${traceId}-o${suffix}`,+ trace_id: traceId,+ project_id: ctx.projectId,+ environment: ctx.environment,+ type: "GENERATION",+ parent_observation_id: null,+ name: model ? `call-${model.key}` : "call-unpriced-vision",+ start_time: startTime,+ end_time: startTime + 900,+ completion_start_time: startTime + 180,+ level: "DEFAULT",+ status_message: null,+ input: JSON.stringify({ prompt: "Summarise the call." }),+ output: "The caller asked about pricing.",+ provided_model_name: model ? model.modelName : UNPRICED_MODEL_NAME,+ internal_model_id: model+ ? (writtenModelIds.get(model.key) ?? null)+ : null,+ provided_usage_details: usage,+ usage_details: usage,+ provided_cost_details: {},+ cost_details: costs ?? {},+ total_cost: costs?.total ?? null,+ model_parameters: JSON.stringify({ temperature: 0.2 }),+ prompt_id: null,+ prompt_name: null,+ prompt_version: null,+ created_at: Date.now(),+ updated_at: Date.now(),+ event_ts: Date.now(),+ });+ },+ );++ const events: EventRecordInsertType[] = withV4+ ? [+ traceToEvent(trace),+ ...observations.map((o) => observationToEvent(o, trace)),+ ]+ : [];++ ctx.log(+ `writing 1 trace, ${observations.length} generations${withV4 ? `, ${events.length} events` : ""}`,+ );+ await createTracesCh([trace]);+ for (const batch of chunk(observations, 1000)) {+ await createObservationsCh(batch);+ }+ for (const batch of chunk(events, 500)) {+ await createEventsCh(batch);+ }++ const modelIds = [...writtenModelIds.values()];+ const verified: Record<string, number> = {+ models: await prisma.model.count({ where: { id: { in: modelIds } } }),+ pricingTiers: await prisma.pricingTier.count({+ where: { modelId: { in: modelIds } },+ }),+ prices: await prisma.price.count({ where: { modelId: { in: modelIds } } }),+ traces: await countRows(+ "traces",+ `project_id = {projectId: String} AND id = {traceId: String}`,+ { projectId: ctx.projectId, traceId },+ "uniqExact(id)",+ ),+ observations: await countRows(+ "observations",+ `project_id = {projectId: String} AND trace_id = {traceId: String}`,+ { projectId: ctx.projectId, traceId },+ "uniqExact(id)",+ ),+ };+ if (withV4) {+ verified.events = await countRows(+ "events_full",+ `project_id = {projectId: String} AND trace_id = {traceId: String}`,+ { projectId: ctx.projectId, traceId },+ "uniqExact(span_id)",+ );+ }++ const expectedPrices = MODELS.reduce(+ (sum, model) =>+ sum ++ model.tiers.reduce((n, tier) => n + Object.keys(tier.prices).length, 0),+ 0,+ );+ const expectedTiers = MODELS.reduce((sum, m) => sum + m.tiers.length, 0);+ if (verified.models < MODELS.length || verified.prices < expectedPrices) {+ throw new SeedError(+ `Readback mismatch: expected ${MODELS.length} models and ${expectedPrices} prices, found ${verified.models} and ${verified.prices}`,+ );+ }+ if (verified.pricingTiers < expectedTiers) {+ throw new SeedError(+ `Readback mismatch: expected ${expectedTiers} pricing tiers, found ${verified.pricingTiers}`,+ );+ }+ if (verified.traces < 1) {+ throw new SeedError("Readback mismatch: the trace row did not land");+ }+ if (verified.observations < observations.length) {+ throw new SeedError(+ `Readback mismatch: expected ${observations.length} observations, found ${verified.observations}`,+ );+ }+ if (withV4 && verified.events < events.length) {+ throw new SeedError(+ `Readback mismatch: expected ${events.length} events_full rows, found ${verified.events}`,+ );+ }++ return {+ scenario: "custom-models",+ target: "clickhouse",+ params,+ projectId: ctx.projectId,+ environment: ctx.environment,+ traceIds: [traceId],+ sessionIds: [],+ counts: {+ models: MODELS.length,+ pricingTiers: MODELS.reduce((sum, m) => sum + m.tiers.length, 0),+ prices: expectedPrices,+ traces: 1,+ observations: observations.length,+ events: events.length,+ },+ verified,+ links: linksFor(modelIds),+ dryRun: false,+ durationMs: Date.now() - startedAt,+ };+};++export const customModelsScenario: ScenarioDefinition = {+ name: "custom-models",+ description:+ "Project-level model definitions (one tiered with a condition-gated second tier and a usage type priced at 0, one single-tier) plus a trace whose generations link to them, and one generation whose model matches no definition so its badge opens the create dialog. Reaches the price editor from a trace, the way users do.",+ supportsV4: true,+ flags: [+ {+ flag: "v4",+ type: "boolean",+ default: false,+ description:+ "also mirror the trace/generations into v4 events_full/events_core",+ },+ ],+ run,+};diff --git a/packages/shared/scripts/seeder/scenarios/index.ts b/packages/shared/scripts/seeder/scenarios/index.tsindex 3f082ad820..31be24f77b 100644--- a/packages/shared/scripts/seeder/scenarios/index.ts+++ b/packages/shared/scripts/seeder/scenarios/index.ts@@ -1,5 +1,6 @@import { agentTimelineScenario } from "./agent-timeline";import { annotationQueueScenario } from "./annotation-queue";+import { customModelsScenario } from "./custom-models";import { deepChainScenario } from "./deep-chain";import { longSessionScenario } from "./long-session";import { manyTracesScenario } from "./many-traces";@@ -23,6 +24,7 @@ export const scenarios: Record<string, ScenarioDefinition> = {"scored-traces": scoredTracesScenario,"session-shapes": sessionShapesScenario,"annotation-queue": annotationQueueScenario,+ "custom-models": customModelsScenario,"support-agent": supportAgentScenario,};