Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

47.8 KB
Raw Blame
/* eslint-disable no-nested-ternary */
import { FileContent, DatasetItemInput } from "./types";
import {
  REALISTIC_TRACE_NAMES,
  REALISTIC_SPAN_NAMES,
  REALISTIC_GENERATION_NAMES,
  REALISTIC_MODELS,
  REALISTIC_AGENT_NAMES,
  REALISTIC_TOOL_NAMES,
  REALISTIC_CHAIN_NAMES,
  REALISTIC_RETRIEVER_NAMES,
  REALISTIC_EVALUATOR_NAMES,
  REALISTIC_EMBEDDING_NAMES,
  REALISTIC_GUARDRAIL_NAMES,
} from "./clickhouse-seed-constants";
import {
  generateDatasetItemId,
  generateDatasetRunItemId,
  generateDatasetRunTraceId,
  generateEvalObservationId,
  generateEvalScoreId,
  generateEvalTraceId,
} from "./seed-helpers";
import { v4 as uuidv4 } from "uuid";
import {
  FAILED_EVAL_TRACE_INTERVAL,
  SEED_EVALUATOR_CONFIGS,
} from "./postgres-seed-constants";
import {
  createTrace,
  createObservation,
  createTraceScore,
  ObservationRecordInsertType,
  ScoreRecordInsertType,
  TraceRecordInsertType,
  DatasetRunItemRecordInsertType,
  createDatasetRunItem,
  createDatasetRunScore,
  UNKNOWN_INGESTION_SDK_VALUE,
  toClickhouseDateTime,
} from "../../../src/server";

/**
 * Generates realistic test data for traces, observations, and scores.
 *
 * Use generateXxxTraces() for creating different data types:
 * - generateDatasetTrace(): For dataset experiment runs (langfuse-prompt-experiment env)
 * - generateEvaluationTraces(): For evaluation data (langfuse-evaluation env)
 * - generateSyntheticTraces(): For large-scale synthetic data (default env)
 */
export class DataGenerator {
  private static instance: DataGenerator;
  private fileContent: FileContent | null = null;

  static getInstance(): DataGenerator {
    if (!DataGenerator.instance) {
      DataGenerator.instance = new DataGenerator();
    }
    return DataGenerator.instance;
  }

  setFileContent(content: FileContent) {
    this.fileContent = content;
  }

  private randomElement<T>(array: T[]): T {
    return array[Math.floor(Math.random() * array.length)];
  }

  private randomBoolean(probability = 0.5): boolean {
    return Math.random() < probability;
  }

  private randomInt(min: number, max: number): number {
    return Math.floor(Math.random() * (max - min + 1)) + min;
  }

  private buildNestedSeedMetadata(
    source: string,
    index: number,
    overrides: Record<string, string> = {},
  ): Record<string, string> {
    const plans = ["free", "pro", "enterprise"];
    const regions = ["eu-central-1", "us-east-1", "ap-south-1"];
    const queues = ["support-chat", "sales-chat", "ops-chat"];
    const priorities = ["low", "normal", "high"];

    return {
      source,
      "customer.id": `customer_${index % 100}`,
      "customer.plan": plans[index % plans.length],
      "customer.region.code": regions[index % regions.length],
      "routing.queue": queues[index % queues.length],
      "routing.priority": priorities[index % priorities.length],
      "flags.beta": index % 2 === 0 ? "true" : "false",
      ...overrides,
    };
  }

  private buildSeedIngestionAttribution(
    projectId: string,
    client: "python-sdk" | "javascript-sdk" | "raw-api",
  ): Pick<
    ScoreRecordInsertType,
    "ingestion_api_key" | "ingestion_sdk_name" | "ingestion_sdk_version"
  > {
    const keySuffix = projectId.slice(-8);

    switch (client) {
      case "python-sdk":
        return {
          ingestion_api_key: `pk-lf-seed-${keySuffix}-python`,
          ingestion_sdk_name: "python",
          ingestion_sdk_version: "4.2.1",
        };
      case "javascript-sdk":
        return {
          ingestion_api_key: `pk-lf-seed-${keySuffix}-javascript`,
          ingestion_sdk_name: "javascript",
          ingestion_sdk_version: "5.1.3",
        };
      case "raw-api":
        return {
          ingestion_api_key: `pk-lf-seed-${keySuffix}-raw-api`,
          ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,
          ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,
        };
    }
  }

  /**
   * Creates dataset run items for dataset runs.
   * Use for: Dataset experiment scenarios.
   */
  generateDatasetRunItem(
    input: DatasetItemInput & { runCreatedAt: number },
    projectId: string,
  ): DatasetRunItemRecordInsertType {
    const datasetRunItemId = generateDatasetRunItemId(
      input.datasetName,
      input.itemIndex,
      projectId,
      input.runNumber || 0,
    );

    // Add small random offset (1-10 seconds) for realistic variation
    const itemCreatedAt = input.runCreatedAt + this.randomInt(1, 10) * 1000;

    return createDatasetRunItem({
      id: datasetRunItemId,
      project_id: projectId,
      trace_id: generateDatasetRunTraceId(
        input.datasetName,
        input.itemIndex,
        projectId,
        input.runNumber || 0,
      ),
      dataset_id: `${input.datasetName}-${projectId.slice(-8)}`,
      dataset_run_id: `demo-dataset-run-${input.runNumber}-${input.datasetName}-${projectId.slice(-8)}`,
      dataset_run_name: `demo-dataset-run-${input.runNumber}-${input.datasetName}`,
      dataset_run_created_at: input.runCreatedAt,
      dataset_run_description:
        (input.runNumber || 0) % 2 === 0 ? "Dataset run description" : "",
      dataset_run_metadata: { key: "value" },
      dataset_item_id: generateDatasetItemId(
        input.datasetName,
        input.itemIndex,
        projectId,
      ),
      dataset_item_input: input.item.input,
      dataset_item_expected_output: input.item.expectedOutput,
      created_at: itemCreatedAt,
      updated_at: itemCreatedAt,
      event_ts: itemCreatedAt,
    });
  }

  /**
   * Creates traces from dataset items for experiment runs.
   * Use for: Dataset experiments scenarios.
   */
  generateDatasetTrace(
    input: DatasetItemInput,
    projectId: string,
  ): TraceRecordInsertType {
    const traceId = generateDatasetRunTraceId(
      input.datasetName,
      input.itemIndex,
      projectId,
      input.runNumber || 0,
    );

    let traceInput: string;
    let traceOutput: string;

    // Transform dataset item based on type
    if (input.datasetName === "demo-countries-dataset") {
      const data = input.item as { input: { country: string }; output: string };
      traceInput = `What is the capital of ${data.input.country}?`;
      traceOutput = `The capital of ${data.input.country} is ${data.output}.`;
    } else if (input.datasetName === "demo-english-transcription-dataset") {
      const data = input.item as { input: { word: string }; output: string };
      traceInput = `What is the IPA transcription of the word "${data.input.word}"?`;
      traceOutput = `The IPA transcription of "${data.input.word}" is ${data.output}.`;
    } else {
      traceInput = JSON.stringify(input.item.input);
      traceOutput = JSON.stringify(input.item.output);
    }

    return createTrace({
      id: traceId,
      project_id: projectId,
      name: `dataset-run-item-${uuidv4()}`,
      input: traceInput,
      output: traceOutput,
      environment: "langfuse-prompt-experiment",
      metadata: { experimentType: "langfuse-prompt-experiment" },
      public: false,
      bookmarked: false,
      session_id: null,
      tags: [],
    });
  }

  /**
   * Creates observations for dataset experiment traces with variable costs/latency.
   * Use for: Dataset experiments requiring detailed observation tracking.
   */
  generateDatasetObservation(
    trace: TraceRecordInsertType,
    input: DatasetItemInput,
    projectId: string,
  ): ObservationRecordInsertType {
    const observationId = `observation-dataset-${input.datasetName}-${input.itemIndex}-${input.runNumber}-${projectId.slice(-8)}`;

    // Generate variable usage and cost for each observation
    const inputTokens = this.randomInt(30, 150);
    const outputTokens = this.randomInt(10, 80);
    const totalTokens = inputTokens + outputTokens;

    // Cost should be fraction of cents (0.0001-0.01 range)
    const inputCost = (inputTokens * this.randomInt(1, 5)) / 1000000; // $0.000001-0.000005 per token
    const outputCost = (outputTokens * this.randomInt(2, 10)) / 1000000; // $0.000002-0.00001 per token
    const totalCost = inputCost + outputCost;

    return createObservation({
      id: observationId,
      trace_id: trace.id,
      project_id: projectId,
      type: "GENERATION",
      name: `dataset-generation-${input.itemIndex}-run-${input.runNumber}`,
      input: trace.input,
      output: trace.output,
      provided_model_name: "gpt-5.4-mini",
      model_parameters: JSON.stringify({ temperature: 0.7 }),
      usage_details: {
        input: inputTokens,
        output: outputTokens,
        total: totalTokens,
      },
      provided_usage_details: {
        input: inputTokens,
        output: outputTokens,
        total: totalTokens,
      },
      cost_details: {
        input: Math.round(inputCost * 100000) / 100000, // Round to 5 decimal places
        output: Math.round(outputCost * 100000) / 100000,
        total: Math.round(totalCost * 100000) / 100000,
      },
      provided_cost_details: {
        input: Math.round(inputCost * 100000) / 100000,
        output: Math.round(outputCost * 100000) / 100000,
        total: Math.round(totalCost * 100000) / 100000,
      },
      total_cost: Math.round(totalCost * 100000) / 100000,
      environment: "langfuse-prompt-experiment",
    });
  }

  /**
   * Creates scores for dataset experiment scores with variable values.
   * Use for: Dataset experiments requiring detailed score tracking.
   */
  generateDatasetScore(
    trace: TraceRecordInsertType,
    input: DatasetItemInput,
    projectId: string,
    scoreNames: string[],
  ): ScoreRecordInsertType {
    const scoreId = `score-dataset-${input.datasetName}-${input.itemIndex}-${input.runNumber}-${projectId.slice(-8)}`;

    return createTraceScore({
      id: scoreId,
      trace_id: trace.id,
      project_id: projectId,
      name: this.randomElement(scoreNames),
      value: Math.random() * 100,
      string_value: undefined,
      data_type: "NUMERIC",
      source: "API",
      environment: "langfuse-prompt-experiment",
      ...this.buildSeedIngestionAttribution(projectId, "python-sdk"),
    });
  }

  /**
   * Creates scores for dataset experiment traces with variable values.
   * Use for: Dataset experiments requiring detailed score tracking.
   */
  generateDatasetRunScore(
    runId: string,
    input: {
      datasetName: string;
      runNumber: number;
    },
    projectId: string,
    scoreNames: string[],
  ): ScoreRecordInsertType {
    const scoreId = `score-dataset-run-${input.datasetName}-${input.runNumber}-${projectId.slice(-8)}`;

    return createDatasetRunScore({
      id: scoreId,
      project_id: projectId,
      dataset_run_id: runId,
      name: this.randomElement(scoreNames),
      value: Math.random() * 100,
      string_value: undefined,
      data_type: "NUMERIC",
      source: "API",
      environment: "langfuse-prompt-experiment",
      ...this.buildSeedIngestionAttribution(projectId, "javascript-sdk"),
    });
  }

  /**
   * Creates large-scale synthetic traces for performance testing.
   * Use for: Load testing, dashboard demos, realistic usage simulation.
   */
  generateSyntheticTraces(
    projectId: string,
    count: number,
  ): TraceRecordInsertType[] {
    const traces: TraceRecordInsertType[] = [];

    for (let i = 0; i < count; i++) {
      const trace = createTrace({
        id: `trace-synthetic-${i}-${projectId.slice(-8)}`,
        project_id: projectId,
        name: this.randomElement(REALISTIC_TRACE_NAMES),
        input: this.generateTraceInput(),
        output: this.generateTraceOutput(),
        user_id: this.randomBoolean(0.3)
          ? `user_${this.randomInt(1, 1000)}`
          : null,
        session_id: this.randomBoolean(0.3)
          ? `session_${this.randomInt(1, 100)}`
          : undefined,
        environment: "default",
        metadata: this.buildNestedSeedMetadata("synthetic", i, {
          generated: "synthetic",
        }),
        tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
        public: this.randomBoolean(0.8),
        bookmarked: this.randomBoolean(0.1),
        release: this.randomBoolean(0.4)
          ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
          : undefined,
        version: this.randomBoolean(0.4)
          ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
          : undefined,
      });

      traces.push(trace);
    }

    return traces;
  }

  generateEvaluationObservations(
    traces: TraceRecordInsertType[],
    observationsPerTrace = 5,
    projectId: string,
  ): ObservationRecordInsertType[] {
    const observations: ObservationRecordInsertType[] = [];

    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
      traces.forEach((trace, traceIndex) => {
        for (let i = 0; i < observationsPerTrace; i++) {
          const obsType = (() => {
            if (this.randomBoolean(0.47)) {
              return "GENERATION";
            }
            if (this.randomBoolean(0.94)) {
              return "SPAN";
            }
            return "EVENT";
          })();

          const observation: ObservationRecordInsertType = createObservation({
            id: generateEvalObservationId(
              evalJobConfiguration.evalTemplateId,
              traceIndex,
              projectId,
              i,
            ),
            trace_id: trace.id,
            project_id: projectId,
            parent_observation_id: undefined,
            type: obsType,
            name:
              obsType === "GENERATION"
                ? this.randomElement(REALISTIC_GENERATION_NAMES)
                : obsType === "SPAN"
                  ? this.randomElement(REALISTIC_SPAN_NAMES)
                  : `event_${i % 10}`,
            level: this.randomBoolean(0.85)
              ? "DEFAULT"
              : this.randomBoolean(0.7)
                ? "DEBUG"
                : this.randomBoolean(0.3)
                  ? "ERROR"
                  : "WARNING",
            input:
              obsType === "GENERATION"
                ? this.randomBoolean(0.4)
                  ? this.fileContent?.heavyMarkdown || "Sample input"
                  : JSON.stringify(this.fileContent?.chatMlJson || {})
                : undefined,
            output:
              obsType === "GENERATION"
                ? this.randomBoolean(0.3)
                  ? JSON.stringify(this.fileContent?.nestedJson || {})
                  : JSON.stringify(this.fileContent?.chatMlJson || {})
                : undefined,
            provided_model_name:
              obsType === "GENERATION"
                ? this.randomElement(REALISTIC_MODELS)
                : undefined,
            model_parameters:
              obsType === "GENERATION"
                ? JSON.stringify({ temperature: 0.7 })
                : undefined,
            usage_details:
              obsType === "GENERATION"
                ? {
                    input: this.randomInt(20, 200),
                    output: this.randomInt(10, 100),
                    total: this.randomInt(30, 300),
                  }
                : undefined,
            provided_usage_details:
              obsType === "GENERATION"
                ? {
                    input: this.randomInt(20, 200),
                    output: this.randomInt(10, 100),
                    total: this.randomInt(30, 300),
                  }
                : undefined,
            cost_details:
              obsType === "GENERATION"
                ? {
                    input: this.randomInt(1, 10) / 100000,
                    output: this.randomInt(1, 20) / 100000,
                    total: this.randomInt(2, 30) / 100000,
                  }
                : undefined,
            provided_cost_details:
              obsType === "GENERATION"
                ? {
                    input: this.randomInt(1, 10) / 100000,
                    output: this.randomInt(1, 20) / 100000,
                    total: this.randomInt(2, 30) / 100000,
                  }
                : undefined,
            environment: "langfuse-evaluation",
          });

          observations.push(observation);
        }
      });
    }

    return observations;
  }

  /**
   * Creates synthetic observations with automatic prompt linking (5% rate).
   * Use for: Large datasets, hierarchical observation structures, cost variation.
   */
  generateSyntheticObservations(
    traces: TraceRecordInsertType[],
    observationsPerTrace = 5,
  ): ObservationRecordInsertType[] {
    const observations: ObservationRecordInsertType[] = [];

    traces.forEach((trace, traceIndex) => {
      if (this.randomBoolean(0.1)) {
        const { observations: workflowObservations } =
          this.generateComprehensiveAIWorkflowTrace(trace.id, trace.project_id);
        observations.push(...workflowObservations);
        return;
      }

      for (let i = 0; i < observationsPerTrace; i++) {
        const obsType = this.randomBoolean(0.8) // More "traditional" types, are more common in app
          ? this.randomElement(["GENERATION", "SPAN", "EVENT"])
          : this.randomElement([
              "AGENT",
              "TOOL",
              "CHAIN",
              "RETRIEVER",
              "EVALUATOR",
              "EMBEDDING",
              "GUARDRAIL",
            ]);

        let observationName: string;
        switch (obsType) {
          case "AGENT":
            observationName = this.randomElement(REALISTIC_AGENT_NAMES);
            break;
          case "TOOL":
            observationName = this.randomElement(REALISTIC_TOOL_NAMES);
            break;
          case "CHAIN":
            observationName = this.randomElement(REALISTIC_CHAIN_NAMES);
            break;
          case "RETRIEVER":
            observationName = this.randomElement(REALISTIC_RETRIEVER_NAMES);
            break;
          case "EVALUATOR":
            observationName = this.randomElement(REALISTIC_EVALUATOR_NAMES);
            break;
          case "EMBEDDING":
            observationName = this.randomElement(REALISTIC_EMBEDDING_NAMES);
            break;
          case "GUARDRAIL":
            observationName = this.randomElement(REALISTIC_GUARDRAIL_NAMES);
            break;
          case "GENERATION":
            observationName = this.randomElement(REALISTIC_GENERATION_NAMES);
            break;
          default:
            observationName = this.randomElement(REALISTIC_SPAN_NAMES);
            break;
        }

        const observation: ObservationRecordInsertType = createObservation({
          id: `obs-synthetic-${traceIndex}-${i}`,
          trace_id: trace.id,
          project_id: trace.project_id,
          parent_observation_id:
            i > 0 ? `obs-synthetic-${traceIndex}-${i - 1}` : undefined,
          type: obsType as any,
          name: observationName,
          input:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? this.generateObservationInput()
              : undefined,
          output:
            obsType === "GENERATION" ||
            obsType === "RETRIEVER" ||
            obsType === "EVALUATOR" ||
            obsType === "GUARDRAIL"
              ? this.generateObservationOutput()
              : undefined,
          provided_model_name:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? this.randomElement(REALISTIC_MODELS)
              : undefined,
          model_parameters:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? JSON.stringify({ temperature: 0.7 })
              : undefined,
          usage_details:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? {
                  input: this.randomInt(20, 200),
                  output: this.randomInt(10, 100),
                  total: this.randomInt(30, 300),
                }
              : undefined,
          provided_usage_details:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? {
                  input: this.randomInt(20, 200),
                  output: this.randomInt(10, 100),
                  total: this.randomInt(30, 300),
                }
              : undefined,
          cost_details:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? {
                  input: this.randomInt(1, 10) / 100000,
                  output: this.randomInt(1, 20) / 100000,
                  total: this.randomInt(2, 30) / 100000,
                }
              : undefined,
          provided_cost_details:
            obsType === "GENERATION" || obsType === "EMBEDDING"
              ? {
                  input: this.randomInt(1, 10) / 100000,
                  output: this.randomInt(1, 20) / 100000,
                  total: this.randomInt(2, 30) / 100000,
                }
              : undefined,
          level: this.randomBoolean(0.85)
            ? "DEFAULT"
            : this.randomBoolean(0.7)
              ? "DEBUG"
              : this.randomBoolean(0.5)
                ? "WARNING"
                : "ERROR",
          environment: trace.environment,
          metadata: this.buildNestedSeedMetadata(
            "synthetic-observation",
            traceIndex * observationsPerTrace + i,
            {
              "observation.type": obsType,
              "workflow.step": String(i + 1),
            },
          ),
        });

        observations.push(observation);
      }
    });

    return observations;
  }

  generateSyntheticScores(
    traces: TraceRecordInsertType[],
    observations: ObservationRecordInsertType[],
    scoresPerTrace = 2,
  ): ScoreRecordInsertType[] {
    const scores: ScoreRecordInsertType[] = [];

    traces.forEach((trace, traceIndex) => {
      for (let i = 0; i < scoresPerTrace; i++) {
        const scoreType = this.randomElement([
          "NUMERIC",
          "CATEGORICAL",
          "BOOLEAN",
        ]);

        let value: number | undefined;
        let stringValue: string | undefined;

        switch (scoreType) {
          case "NUMERIC":
            value = Math.random() * 100;
            break;
          case "CATEGORICAL":
            stringValue = `category_${this.randomInt(1, 5)}`;
            break;
          case "BOOLEAN":
            value = this.randomBoolean() ? 1 : 0;
            stringValue = value === 1 ? "True" : "False";
            break;
        }

        const score: ScoreRecordInsertType = createTraceScore({
          id: `score-synthetic-${traceIndex}-${i}`,
          project_id: trace.project_id,
          trace_id: trace.id,
          observation_id: this.randomBoolean(0.1)
            ? this.randomElement(
                observations.filter((o) => o.trace_id === trace.id),
              )?.id
            : undefined,
          name: `metric_${this.randomInt(1, 10)}`,
          value,
          string_value: stringValue,
          data_type: scoreType as any,
          source: "API",
          comment: "Generated score\ntest",
          environment: trace.environment,
          ...this.buildSeedIngestionAttribution(
            trace.project_id,
            this.randomElement(["python-sdk", "javascript-sdk", "raw-api"]),
          ),
        });

        scores.push(score);
      }
    });

    return scores;
  }

  /**
   * Creates a workflow trace with all possible observation types.
   */
  generateComprehensiveAIWorkflowTrace(
    traceId: string,
    projectId: string,
  ): {
    trace: TraceRecordInsertType;
    observations: ObservationRecordInsertType[];
  } {
    // Create the main trace
    const trace = createTrace({
      id: traceId,
      project_id: projectId,
      name: "AI-Agent-Workflow",
      input:
        "Analyze and summarize the latest research papers on quantum computing",
      output:
        "Here is a comprehensive summary of quantum computing research trends with key insights and recommendations.",
      user_id: this.randomBoolean(0.3)
        ? `user_${this.randomInt(1, 1000)}`
        : null,
      session_id: this.randomBoolean(0.3)
        ? `session_${this.randomInt(1, 100)}`
        : undefined,
      environment: "default",
      metadata: { workflowType: "comprehensive-ai", purpose: "demonstration" },
      tags: ["ai-agent", "multi-step", "comprehensive"],
      public: true,
      bookmarked: this.randomBoolean(0.2),
    });

    const observations: ObservationRecordInsertType[] = [];
    const baseTime = Date.now();

    // 1. AGENT - Main coordinator
    observations.push(
      createObservation({
        id: `${traceId}-agent`,
        trace_id: trace.id,
        project_id: projectId,
        type: "AGENT",
        name: this.randomElement(REALISTIC_AGENT_NAMES),
        input: "Plan and coordinate the research analysis workflow",
        output:
          "Workflow planned: retrieve documents → create embeddings → analyze → evaluate → check safety",
        start_time: baseTime,
        end_time: baseTime + 500,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: { role: "coordinator", step: "1" },
      }),
    );

    // 2. RETRIEVER - Document retrieval
    observations.push(
      createObservation({
        id: `${traceId}-retriever`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-agent`,
        type: "RETRIEVER",
        name: this.randomElement(REALISTIC_RETRIEVER_NAMES),
        input: "query: quantum computing research papers 2024",
        output: "Retrieved 15 relevant research papers from arXiv and IEEE",
        start_time: baseTime + 500,
        end_time: baseTime + 2000,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: { documentsFound: "15", sources: "arXiv,IEEE" },
      }),
    );

    // 3. EMBEDDING - Create document embeddings
    observations.push(
      createObservation({
        id: `${traceId}-embedding`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-retriever`,
        type: "EMBEDDING",
        name: this.randomElement(REALISTIC_EMBEDDING_NAMES),
        input: "15 research paper abstracts and titles",
        output: "Generated 1536-dimensional embeddings for semantic similarity",
        start_time: baseTime + 2000,
        end_time: baseTime + 3500,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: {
          embeddingModel: "text-embedding-3-small",
          dimensions: "1536",
        },
        usage_details: {
          input: this.randomInt(2000, 4000),
          total: this.randomInt(2000, 4000),
        },
        provided_usage_details: {
          input: this.randomInt(2000, 4000),
          total: this.randomInt(2000, 4000),
        },
        cost_details: {
          input: this.randomInt(5, 15) / 100000,
          total: this.randomInt(5, 15) / 100000,
        },
        provided_cost_details: {
          input: this.randomInt(5, 15) / 100000,
          total: this.randomInt(5, 15) / 100000,
        },
      }),
    );

    // 4. CHAIN - Processing pipeline
    observations.push(
      createObservation({
        id: `${traceId}-chain`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-embedding`,
        type: "CHAIN",
        name: this.randomElement(REALISTIC_CHAIN_NAMES),
        input: "Research papers with embeddings",
        output:
          "Processed and analyzed 15 papers through multi-step analysis chain",
        start_time: baseTime + 3500,
        end_time: baseTime + 8000,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: { steps: "4", processed: "15" },
      }),
    );

    // 5. TOOL - External API call for additional context
    observations.push(
      createObservation({
        id: `${traceId}-tool`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-chain`,
        type: "TOOL",
        name: this.randomElement(REALISTIC_TOOL_NAMES),
        input: "Search for quantum computing market trends",
        output:
          "Market data: $1.2B industry, 25% YoY growth, key players identified",
        start_time: baseTime + 8000,
        end_time: baseTime + 10000,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: { toolType: "api-call", endpoint: "market-research" },
      }),
    );

    // 6. GENERATION - Final summary generation
    const toolObservation = observations[observations.length - 1];
    const toolName = toolObservation.name || "WebSearchTool";
    observations.push(
      createObservation({
        id: `${traceId}-generation`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-tool`,
        type: "GENERATION",
        name: this.randomElement(REALISTIC_GENERATION_NAMES),
        input:
          "Synthesize research analysis and market data into comprehensive summary",
        output:
          "Generated comprehensive 2000-word analysis of quantum computing research trends",
        provided_model_name: this.randomElement(REALISTIC_MODELS),
        model_parameters: JSON.stringify({
          temperature: 0.3,
          max_tokens: 2000,
        }),
        start_time: baseTime + 10000,
        end_time: baseTime + 15000,
        level: "DEFAULT",
        environment: trace.environment,
        usage_details: {
          input: this.randomInt(1500, 2500),
          output: this.randomInt(1800, 2200),
          total: this.randomInt(3300, 4700),
        },
        provided_usage_details: {
          input: this.randomInt(1500, 2500),
          output: this.randomInt(1800, 2200),
          total: this.randomInt(3300, 4700),
        },
        cost_details: {
          input: this.randomInt(15, 25) / 100000,
          output: this.randomInt(35, 45) / 100000,
          total: this.randomInt(50, 70) / 100000,
        },
        provided_cost_details: {
          input: this.randomInt(15, 25) / 100000,
          output: this.randomInt(35, 45) / 100000,
          total: this.randomInt(50, 70) / 100000,
        },
        tool_definitions: {
          WebSearchTool: "Search the web for information",
          CalculatorTool: "Perform mathematical calculations",
          WeatherForecastTool: "Get weather forecasts and data",
          EmailSenderTool: "Send emails to recipients",
        },
        tool_calls: ["call_workflow_1"],
        tool_call_names: [toolName],
      }),
    );

    // 7. EVALUATOR - Quality evaluation
    observations.push(
      createObservation({
        id: `${traceId}-evaluator`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-generation`,
        type: "EVALUATOR",
        name: this.randomElement(REALISTIC_EVALUATOR_NAMES),
        input: "Evaluate summary quality, accuracy, and completeness",
        output: "Quality score: 8.7/10, High accuracy, Comprehensive coverage",
        start_time: baseTime + 15000,
        end_time: baseTime + 16500,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: {
          qualityScore: "8.7",
          accuracy: "high",
          completeness: "comprehensive",
        },
      }),
    );

    // 8. GUARDRAIL - Safety and compliance check
    observations.push(
      createObservation({
        id: `${traceId}-guardrail`,
        trace_id: trace.id,
        project_id: projectId,
        parent_observation_id: `${traceId}-evaluator`,
        type: "GUARDRAIL",
        name: this.randomElement(REALISTIC_GUARDRAIL_NAMES),
        input: "Check content for safety, bias, and compliance issues",
        output:
          "✓ Content approved: No safety issues, Low bias detected, Compliant",
        start_time: baseTime + 16500,
        end_time: baseTime + 17000,
        level: "DEFAULT",
        environment: trace.environment,
        metadata: {
          safetyCheck: "passed",
          biasLevel: "low",
          compliance: "approved",
        },
      }),
    );

    return { trace, observations };
  }

  /**
   * Creates evaluation traces for testing evaluator configurations.
   * Use for: Evaluation testing, score validation, evaluator development.
   */
  generateEvaluationTraces(
    projectId: string,
    count: number,
  ): TraceRecordInsertType[] {
    const traces: TraceRecordInsertType[] = [];

    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
      for (let i = 0; i < count; i++) {
        const traceId = generateEvalTraceId(
          evalJobConfiguration.evalTemplateId,
          i,
          projectId,
        );
        const trace = createTrace({
          id: traceId,
          session_id: null,
          project_id: projectId,
          name: this.randomElement(REALISTIC_TRACE_NAMES),
          input: this.generateEvaluationInput(),
          output: this.generateEvaluationOutput(),
          user_id: this.randomBoolean(0.3)
            ? `user_${this.randomInt(1, 1000)}`
            : null,
          environment: "langfuse-evaluation",
          metadata: { purpose: "evaluation" },
          tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
          public: this.randomBoolean(0.8),
          bookmarked: this.randomBoolean(0.1),
          release: this.randomBoolean(0.4)
            ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
            : null,
          version: this.randomBoolean(0.4)
            ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
            : null,
        });

        traces.push(trace);
      }
    }

    return traces;
  }

  private generateTraceInput(): string {
    if (!this.fileContent) return "Sample input";

    // Match original logic: 30% chance of heavy markdown, otherwise chatML
    return this.randomBoolean(0.3)
      ? this.fileContent.heavyMarkdown
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  private generateTraceOutput(): string {
    if (!this.fileContent) return "Sample output";

    // Match original logic: 20% chance of nested JSON, otherwise chatML
    return this.randomBoolean(0.2)
      ? JSON.stringify(this.fileContent.nestedJson)
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  private generateObservationInput(): string {
    if (!this.fileContent) return "Sample observation input";

    // Match original logic: 40% chance of heavy markdown, otherwise chatML
    return this.randomBoolean(0.4)
      ? this.fileContent.heavyMarkdown
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  private generateObservationOutput(): string {
    if (!this.fileContent) return "Sample observation output";

    // Match original logic: 30% chance of nested JSON, otherwise chatML
    return this.randomBoolean(0.3)
      ? JSON.stringify(this.fileContent.nestedJson)
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  private generateEvaluationInput(): string {
    if (!this.fileContent) return "Evaluation input";

    return this.randomBoolean(0.3)
      ? this.fileContent.heavyMarkdown
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  private generateEvaluationOutput(): string {
    if (!this.fileContent) return "Evaluation output";

    return this.randomBoolean(0.2)
      ? JSON.stringify(this.fileContent.nestedJson)
      : JSON.stringify(this.fileContent.chatMlJson);
  }

  /**
   * Creates realistic support chat session data with conversational flow.
   * Use for: Demonstrating session-based conversations with tool calls and scoring.
   */
  generateSupportChatSessionData(projectId: string): {
    traces: TraceRecordInsertType[];
    observations: ObservationRecordInsertType[];
    scores: ScoreRecordInsertType[];
  } {
    // Domain: Art museum membership support (non-tech)
    const dialogues = [
      {
        user: "Hi—my family membership at the City Art Museum was charged twice this month.",
        assistant:
          "Sorry about that. I can help. Could you share your membership ID and the last 4 digits of the card?",
      },
      {
        user: "Membership ID is CAM-48291, card ending 4417.",
        assistant: "Thanks. I'll check your billing records now.",
        tool: {
          name: "billing.lookup",
          input: { membershipId: "CAM-48291", last4: "4417" },
          output: {
            duplicateCharge: true,
            amounts: [95.0, 95.0],
            date: "2025-08-28",
          },
        },
      },
      {
        user: "Okay.",
        assistant:
          "I found a duplicate charge on 2025-08-28 for $95. I can issue a refund to the original payment method. Shall I proceed?",
      },
      {
        user: "Yes, please issue the refund.",
        assistant: "Processing your refund now. One moment.",
        tool: {
          name: "billing.refund",
          input: { membershipId: "CAM-48291", amount: 95.0 },
          output: { status: "success", refundId: "RFND-20931" },
        },
      },
      {
        user: "Thank you!",
        assistant:
          "Refund RFND-20931 has been issued. You'll see it on your statement within 3–5 business days.",
      },
      {
        user: "No, that's all. Appreciate the quick help!",
        assistant: "Happy to help. Enjoy your next visit to the museum!",
      },
      // a couple more lightweight turns for scrolling realism
      {
        user: "Oh, and do members get early access to exhibitions?",
        assistant:
          "Yes—members get a 48-hour early booking window and a preview evening invite.",
      },
      {
        user: "Perfect.",
        assistant: "You're all set. Have a great day!",
      },
    ];

    const now = Date.now();
    const traces: TraceRecordInsertType[] = dialogues.map((d, index) => ({
      id: `support-chat-${index}-${projectId.slice(-8)}`,
      timestamp: toClickhouseDateTime(now + index * 1000),
      name: "SupportChatSession",
      user_id: null,
      metadata: this.buildNestedSeedMetadata("support-chat", index, {
        scenario: "support-chat",
        "routing.queue": "membership-support",
      }),
      release: null,
      version: null,
      project_id: projectId,
      environment: "default",
      public: false,
      bookmarked: false,
      tags: ["support", "chat", "session"],
      input: JSON.stringify(
        d.tool
          ? {
              messages: [
                { role: "user", content: d.user },
                { role: "assistant", content: d.assistant },
                {
                  role: "tool",
                  name: d.tool.name,
                  content: d.tool.output,
                },
              ],
            }
          : { messages: [{ role: "user", content: d.user }] },
      ),
      output: JSON.stringify({ role: "assistant", content: d.assistant }),
      session_id: "support-chat-session",
      created_at: toClickhouseDateTime(now + index * 1000),
      updated_at: toClickhouseDateTime(now + index * 1000 + 500),
      event_ts: toClickhouseDateTime(now + index * 1000),
      is_deleted: 0,
    }));

    // Create one GENERATION observation per trace
    const observations: ObservationRecordInsertType[] = dialogues
      .map((d, index) => {
        const start = now + index * 1000 + 50;
        const end = start + 400 + Math.floor(Math.random() * 400);
        const inputTokens = 80 + Math.floor(Math.random() * 60);
        const outputTokens = 60 + Math.floor(Math.random() * 60);
        const totalTokens = inputTokens + outputTokens;

        const baseGen: ObservationRecordInsertType = {
          id: `support-chat-${index}-${projectId.slice(-8)}-gen`,
          trace_id: `support-chat-${index}-${projectId.slice(-8)}`,
          project_id: projectId,
          type: "GENERATION",
          parent_observation_id: null,
          environment: "default",
          start_time: toClickhouseDateTime(start),
          end_time: toClickhouseDateTime(end),
          name: "llm-generation",
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          version: null,
          input: JSON.stringify({
            messages: [
              { role: "user", content: d.user },
              d.tool
                ? {
                    role: "tool",
                    name: d.tool.name,
                    content: d.tool.output,
                  }
                : undefined,
            ].filter(Boolean),
          }),
          output: JSON.stringify({ role: "assistant", content: d.assistant }),
          provided_model_name: "gpt-5.4-mini",
          internal_model_id: null,
          model_parameters: JSON.stringify({ temperature: 0.2 }),
          provided_usage_details: {
            input: inputTokens,
            output: outputTokens,
            total: totalTokens,
          },
          usage_details: {
            input: inputTokens,
            output: outputTokens,
            total: totalTokens,
          },
          provided_cost_details: {
            input: Math.round(inputTokens * 2) / 1_000_000,
            output: Math.round(outputTokens * 3) / 1_000_000,
            total: Math.round(totalTokens * 5) / 1_000_000,
          },
          cost_details: {
            input: Math.round(inputTokens * 2) / 1_000_000,
            output: Math.round(outputTokens * 3) / 1_000_000,
            total: Math.round(totalTokens * 5) / 1_000_000,
          },
          total_cost: Math.round(totalTokens * 5) / 1_000_000,
          completion_start_time: toClickhouseDateTime(start + 120),
          prompt_id: null,
          prompt_name: null,
          prompt_version: null,
          created_at: toClickhouseDateTime(start),
          updated_at: toClickhouseDateTime(end),
          event_ts: toClickhouseDateTime(start),
          is_deleted: 0,
          tool_definitions: d.tool
            ? {
                "billing.lookup":
                  "Look up billing records and transaction history",
                "billing.refund":
                  "Issue refunds for duplicate or erroneous charges",
              }
            : undefined,
          tool_calls: d.tool ? [`call_${index}`] : undefined,
          tool_call_names: d.tool ? [d.tool.name] : undefined,
        };

        if (!d.tool) return [baseGen];

        const toolObs: ObservationRecordInsertType = {
          id: `support-chat-${index}-${projectId.slice(-8)}-tool`,
          trace_id: `support-chat-${index}-${projectId.slice(-8)}`,
          project_id: projectId,
          type: "TOOL",
          parent_observation_id: null,
          environment: "default",
          start_time: toClickhouseDateTime(start - 40),
          end_time: toClickhouseDateTime(start - 5),
          name: d.tool.name,
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          version: null,
          input: JSON.stringify(d.tool.input),
          output: JSON.stringify(d.tool.output),
          provided_model_name: null,
          internal_model_id: null,
          model_parameters: null,
          provided_usage_details: {},
          usage_details: {},
          provided_cost_details: {},
          cost_details: {},
          total_cost: null,
          completion_start_time: null,
          prompt_id: null,
          prompt_name: null,
          prompt_version: null,
          created_at: toClickhouseDateTime(start - 40),
          updated_at: toClickhouseDateTime(start - 5),
          event_ts: toClickhouseDateTime(start - 40),
          is_deleted: 0,
          tool_definitions: undefined,
          tool_calls: undefined,
        };

        return [toolObs, baseGen];
      })
      .flat();

    // Create a couple of scores per trace
    const scores: ScoreRecordInsertType[] = dialogues
      .map((_, index) => {
        const baseTs = now + index * 1000 + 600;
        const helpfulness: ScoreRecordInsertType = {
          id: `support-chat-${index}-${projectId.slice(-8)}-score-helpfulness`,
          project_id: projectId,
          trace_id: `support-chat-${index}-${projectId.slice(-8)}`,
          session_id: null,
          dataset_run_id: null,
          observation_id: null,
          environment: "default",
          name: "helpfulness",
          value: 70 + Math.random() * 25,
          source: "API",
          comment: "Heuristic helpfulness score",
          metadata: {},
          author_user_id: null,
          config_id: null,
          data_type: "NUMERIC",
          string_value: null,
          long_string_value: "",
          queue_id: null,
          ...this.buildSeedIngestionAttribution(projectId, "javascript-sdk"),
          created_at: toClickhouseDateTime(baseTs),
          updated_at: toClickhouseDateTime(baseTs),
          timestamp: toClickhouseDateTime(baseTs),
          event_ts: toClickhouseDateTime(baseTs),
          is_deleted: 0,
        };

        const safeVal = Math.random() > 0.1 ? 1 : 0;
        const safety: ScoreRecordInsertType = {
          id: `support-chat-${index}-${projectId.slice(-8)}-score-safety`,
          project_id: projectId,
          trace_id: `support-chat-${index}-${projectId.slice(-8)}`,
          session_id: null,
          dataset_run_id: null,
          observation_id: null,
          environment: "default",
          name: "safe",
          value: safeVal,
          long_string_value: "",
          source: "API",
          comment: "Content safety",
          metadata: {},
          author_user_id: null,
          config_id: null,
          data_type: "BOOLEAN",
          string_value: safeVal === 1 ? "True" : "False",
          queue_id: null,
          ...this.buildSeedIngestionAttribution(projectId, "python-sdk"),
          created_at: toClickhouseDateTime(baseTs + 10),
          updated_at: toClickhouseDateTime(baseTs + 10),
          timestamp: toClickhouseDateTime(baseTs + 10),
          event_ts: toClickhouseDateTime(baseTs + 10),
          is_deleted: 0,
        };

        // Optional: resolution score on last turn
        const isFinal = index === dialogues.length - 1;
        const resolved: ScoreRecordInsertType | null = isFinal
          ? {
              id: `support-chat-${index}-${projectId.slice(-8)}-score-resolved`,
              project_id: projectId,
              trace_id: `support-chat-${index}-${projectId.slice(-8)}`,
              session_id: null,
              dataset_run_id: null,
              observation_id: null,
              environment: "default",
              name: "resolved",
              long_string_value: "",
              value: 1,
              source: "API",
              comment: "Conversation resolved",
              metadata: {},
              author_user_id: null,
              config_id: null,
              data_type: "BOOLEAN",
              string_value: "True",
              queue_id: null,
              ...this.buildSeedIngestionAttribution(projectId, "raw-api"),
              created_at: toClickhouseDateTime(baseTs + 20),
              updated_at: toClickhouseDateTime(baseTs + 20),
              timestamp: toClickhouseDateTime(baseTs + 20),
              event_ts: toClickhouseDateTime(baseTs + 20),
              is_deleted: 0,
            }
          : null;

        return [helpfulness, safety, resolved].filter(
          Boolean,
        ) as ScoreRecordInsertType[];
      })
      .flat();

    return { traces, observations, scores };
  }

  /**
   * Creates exactly one score per evaluation trace with prefixed IDs.
   * Use for: Evaluation traces that need score validation, evaluator testing.
   */
  generateEvaluationScores(
    traces: TraceRecordInsertType[],
    _observations: ObservationRecordInsertType[],
    projectId: string,
  ): ScoreRecordInsertType[] {
    const scores: ScoreRecordInsertType[] = [];

    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
      traces.forEach((trace, traceIndex) => {
        if (traceIndex % FAILED_EVAL_TRACE_INTERVAL === 0) return;
        // Create exactly one score per evaluation trace with prefixed ID
        const score: ScoreRecordInsertType = createTraceScore({
          id: generateEvalScoreId(
            evalJobConfiguration.evalTemplateId,
            traceIndex,
            projectId,
          ), // Use prefixed ID pattern
          project_id: projectId,
          trace_id: trace.id,
          observation_id: undefined, // Score is for the entire trace, not a specific observation
          name: `evaluation_score-${evalJobConfiguration.evalTemplateId}`,
          value: Math.random() * 100, // Random evaluation score 0-100
          string_value: undefined,
          data_type: "NUMERIC",
          source: "EVAL",
          comment: "Evaluation trace score",
          environment: trace.environment,
          ...this.buildSeedIngestionAttribution(projectId, "python-sdk"),
        });

        scores.push(score);
      });
    }

    return scores;
  }
}