Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

4.4 KB
Raw Blame
// Builds the "project data context" block injected into the v4 AI-filter prompt
// so the model maps natural language onto the project's ACTUAL columns, values,
// and metadata keys — instead of guessing (e.g. `type:chat` or
// `traceName:membership-support`). Reuses data already loaded for the bar:
// observed filter values (from `filterOptions`) + metadata keys sampled from the
// currently visible rows + the current result count (empty-state awareness).
//
// Everything is capped so the added context stays small (window/cost).

import type { ObservedOptions } from "./observed-options";
import { EVENTS_FIELD_REGISTRY, type FieldRegistry } from "./fields";

export function aiContextObservedOptionsKeys(
  registry: FieldRegistry = EVENTS_FIELD_REGISTRY,
): readonly string[] {
  return registry.aiContextFields.map(
    ({ observedOptionsKey }) => observedOptionsKey,
  );
}

const MAX_VALUES_PER_COL = 25;
const MAX_METADATA_KEYS = 30;
const MAX_VALUE_LEN = 40;
// Stay well under the endpoint's `dataContext: z.string().max(16000)` cap.
const MAX_CONTEXT_CHARS = 12000;

function safeJsonParse(s: string): unknown {
  try {
    return JSON.parse(s);
  } catch {
    return undefined;
  }
}

/** Flatten nested metadata into dot-path keys with one example leaf value. */
function collectMetadataKeys(
  md: unknown,
  prefix: string,
  out: Map<string, string>,
  depth: number,
): void {
  if (depth > 3 || out.size >= MAX_METADATA_KEYS) return;
  if (md === null || typeof md !== "object" || Array.isArray(md)) return;
  for (const [k, v] of Object.entries(md as Record<string, unknown>)) {
    if (out.size >= MAX_METADATA_KEYS) break;
    const path = prefix ? `${prefix}.${k}` : k;
    if (v !== null && typeof v === "object" && !Array.isArray(v)) {
      collectMetadataKeys(v, path, out, depth + 1);
    } else if (!out.has(path)) {
      const example =
        v === null || v === undefined || Array.isArray(v)
          ? ""
          : String(v).slice(0, MAX_VALUE_LEN);
      out.set(path, example);
    }
  }
}

export function buildAiContext(args: {
  observed: ObservedOptions | undefined;
  /** Metadata objects from the currently visible rows (sampled by the caller). */
  sampleMetadata: unknown[];
  /** Number of currently-loaded/visible rows (so the model can warn when 0). */
  resultCount: number | null;
  registry?: FieldRegistry;
}): string | undefined {
  const {
    observed,
    sampleMetadata,
    resultCount,
    registry = EVENTS_FIELD_REGISTRY,
  } = args;
  const sections: string[] = [];

  if (observed) {
    const valueLines: string[] = [];
    for (const {
      observedOptionsKey,
      promptLabel,
    } of registry.aiContextFields) {
      const vals = (observed[observedOptionsKey] ?? [])
        .map((o) => o.value)
        .filter((v) => v.length > 0)
        .slice(0, MAX_VALUES_PER_COL)
        .map((v) => v.slice(0, MAX_VALUE_LEN));
      if (vals.length > 0)
        valueLines.push(`- ${promptLabel}: ${vals.join(", ")}`);
    }
    if (valueLines.length > 0) {
      sections.push(
        [
          "Observed values per column (use these exact values; lists may be truncated):",
          ...valueLines,
        ].join("\n"),
      );
    }
  }

  const mdKeys = new Map<string, string>();
  for (const md of sampleMetadata) {
    // Langfuse returns row metadata as a JSON-encoded string; parse it first.
    const parsed = typeof md === "string" ? safeJsonParse(md) : md;
    collectMetadataKeys(parsed, "", mdKeys, 0);
    if (mdKeys.size >= MAX_METADATA_KEYS) break;
  }
  if (mdKeys.size > 0) {
    const keyLines = [...mdKeys.entries()].map(
      ([k, ex]) => `- metadata.${k}${ex ? ` (e.g. "${ex}")` : ""}`,
    );
    sections.push(
      [
        "Observed metadata keys (filter as metadata.<key>; match the user's phrase to one of these):",
        ...keyLines,
      ].join("\n"),
    );
  }

  if (resultCount !== null) {
    sections.push(
      resultCount === 0
        ? "The current filters match no visible rows — they may be too strict; consider broadening."
        : `The current view is showing ${resultCount} row(s).`,
    );
  }

  if (sections.length === 0) return undefined;
  // Hard cap so the context can never exceed the endpoint's input limit
  // (router `dataContext: z.string().max(16000)`) and trigger a Zod 400, even
  // with many columns/keys. Per-value/per-key caps above keep us well under this.
  const out = sections.join("\n\n");
  return out.length > MAX_CONTEXT_CHARS ? out.slice(0, MAX_CONTEXT_CHARS) : out;
}