Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

21.2 KB
Raw Blame
import { randomUUID } from "crypto";

import { JobConfigState, Prisma, type PrismaClient } from "../../../db";
import { env } from "../../../env";
import { TriggerEventSource } from "../../../domain/automations";
import { matchesTriggerFilter } from "../../../server/automations";
import {
  instrumentAsync,
  instrumentSync,
} from "../../../server/instrumentation";
import { logger } from "../../../server/logger";
import {
  getTriggerConfigurations as defaultGetTriggerConfigurations,
  type TriggerDomainWithActions,
} from "../../../server/repositories/automation-repository";
import { executeQuery as defaultExecuteQuery } from "../../query/server/queryExecutor";
import { getViewDeclaration } from "../../query/dataModel";
import {
  decodeFiltersGeneric,
  encodeFiltersGeneric,
} from "../../filters/filterQueryEncoding";
import { scoresTableCols } from "../../../tableDefinitions/scoresTable";
import { scoresTableUiColumnDefinitions } from "../../../server/tableMappings/mapScoresTable";
import { COMPATIBLE_FILTER_TYPES } from "../../../server/queries/clickhouse-sql/filterTypeCompatibility";
import type { FilterState } from "../../../types";
import type { QueryType } from "../../query/types";
import { isValidQuery } from "../isValidQuery";
import {
  MonitorQueueEventSchema,
  type MonitorQueueEvent,
  type MonitorQueueEventInput,
  type MonitorWebhookInput,
} from "../scheduler/types";
import { monitorFromPrisma, windowToMs } from "../service/helpers";
import {
  monitorEvaluationOffsetMs,
  MonitorSeveritySchema,
  MonitorStatusSchema,
  type MonitorAlert,
  type MonitorSeverity,
  type MonitorWindow,
  type MonitorView,
  type Monitor,
} from "../types";
import { applyStateMachine, type MonitorCompletion } from "./applyStateMachine";
import { computeSeverity } from "./computeSeverity";
import { renderAlertMessage } from "./renderAlertMessage";

export { monitorEvaluationOffsetMs } from "../types";

/** ErrorBadQuery sentinels a metric rejected by monitor query validation. */
export const ErrorBadQuery = Symbol("ErrorBadQuery");

/** MonitorProcessor evaluates queued monitor events and emits MonitorAlerts. */
export class MonitorProcessor {
  constructor(
    private readonly db: PrismaClient,
    private readonly publish: MonitorPublisher,
    private readonly executeQuery: QueryExecutor = defaultExecuteQuery,
    private readonly getTriggerConfigurations: GetTriggerConfigurations = defaultGetTriggerConfigurations,
  ) {}

  /** process evaluates one queued monitor event and publishes any resulting alerts; the input is parsed to recoerce dates the queue serialized to strings. */
  async process(input: MonitorQueueEventInput, now: Date): Promise<void> {
    const event = MonitorQueueEventSchema.parse(input);
    return instrumentAsync({ name: "process" }, async (span) => {
      const monitors = await instrumentAsync({ name: "claimMonitors" }, () =>
        this.claimMonitors(event, now),
      );
      span.setAttribute("monitors", monitors.length);
      if (monitors.length === 0) return;

      const [metrics, triggers] = await Promise.all([
        instrumentAsync({ name: "queryMetrics" }, () =>
          this.queryMetrics(event),
        ),
        instrumentAsync({ name: "getTriggerConfigurations" }, () =>
          this.getTriggerConfigurations({
            projectId: event.projectId,
            eventSource: TriggerEventSource.Monitor,
            status: JobConfigState.ACTIVE,
          }),
        ),
      ]);

      span.setAttribute("metrics", Object.keys(metrics).length);
      span.setAttribute("triggers", triggers.length);

      const [completions, monitorWebhookInputs] = instrumentSync(
        { name: "processMonitors" },
        () =>
          processMonitors({
            monitors,
            metrics,
            triggers,
            now,
            runAt: event.runAt,
            publishedAt: event.publishedAt,
          }),
      );
      span.setAttribute("monitorWebhookInputs", monitorWebhookInputs.length);

      await instrumentAsync({ name: "publishWebhookInputs" }, () =>
        this.publishWebhookInputs(monitorWebhookInputs),
      );

      await instrumentAsync({ name: "complete" }, () =>
        this.complete({ projectId: event.projectId, completions }),
      );
    });
  }

  /** claimMonitors conditionally claims the event's monitors for this worker, returning the rows it won. */
  private async claimMonitors(
    event: MonitorQueueEvent,
    now: Date,
  ): Promise<Monitor[]> {
    if (event.monitors.length === 0) return [];
    const prismaMonitors = await this.db.monitor.updateManyAndReturn({
      where: {
        id: { in: event.monitors.map((m) => m.monitorId) },
        projectId: event.projectId,
        status: MonitorStatusSchema.enum.ACTIVE, // active monitors for the
        lastPublishedAt: { lte: event.publishedAt }, // newest event
        AND: [
          // not already claimed
          {
            OR: [
              { lastClaimedAt: null },
              { lastClaimedAt: { lte: event.publishedAt } },
            ],
          },
          // and not yet completed
          {
            OR: [
              { lastCompletedAt: null },
              { lastCompletedAt: { lt: event.publishedAt } },
            ],
          },
        ],
      },
      data: { lastClaimedAt: now },
    });
    return prismaMonitors.map(monitorFromPrisma);
  }

  /** queryMetrics pre-screens the batch's metrics against the v2 data model, runs the accepted ones, and returns each metric keyed by `${aggregation}_${measure}` as a number, null, or the ErrorBadQuery sentinel. */
  private async queryMetrics(event: MonitorQueueEvent): Promise<MetricMap> {
    const validation = isValidQuery({
      view: event.view,
      metrics: event.metrics,
      filters: event.filters,
    });
    const metricMap: MetricMap = {};
    for (const metric of validation.rejected) {
      metricMap[metricKey(metric)] = ErrorBadQuery;
    }
    if (validation.accepted.length === 0) return metricMap;

    try {
      const rows = await this.executeQuery(
        event.projectId,
        buildMonitorQuery(validation.accepted, event),
        "v2",
        true,
      );
      const row = (rows[0] ?? {}) as Record<string, unknown>;
      for (const metric of validation.accepted) {
        const key = metricKey(metric);
        metricMap[key] = parseNumericValue(row[key]);
      }
      metricMap["count_count"] = parseNumericValue(row["count_count"]);
    } catch (error) {
      logger.error("monitor processor failed", {
        error: error,
        projectId: event.projectId,
        schedulerBatchId: event.schedulerBatchId.toString(),
        monitorIds: event.monitors.map((m) => m.monitorId),
      });
      throw error;
    }
    return metricMap;
  }

  private async publishWebhookInputs(
    inputs: MonitorWebhookInput[],
  ): Promise<void> {
    for (const input of inputs) {
      await this.publish(input);
    }
  }

  /** complete writes each monitor's post-evaluation lifecycle stamps in one statement. */
  private async complete(args: {
    projectId: string;
    completions: MonitorCompletion[];
  }): Promise<void> {
    if (args.completions.length === 0) return;
    await this.db.$transaction([
      // tz-naive columns are read back as UTC by Prisma; pin the session so raw casts store UTC wall-clock
      this.db.$executeRawUnsafe(`SET LOCAL TIME ZONE 'UTC'`),
      this.db.$executeRaw(
        buildCompleteQuery({
          projectId: args.projectId,
          completions: args.completions,
        }),
      ),
    ]);
  }
}

/** buildMonitorQuery converts the accepted metrics of a MonitorQueueEvent into the scalar QueryType executeQuery accepts. */
function buildMonitorQuery(
  acceptedMetrics: QueryType["metrics"],
  event: MonitorQueueEvent,
): QueryType {
  const { fromTimestamp, toTimestamp } = evaluationWindow(
    event.window,
    event.runAt,
  );
  const metrics = dedupeMetrics([
    ...acceptedMetrics,
    { measure: "count", aggregation: "count" as const },
  ]);
  return {
    view: event.view,
    dimensions: [],
    metrics,
    filters: event.filters,
    timeDimension: null,
    fromTimestamp: fromTimestamp.toISOString(),
    toTimestamp: toTimestamp.toISOString(),
    orderBy: null,
  };
}

/** dedupeMetrics drops duplicate metrics keyed by `${aggregation}_${measure}` so the appended row-count metric never collides with an existing count metric. */
function dedupeMetrics(metrics: QueryType["metrics"]): QueryType["metrics"] {
  const seen = new Set<string>();
  return metrics.filter((m) => {
    const key = metricKey(m);
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
}

/** metricKey is the `${aggregation}_${measure}` column name a metric resolves to in the query result row. */
function metricKey(metric: { measure: string; aggregation: string }): string {
  return `${metric.aggregation}_${metric.measure}`;
}

/** evaluationWindow returns the `[runAt - window, runAt]` edges, both shifted back by monitorEvaluationOffsetMs. */
function evaluationWindow(
  window: MonitorWindow,
  runAt: Date,
): {
  fromTimestamp: Date;
  toTimestamp: Date;
} {
  const windowMs = Number(windowToMs(window));
  const toTimestamp = new Date(runAt.getTime() - monitorEvaluationOffsetMs);
  const fromTimestamp = new Date(toTimestamp.getTime() - windowMs);
  return { fromTimestamp, toTimestamp };
}

/** parseNumericValue coerces a ClickHouse cell to number | null, mapping missing or non-finite values to null. */
function parseNumericValue(raw: unknown): number | null {
  if (raw === null || raw === undefined) return null;
  if (typeof raw === "number") return Number.isFinite(raw) ? raw : null;
  const n = Number(raw);
  return Number.isFinite(n) ? n : null;
}

/** processMonitors evaluates every claimed monitor, collecting the completions to persist and the webhook inputs to publish. */
function processMonitors(args: {
  monitors: Monitor[];
  metrics: MetricMap;
  triggers: TriggerDomainWithActions[];
  now: Date;
  runAt: Date;
  publishedAt: Date;
}): [MonitorCompletion[], MonitorWebhookInput[]] {
  const completions: MonitorCompletion[] = [];
  const monitorWebhookInputs: MonitorWebhookInput[] = [];
  for (const monitor of args.monitors) {
    const [completion, inputs] = processMonitor({
      monitor,
      metrics: args.metrics,
      triggers: args.triggers,
      now: args.now,
      runAt: args.runAt,
      publishedAt: args.publishedAt,
    });
    completions.push(completion);
    monitorWebhookInputs.push(...inputs);
  }
  return [completions, monitorWebhookInputs];
}

/** processMonitor evaluates one monitor and returns its lifecycle completion plus any webhook inputs to publish; a bad-query metric short-circuits to an ERROR_BAD_QUERY completion with no alert. */
function processMonitor(args: {
  monitor: Monitor;
  metrics: MetricMap;
  triggers: TriggerDomainWithActions[];
  now: Date;
  runAt: Date;
  publishedAt: Date;
}): [MonitorCompletion, MonitorWebhookInput[]] {
  const { monitor, metrics, triggers, now, runAt, publishedAt } = args;
  const value = getMetricValue(metrics, monitor.metric);
  if (value === ErrorBadQuery) {
    return [
      {
        monitorId: monitor.id,
        lastClaimedAt: now,
        lastCompletedAt: now,
        publishedAt,
        status: MonitorStatusSchema.enum.ERROR_BAD_QUERY,
        severity: MonitorSeveritySchema.enum.PAUSED,
        severityChangedAt: now,
        alertedAt: monitor.alertedAt,
      },
      [],
    ];
  }

  const severity = computeSeverity({
    value,
    noData: monitor.noData,
    prevSeverity: monitor.severity,
    operator: monitor.thresholdOperator,
    alertThreshold: monitor.alertThreshold,
    warningThreshold: monitor.warningThreshold ?? null,
  });

  const { completion, emit } = applyStateMachine({
    prev: monitor,
    next: { severity },
    now,
    publishedAt,
  });
  if (!emit) return [completion, []];

  const automations = getAutomations({ monitor, completion, triggers });
  if (automations.length === 0) return [completion, []];

  const alert = buildAlert({ prev: monitor, next: completion, runAt, value });
  return [completion, toMonitorWebhookInputs({ alert, automations, now })];
}

/** getMetricValue reads a monitor's scalar result from the metrics map, returning the ErrorBadQuery sentinel before the zero-count NO_DATA gate so a rejected metric flips rather than reads NO_DATA. */
function getMetricValue(
  metrics: MetricMap,
  metric: Monitor["metric"],
): MetricValue {
  const value = metrics[metricKey(metric)];
  if (value === ErrorBadQuery) return ErrorBadQuery;
  if (metrics["count_count"] === 0) return null;
  return value ?? null;
}

/** getAutomations returns the automations under every trigger that consumes this alert. */
function getAutomations(args: {
  monitor: Monitor;
  completion: MonitorCompletion;
  triggers: TriggerDomainWithActions[];
}): TriggerDomainWithActions["automations"] {
  const filterData = {
    severity: args.completion.severity,
    triggerIds: args.monitor.triggerIds,
  };
  return args.triggers
    .filter((trigger) => matchesTriggerFilter(filterData, trigger))
    .flatMap((trigger) => trigger.automations);
}

/** buildAlert assembles the MonitorAlert payload from the monitor row and state-machine completion. */
function buildAlert(args: {
  prev: Monitor;
  next: MonitorCompletion;
  runAt: Date;
  value: number | null;
}): MonitorAlert {
  const { prev, next, runAt, value } = args;
  const { fromTimestamp, toTimestamp } = evaluationWindow(prev.window, runAt);
  return {
    monitorId: prev.id,
    projectId: prev.projectId,
    severity: next.severity,
    timestamp: runAt,
    fromTimestamp,
    toTimestamp,
    permalink: buildPermalink(prev.projectId, prev.id),
    // Only a threshold-cross (ALERT/WARNING) links to the breaching data window;
    // recovery (OK) and lifecycle states (NO_DATA/UNKNOWN/PAUSED) would point at
    // a recovered/empty window, so they carry no data link.
    dataPermalink: isBreaching(next.severity)
      ? buildDataWindowPermalink(
          prev.projectId,
          prev.view,
          fromTimestamp,
          toTimestamp,
          prev.filters,
        )
      : undefined,
    message: renderAlertMessage({ monitor: prev, completion: next }),
    view: prev.view,
    filters: prev.filters,
    window: prev.window,
    monitorName: prev.name,
    metric: prev.metric,
    metricValue: value,
    thresholdOperator: prev.thresholdOperator,
    alertThreshold: prev.alertThreshold,
    warningThreshold: prev.warningThreshold,
    alertedAt: next.alertedAt ?? undefined,
  };
}

/** buildPermalink composes the absolute Langfuse URL for a monitor, or undefined when NEXTAUTH_URL is unset. */
export function buildPermalink(
  projectId: string,
  monitorId: string,
): string | undefined {
  if (!env.NEXTAUTH_URL) return undefined;
  const base = env.NEXTAUTH_URL.replace(/\/$/, "");
  return `${base}/project/${projectId}/alerts/${monitorId}`;
}

/** isBreaching returns true for the threshold-cross severities (ALERT/WARNING) whose alert should deep-link to the breaching data window; OK (recovery) and the lifecycle states (NO_DATA/UNKNOWN/PAUSED) return false. */
export function isBreaching(severity: MonitorSeverity): boolean {
  return (
    severity === MonitorSeveritySchema.enum.ALERT ||
    severity === MonitorSeveritySchema.enum.WARNING
  );
}

/**
 * buildDataWindowPermalink composes the absolute Langfuse data-table URL scoped
 * to the breaching evaluation window, or undefined when NEXTAUTH_URL is unset.
 *
 * The window is encoded as the table's custom `?dateRange=<fromMs>-<toMs>`
 * param (absolute epoch-ms range; the client date-range parser accepts custom
 * ranges directly, bypassing the preset gating). Score views carry their
 * input predicates and implicit data-type restrictions to the scores table.
 * A score link is omitted when the destination cannot preserve every filter.
 */
export function buildDataWindowPermalink(
  projectId: string,
  view: MonitorView,
  fromTimestamp: Date,
  toTimestamp: Date,
  filters: FilterState = [],
): string | undefined {
  if (!env.NEXTAUTH_URL) return undefined;
  const base = env.NEXTAUTH_URL.replace(/\/$/, "");
  const dateRange = `${fromTimestamp.getTime()}-${toTimestamp.getTime()}`;
  if (view === "observations") {
    return `${base}/project/${projectId}/observations?dateRange=${dateRange}`;
  }

  const tableFilters: FilterState = [];
  const segments = getViewDeclaration(view, "v2").segments.map((filter) => ({
    ...filter,
    column: filter.column === "data_type" ? "dataType" : filter.column,
  }));
  for (const filter of [...filters, ...segments]) {
    // Monitor booleans use the numeric 0/1 value. The table's Boolean Value
    // facet uses string_value, which may be absent on existing boolean scores.
    if (filter.column === "booleanValue" && filter.type === "boolean") {
      const value = filter.operator === "=" ? filter.value : !filter.value;
      tableFilters.push({
        column: "value",
        type: "number",
        operator: "=",
        value: Number(value),
      });
      continue;
    }

    // Trace-derived dimensions use different joins in the monitor and the
    // legacy/events-backed scores readers. Only score-owned fields are portable.
    const mapping = scoresTableUiColumnDefinitions.find(
      (column) =>
        column.uiTableId === filter.column &&
        column.clickhouseTableName === "scores",
    );
    const definition = scoresTableCols.find(
      (column) => column.id === filter.column,
    );
    if (
      !mapping ||
      mapping.emptyEqualsNull ||
      !definition ||
      (filter.type !== "null" &&
        !COMPATIBLE_FILTER_TYPES[definition.type]?.includes(filter.type))
    ) {
      return undefined;
    }

    if (filter.type === "boolean") {
      tableFilters.push({
        ...filter,
        operator: "=",
        value: filter.operator === "=" ? filter.value : !filter.value,
      });
    } else {
      tableFilters.push(filter);
    }
  }

  let encodedFilters: string;
  try {
    encodedFilters = encodeFiltersGeneric(tableFilters);
    // The table canonicalizes its URL after decoding. Keys containing legacy
    // delimiters or percent escapes must survive that round-trip unchanged.
    if (
      encodeFiltersGeneric(decodeFiltersGeneric(encodedFilters)) !==
      encodedFilters
    ) {
      return undefined;
    }
  } catch {
    return undefined;
  }

  const query = new URLSearchParams({
    dateRange,
    filter: encodedFilters,
    // Alert filters own the environment scope, including an unrestricted one.
    showAllEnvironments: "true",
  });
  const href = `${base}/project/${projectId}/scores?${query.toString()}`;
  // Slack section text is limited to 3000 characters. Dropping predicates to
  // shorten a link would show scores outside the evaluated query.
  return href.length <= 3000 ? href : undefined;
}

/** toMonitorWebhookInputs fans an alert out to one webhook input per matched automation. */
function toMonitorWebhookInputs(args: {
  alert: MonitorAlert;
  automations: TriggerDomainWithActions["automations"];
  now: Date;
}): MonitorWebhookInput[] {
  return args.automations.map((automation) => {
    const executionId = randomUUID();
    return {
      projectId: args.alert.projectId,
      automationId: automation.id,
      executionId,
      payload: {
        id: executionId,
        timestamp: args.now,
        type: "monitor-alert",
        apiVersion: "v1",
        payload: args.alert,
      },
    };
  });
}

/** buildCompleteQuery builds the bulk UPDATE that lands every monitor's post-evaluation stamps. */
function buildCompleteQuery(args: {
  projectId: string;
  completions: MonitorCompletion[];
}): Prisma.Sql {
  const valueRows = Prisma.join(
    args.completions.map(
      (c) =>
        Prisma.sql`(${c.monitorId}, ${c.lastClaimedAt}::timestamptz, ${c.lastCompletedAt}::timestamptz, ${c.publishedAt}::timestamptz, ${c.status}::"MonitorStatus", ${c.severity}::"MonitorSeverity", ${c.severityChangedAt}::timestamptz, ${c.alertedAt}::timestamptz)`,
    ),
    ", ",
  );
  return Prisma.sql`
    UPDATE monitors AS m
    SET
      last_completed_at = data.last_completed_at,
      status = data.status,
      severity = data.severity,
      severity_changed_at = data.severity_changed_at,
      alerted_at = data.alerted_at
    FROM (VALUES ${valueRows}) AS data(
      id,
      last_claimed_at,
      last_completed_at,
      published_at,
      status,
      severity,
      severity_changed_at,
      alerted_at
    )
    WHERE m.id = data.id
      AND m.project_id = ${args.projectId}
      AND m.last_claimed_at = data.last_claimed_at -- no-op if another worker re-claimed since
      AND m.status = 'ACTIVE' -- no-op if the user paused since claim
      AND m.last_published_at = data.published_at -- no-op if the scheduler rescued/republished since claim
  `;
}

/** MetricValue is one metric's evaluated result: a number, null (missing/non-finite), or the ErrorBadQuery sentinel. */
export type MetricValue = number | null | typeof ErrorBadQuery;

/** MetricMap keys each metric's MetricValue by `${aggregation}_${measure}`. */
type MetricMap = Record<string, MetricValue>;

/** MonitorPublisher publishes one MonitorWebhookInput onto the webhook queue. */
export type MonitorPublisher = (input: MonitorWebhookInput) => Promise<void>;

/** QueryExecutor runs a monitor's ClickHouse query. */
export type QueryExecutor = typeof defaultExecuteQuery;

/** GetTriggerConfigurations loads the trigger configurations matching a filter. */
export type GetTriggerConfigurations = typeof defaultGetTriggerConfigurations;