Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

1.9 KB
Raw Blame
import {
  isS3SlowDownError,
  logger,
  traceException,
} from "@langfuse/shared/src/server";

import { isFinalBullmqAttempt } from "../features/integrations/bullmqAttempts";

/**
 * Log a failed queue attempt. Retryable S3 SlowDown stays a warning until the
 * BullMQ attempt budget is spent, then it is logged as an error.
 *
 * `attemptsIncludeCurrentFailure` is set when BullMQ has already incremented
 * `attemptsMade` (the worker `failed` handler, which runs after `moveToFailed`).
 * Processors see the pre-increment count and leave the flag unset.
 */
export function logRetryableJobFailure(params: {
  message: string;
  error: unknown;
  job?: AttemptState;
  attemptsIncludeCurrentFailure?: boolean;
  fields?: Record<string, unknown>;
}): void {
  const { message, error, job, attemptsIncludeCurrentFailure, fields } = params;
  const payload = fields ? { error, ...fields } : error;
  if (
    isS3SlowDownError(error) &&
    job &&
    !isTerminalAttempt(job, error, attemptsIncludeCurrentFailure)
  ) {
    logger.warn(message, payload);
    return;
  }

  logger.error(message, payload);
  traceException(error);
}

/**
 * Whether a non-SlowDown failure has used up `budget` attempts. Lets a queue
 * keep a long retry budget for S3 throttling without extending it to failures
 * that retrying will not fix.
 */
export function exceedsNonSlowDownAttemptBudget(
  job: AttemptState,
  error: unknown,
  budget: number,
): boolean {
  return !isS3SlowDownError(error) && (job.attemptsMade ?? 0) >= budget - 1;
}

function isTerminalAttempt(
  job: AttemptState,
  error: unknown,
  attemptsIncludeCurrentFailure: boolean | undefined,
): boolean {
  const attemptsMade = job.attemptsMade ?? 0;
  const observed = attemptsIncludeCurrentFailure
    ? Math.max(0, attemptsMade - 1)
    : attemptsMade;
  return isFinalBullmqAttempt(
    { attemptsMade: observed, opts: job.opts },
    error,
  );
}

interface AttemptState {
  attemptsMade?: number;
  opts?: { attempts?: number };
}