Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

19.5 KB
Raw Blame
/**
 * Proxy Logger — Hybrid in-memory + SQLite persistence
 *
 * Keeps a fast in-memory ring buffer for real-time dashboard AND
 * persists to SQLite so logs survive server restarts.
 *
 * Pattern follows callLogs.js (T-15 decomposition).
 */
import { v4 as uuidv4 } from "uuid";
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts";
import { sanitizeTimingMs } from "@omniroute/open-sse/utils/timingMs.ts";
import { getDbInstance, isCloud, isBuildPhase } from "./db/core";
import { ensureProxyLogsColumns } from "./db/schemaColumns";
import { normalizeProxyHostForLog } from "./proxyLogHost";
import { isWriteLockError, scheduleWriteRetry } from "./usage/usageWriteRetry";

// Re-exported for existing callers; the helper lives in a zero-import leaf so DB modules
// can use it without loading this module (which hydrates from SQLite at import time).
export { normalizeProxyHostForLog };

const shouldPersistToDisk = !isCloud && !isBuildPhase;

const MAX_IN_MEMORY_ENTRIES = 200;

interface ProxyInfo {
  type: string;
  host: string;
  port: number | string;
  /** Registry name (e.g. `murphy-eu-fr`) — carried by registry resolution so the
   *  proxy log can identify a leg even when many entries share host:port. */
  name?: string;
}

interface ProxyLogEntry {
  id: string;
  timestamp: string;
  status: string;
  proxy: ProxyInfo | null;
  level: string;
  levelId: string | null;
  provider: string | null;
  targetUrl: string | null;
  clientIp: string | null;
  /** Outbound/egress IP the upstream actually saw (null until probed). The
   * historical clientIp is the INBOUND IP (x-forwarded-for); egressIp answers
   * "by which IP is this account leaving" — critical for rotating providers. */
  egressIp: string | null;
  latencyMs: number;
  error: string | null;
  connectionId: string | null;
  comboId: string | null;
  // `account` is the configured connection id prefix (connectionId slice);
  // `rotationAccount` is the masked id of the rotation account that served the
  // request (multi-account anonymous rotation only, null otherwise). Both stay
  // masked prefixes — never a full account id.
  account: string | null;
  /** Masked serving-account id of the rotation executor (null unless set). */
  rotationAccount: string | null;
  /** Request correlation id shared with call_logs (null unless set). */
  correlationId: string | null;
  tlsFingerprint: boolean;
  /** HTTP status the provider actually returned; null when no response was received. */
  upstreamStatus: number | null;
  /** 1-based position of this row within its request journal; null for unjournaled rows. */
  attemptNumber: number | null;
  /** Outcome of this send within its request journal; null for unjournaled rows. */
  attemptIssue: "served" | "abandoned" | null;
  /** Send start -> response headers received; null when unknown (network throw). */
  headersMs: number | null;
  /** Send start -> first useful body byte; null until the byte arrives. */
  firstChunkMs: number | null;
}

type ProxyLogInput = Partial<ProxyLogEntry> & {
  publicIp?: string | null;
};

interface ProxyLogFilters {
  status?: string;
  type?: string;
  provider?: string;
  level?: string;
  search?: string;
  limit?: number;
}

const proxyLogs: ProxyLogEntry[] = [];

// `public_ip` is the historical SQLite column name; API/UI expose the value as clientIp.

// ──────────────── Startup: hydrate from DB ────────────────

function loadFromDb() {
  if (!shouldPersistToDisk) return;
  try {
    const db = getDbInstance();
    // Self-heal the proxy_logs schema before reading/writing (migration 134
    // guarantees egress_ip on every migrated DB; this covers restored/odd states).
    ensureProxyLogsColumns(db);
    const rows = db
      .prepare("SELECT * FROM proxy_logs ORDER BY timestamp DESC LIMIT ?")
      .all(MAX_IN_MEMORY_ENTRIES) as any[];

    for (const row of rows) {
      proxyLogs.push({
        id: row.id,
        timestamp: row.timestamp,
        status: row.status || "success",
        proxy: row.proxy_host
          ? {
              type: row.proxy_type,
              host: row.proxy_host,
              port: row.proxy_port,
              name: row.proxy_name || undefined,
            }
          : null,
        level: row.level || "direct",
        levelId: row.level_id || null,
        provider: row.provider || null,
        targetUrl: row.target_url || null,
        clientIp: row.public_ip || null,
        egressIp: row.egress_ip || null,
        latencyMs: row.latency_ms || 0,
        error: row.error || null,
        connectionId: row.connection_id || null,
        comboId: row.combo_id || null,
        account: row.account || null,
        rotationAccount: row.rotation_account || null,
        correlationId: row.correlation_id || null,
        tlsFingerprint: row.tls_fingerprint === 1,
        upstreamStatus: typeof row.upstream_status === "number" ? row.upstream_status : null,
        attemptNumber: typeof row.attempt_number === "number" ? row.attempt_number : null,
        attemptIssue:
          row.attempt_issue === "served" || row.attempt_issue === "abandoned"
            ? row.attempt_issue
            : null,
        headersMs: sanitizeTimingMs(row.headers_ms),
        firstChunkMs: sanitizeTimingMs(row.first_chunk_ms),
      });
    }

    if (proxyLogs.length > 0) {
      console.log(`[proxyLogger] Loaded ${proxyLogs.length} proxy logs from SQLite`);
    }
  } catch (err: any) {
    console.warn(
      "[proxyLogger] Failed to load from DB:",
      sanitizeErrorMessage(err) || "Proxy log hydration failed"
    );
  }
}

loadFromDb();

// Default-off override that restores the verbose [ProxyEgress] console line (raw
// client/egress IPs + account prefix). Kept OFF by default so the process log leaks
// neither IPs nor the account prefix. Deliberately NOT coupled to debugMode
// (src/lib/db/settings.ts defaults debugMode to true) — this verbosity is opt-in only.
// Storage (in-memory ring buffer + SQLite) is untouched and always keeps full IPs.

/** Read at call time so tests can toggle it between imports. */
export function isProxyLogIncludeIps(): boolean {
  return process.env.PROXY_LOG_INCLUDE_IPS === "true" || process.env.PROXY_LOG_INCLUDE_IPS === "1";
}

/**
 * Pure formatter for the [ProxyEgress] process-log line (#10348). At the default level it
 * emits a short, IP/prefix-free summary; when details are opted in it restores the full
 * verbose line including client/egress IPs and the account. Extracted as a separate
 * function so it is unit-testable without patching console.log and so the change never
 * grows logProxyEvent itself.
 */
export function formatProxyEgressConsoleLine(params: {
  provider: string | null;
  account: string | null;
  clientIp: string | null;
  egressIp: string | null;
  level: string;
  proxyHost: string | null | undefined;
  proxyName?: string | null | undefined;
  status: string;
  includeDetails?: boolean;
}): string {
  const provider = params.provider || "-";
  const status = params.status;
  if (!params.includeDetails) {
    return `[ProxyEgress] ${provider} status=${status}`;
  }
  const proxy = params.proxyHost ? `:${params.proxyHost}` : "";
  const name = params.proxyName ? ` name=${params.proxyName}` : "";
  return (
    `[ProxyEgress] ${provider}/${params.account || "-"} ` +
    `in=${params.clientIp || "?"} out=${params.egressIp || "?"} ` +
    `proxy=${params.level}${proxy}${name} status=${status}`
  );
}

// ──────────────── Log a proxy event ────────────────

export function logProxyEvent(entry: ProxyLogInput) {
  const safeError =
    entry.error === null || entry.error === undefined || entry.error === ""
      ? null
      : sanitizeErrorMessage(entry.error) || "Proxy request failed";
  const log: ProxyLogEntry = {
    id: uuidv4(),
    timestamp: new Date().toISOString(),
    status: entry.status || "success",
    proxy: entry.proxy
      ? {
          ...entry.proxy,
          host: normalizeProxyHostForLog(entry.proxy.host) ?? entry.proxy.host,
        }
      : null,
    level: entry.level || "direct",
    levelId: entry.levelId || null,
    provider: entry.provider || null,
    targetUrl: entry.targetUrl || null,
    clientIp: entry.clientIp ?? entry.publicIp ?? null,
    egressIp: entry.egressIp ?? null,
    latencyMs: entry.latencyMs || 0,
    error: safeError,
    connectionId: entry.connectionId || null,
    comboId: entry.comboId || null,
    account: entry.account || null,
    rotationAccount: entry.rotationAccount || null,
    correlationId: entry.correlationId || null,
    tlsFingerprint: entry.tlsFingerprint || false,
    upstreamStatus: entry.upstreamStatus ?? null,
    attemptNumber:
      typeof entry.attemptNumber === "number" && Number.isInteger(entry.attemptNumber)
        ? entry.attemptNumber
        : null,
    attemptIssue:
      entry.attemptIssue === "served" || entry.attemptIssue === "abandoned"
        ? entry.attemptIssue
        : null,
    headersMs: sanitizeTimingMs(entry.headersMs),
    firstChunkMs: sanitizeTimingMs(entry.firstChunkMs),
  };

  // Structured egress line so the operator can confirm, in the proxy logs, which
  // IP each account is entering (clientIp) and leaving (egressIp) by.
  if (log.proxy || log.egressIp) {
    console.log(
      formatProxyEgressConsoleLine({
        provider: log.provider,
        account: log.account,
        clientIp: log.clientIp,
        egressIp: log.egressIp,
        level: log.level,
        proxyHost: log.proxy?.host,
        proxyName: log.proxy?.name,
        status: log.status,
        includeDetails: isProxyLogIncludeIps(),
      })
    );
  }

  // 1. In-memory ring buffer (newest first)
  proxyLogs.unshift(log);
  if (proxyLogs.length > MAX_IN_MEMORY_ENTRIES) {
    proxyLogs.length = MAX_IN_MEMORY_ENTRIES;
  }

  // 2. Queue for background batch persistence (SQLite / Redis)
  if (shouldPersistToDisk) {
    enqueueProxyLog(log);
  }

  return log;
}

// ──────────────── Background Batch Persistence ────────────────

const BATCH_FLUSH_INTERVAL_MS = 1000;
const BATCH_SIZE_THRESHOLD = 100;
// Memory cap for one deferred proxy batch: summed targetUrl + error lengths,
// no serialization on the failure path. A batch above the cap is dropped with
// the existing log line instead of being held for the retry.
const PROXY_RETRY_BATCH_BYTES = 256 * 1024;

let pendingLogsQueue: ProxyLogEntry[] = [];
let batchTimer: NodeJS.Timeout | null = null;
// Generation guard: clearProxyLogs() bumps the counter, and a deferred retry
// armed before the clear abandons its batch instead of re-inserting it.
let proxyLogGeneration = 0;

function proxyBatchSizeBytes(batch: ProxyLogEntry[]): number {
  let size = 0;
  for (const entry of batch) {
    size += (entry.targetUrl?.length ?? 0) + (entry.error?.length ?? 0);
  }
  return size;
}

function logProxyBatchFailure(err: unknown): void {
  console.warn(
    "[proxyLogger] Failed to write proxy log batch to disk:",
    sanitizeErrorMessage(err) || "Proxy log persistence failed"
  );
}

function writeProxyBatch(batch: ProxyLogEntry[], db: ReturnType<typeof getDbInstance>): void {
  const insertStmt = db.prepare(
    `INSERT INTO proxy_logs (id, timestamp, status, proxy_type, proxy_host, proxy_port, proxy_name,
      level, level_id, provider, target_url, public_ip, egress_ip, latency_ms, error,
      connection_id, combo_id, account, rotation_account, correlation_id, tls_fingerprint, upstream_status,
      attempt_number, attempt_issue, headers_ms, first_chunk_ms)
    VALUES (@id, @timestamp, @status, @proxyType, @proxyHost, @proxyPort, @proxyName,
      @level, @levelId, @provider, @targetUrl, @clientIp, @egressIp, @latencyMs, @error,
      @connectionId, @comboId, @account, @rotationAccount, @correlationId, @tlsFingerprint, @upstreamStatus,
      @attemptNumber, @attemptIssue, @headersMs, @firstChunkMs)`
  );

  const transaction = db.transaction((entries: ProxyLogEntry[]) => {
    for (const item of entries) {
      insertStmt.run({
        id: item.id,
        timestamp: item.timestamp,
        status: item.status,
        proxyType: item.proxy?.type || null,
        proxyHost: item.proxy?.host || null,
        proxyPort: item.proxy?.port ? Number(item.proxy.port) : null,
        proxyName: item.proxy?.name || null,
        level: item.level,
        levelId: item.levelId,
        provider: item.provider,
        targetUrl: item.targetUrl,
        clientIp: item.clientIp,
        egressIp: item.egressIp,
        latencyMs: item.latencyMs,
        error: item.error,
        connectionId: item.connectionId,
        comboId: item.comboId,
        account: item.account,
        rotationAccount: item.rotationAccount,
        correlationId: item.correlationId,
        tlsFingerprint: item.tlsFingerprint ? 1 : 0,
        upstreamStatus: item.upstreamStatus,
        attemptNumber: item.attemptNumber,
        attemptIssue: item.attemptIssue,
        headersMs: item.headersMs,
        firstChunkMs: item.firstChunkMs,
      });
    }
  });

  transaction(batch);
}

function ensureBatchTimer() {
  if (batchTimer) return;
  batchTimer = setInterval(() => {
    flushProxyLogsSync();
  }, BATCH_FLUSH_INTERVAL_MS);
  if (typeof batchTimer.unref === "function") {
    batchTimer.unref();
  }
}

function enqueueProxyLog(log: ProxyLogEntry) {
  pendingLogsQueue.push(log);
  ensureBatchTimer();
  if (pendingLogsQueue.length >= BATCH_SIZE_THRESHOLD) {
    flushProxyLogsSync();
  }
}

export function flushProxyLogsSync() {
  if (pendingLogsQueue.length === 0) return;
  const batch = pendingLogsQueue;
  pendingLogsQueue = [];

  // 1. If Redis driver is active, asynchronously publish batch to Redis Stream/Channel
  if (process.env.QUOTA_STORE_DRIVER === "redis" || process.env.QUOTA_STORE_REDIS_URL) {
    try {
      import("@/lib/quota/redisQuotaStore")
        .then(({ getRedisQuotaStore }) => {
          const store = getRedisQuotaStore(process.env.QUOTA_STORE_REDIS_URL || "");
          const client = (store as any)?.client;
          if (client && typeof client.publish === "function") {
            for (const entry of batch) {
              client.publish("omniroute:proxy_logs", JSON.stringify(entry)).catch(() => {});
            }
          }
        })
        .catch(() => {});
    } catch {
      /* ignore redis pub errors */
    }
  }

  // Keep the same database for both attempts; resets must not redirect old batches.
  let db: ReturnType<typeof getDbInstance> | undefined;
  try {
    db = getDbInstance();
    writeProxyBatch(batch, db);
  } catch (err: any) {
    if (!db || !isWriteLockError(err)) {
      logProxyBatchFailure(err);
      return;
    }
    if (proxyBatchSizeBytes(batch) > PROXY_RETRY_BATCH_BYTES) {
      logProxyBatchFailure(err);
      return;
    }
    const generation = proxyLogGeneration;
    const originalDb = db;
    void scheduleWriteRetry(() => {
      if (generation !== proxyLogGeneration) return;
      writeProxyBatch(batch, originalDb);
    }, logProxyBatchFailure);
  }
}

// ──────────────── Deferred timing patch ────────────────

// Bounded join key for late first-chunk arrivals: log id -> queued entry ref.
// Registered only when the first byte is still unknown at journal time; every
// entry leaves through exactly one path below (notify, settle-without-byte,
// cancel/error, cap eviction, clear). Cap mirrors the ring buffer so the
// registry never retains more than memory already does.
export const TIMING_LINK_CAP = 200;
const pendingFirstChunk = new Map<string, ProxyLogEntry>();

function evictOldestTimingLink(): void {
  const oldest = pendingFirstChunk.keys().next();
  if (!oldest.done) pendingFirstChunk.delete(oldest.value);
}

/**
 * Totest seam: current registry size (bounded by TIMING_LINK_CAP).
 */
export function pendingFirstChunkSizeForTests(): number {
  return pendingFirstChunk.size;
}

/**
 * Link a journaled row to its still-open upstream body. Called by the journal
 * layer right after logProxyEvent returns the entry, when the first byte has
 * not arrived yet. No-ops (no entry) when the timing is already known or when
 * there is no body to wait for.
 */
export function linkPendingFirstChunk(
  id: string,
  entry: ProxyLogEntry,
  timingKnown: boolean,
  hasBody: boolean
): void {
  if (timingKnown || !hasBody) return;
  if (pendingFirstChunk.size >= TIMING_LINK_CAP) evictOldestTimingLink();
  pendingFirstChunk.set(id, entry);
}

function dropPendingFirstChunk(id: string): void {
  pendingFirstChunk.delete(id);
}

/**
 * Settle a linked row once the first useful body byte arrives (or never does).
 * Before the batch flush the queued object is mutated in place so the INSERT
 * carries the value; after the flush the row is patched by id and the
 * in-memory copy is updated. Every path drops the registry entry.
 * The patch callback keeps this module decoupled from the db writer: the
 * journal/capture layer passes updateAttemptTiming from the owned db module.
 */
export function settlePendingFirstChunk(
  id: string,
  firstChunkMs: number | null,
  patchRow?: (id: string, patch: { firstChunkMs: number | null }) => boolean
): void {
  const entry = pendingFirstChunk.get(id);
  dropPendingFirstChunk(id);
  if (!entry) return;
  const clean = sanitizeTimingMs(firstChunkMs);
  entry.firstChunkMs = clean;
  if (!shouldPersistToDisk) return;
  if (pendingLogsQueue.includes(entry)) return;
  try {
    patchRow?.(id, { firstChunkMs: clean });
  } catch {
    // Deferred visibility is best-effort; the in-memory copy above stays correct.
  }
}

// ──────────────── Query ────────────────

/**
 * Get proxy logs with optional filters.
 * Reads from in-memory for speed (already hydrated from DB on startup).
 */
export function getProxyLogs(filters: ProxyLogFilters = {}) {
  let logs = [...proxyLogs];

  if (filters.status) {
    if (filters.status === "ok") {
      logs = logs.filter((l) => l.status === "success");
    } else {
      logs = logs.filter((l) => l.status === filters.status);
    }
  }

  if (filters.type) {
    logs = logs.filter((l) => l.proxy?.type === filters.type);
  }

  if (filters.provider) {
    logs = logs.filter((l) => l.provider === filters.provider);
  }

  if (filters.level) {
    logs = logs.filter((l) => l.level === filters.level);
  }

  if (filters.search) {
    const q = filters.search.toLowerCase();
    logs = logs.filter(
      (l) =>
        (l.proxy?.host || "").toLowerCase().includes(q) ||
        (l.proxy?.name || "").toLowerCase().includes(q) ||
        (l.provider || "").toLowerCase().includes(q) ||
        (l.targetUrl || "").toLowerCase().includes(q) ||
        (l.clientIp || "").toLowerCase().includes(q) ||
        (l.egressIp || "").toLowerCase().includes(q) ||
        (l.level || "").toLowerCase().includes(q) ||
        (l.error || "").toLowerCase().includes(q) ||
        (l.account || "").toLowerCase().includes(q)
    );
  }

  const limit = filters.limit || 300;
  return logs.slice(0, limit);
}

// ──────────────── Clear ────────────────

export function clearProxyLogs() {
  proxyLogs.length = 0;
  pendingLogsQueue = [];
  pendingFirstChunk.clear();
  // Invalidate deferred retries armed before this clear so they abandon
  // their batch instead of re-inserting cleared rows.
  proxyLogGeneration++;

  if (shouldPersistToDisk) {
    try {
      const db = getDbInstance();
      db.prepare("DELETE FROM proxy_logs").run();
    } catch (err: any) {
      console.warn(
        "[proxyLogger] Failed to clear DB:",
        sanitizeErrorMessage(err) || "Proxy log cleanup failed"
      );
    }
  }
}