Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

19.8 KB
Raw Blame
import type {
  PullRequestChecks,
  PullRequestMergeable,
  PullRequestState,
  TConversationPullRequest,
} from 'librechat-data-provider';
import type { PullRequestFetch } from './shared';
import type { PullRequestSource } from './types';
import {
  isCount,
  isRecord,
  COMMIT_ID,
  isPageUrl,
  retryAfterMs,
  isPathSegment,
  MAX_TITLE_LENGTH,
  DEFAULT_LOOKUP_TIMEOUT_MS,
  DEFAULT_REQUEST_TIMEOUT_MS,
  DEFAULT_MAX_CANDIDATE_PAGES,
  DEFAULT_MAX_HEAD_COMPARISONS,
  DEFAULT_MAX_CANDIDATE_PULL_REQUESTS,
} from './shared';
import { logDiscoveryDiagnostic } from './diagnostics';
import { isAllowedRepository } from './repository';
import { PullRequestSourceError } from './types';

export type { PullRequestFetch } from './shared';

export const DEFAULT_GITHUB_HOST = 'github.com';
const DEFAULT_MAX_CHECK_RUN_PAGES = 10;
const CHECK_RUN_PAGE_SIZE = 100;
/**
 * Comparing `recorded...candidate`: `ahead` means the candidate builds on the recorded commit and
 * `identical` that it is the recorded commit. `behind` means the candidate does not contain it
 * (the recorded commit is newer), and `diverged` that the histories split, so neither matches.
 */
const CONTAINS_RECORDED_STATUSES = new Set(['ahead', 'identical']);
const REPO_PATTERN = /^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/;
const FAILING_CONCLUSIONS = new Set([
  'failure',
  'timed_out',
  'cancelled',
  'action_required',
  'startup_failure',
  'stale',
]);

type GitHubPull = {
  number: number;
  title: string;
  url: string;
  state: 'open' | 'closed';
  merged: boolean;
  draft: boolean;
  additions: number;
  deletions: number;
  mergeable: boolean | null;
  /** The head this detail describes; the check runs are read for this same commit. */
  headSha: string;
  /** The repository that head lives in; a fork's checks run there, not in the base repository. */
  headRepo?: string;
  /** The branch the pull request merges from, in whichever repository it lives; null if unsaid. */
  headRef: string | null;
};

type GitHubCheckRun = { status: string; conclusion: string | null };

/**
 * The REST API of a GitHub host: `api.github.com` for github.com, `api.<host>` for a data residency
 * tenant on `ghe.com`, and `/api/v3` on the host itself for GitHub Enterprise Server.
 */
export function gitHubApiBase(host: string): string {
  if (host === DEFAULT_GITHUB_HOST) return 'https://api.github.com';
  if (host.endsWith('.ghe.com')) return `https://api.${host}`;
  return `https://${host}/api/v3`;
}

function parsePull(value: unknown, host: string): GitHubPull {
  if (
    !isRecord(value) ||
    !isCount(value.number) ||
    typeof value.title !== 'string' ||
    !isPageUrl(value.html_url, host) ||
    (value.state !== 'open' && value.state !== 'closed') ||
    typeof value.merged !== 'boolean' ||
    typeof value.draft !== 'boolean' ||
    !isCount(value.additions) ||
    !isCount(value.deletions) ||
    (value.mergeable !== null && typeof value.mergeable !== 'boolean') ||
    !isRecord(value.head) ||
    typeof value.head.sha !== 'string' ||
    !COMMIT_ID.test(value.head.sha)
  ) {
    throw new PullRequestSourceError('UPSTREAM_ERROR');
  }
  const headRepo =
    isRecord(value.head.repo) && typeof value.head.repo.full_name === 'string'
      ? value.head.repo.full_name
      : undefined;
  return {
    number: value.number,
    title: value.title,
    url: value.html_url,
    state: value.state,
    merged: value.merged,
    draft: value.draft,
    additions: value.additions,
    deletions: value.deletions,
    mergeable: value.mergeable,
    headSha: value.head.sha,
    ...(headRepo ? { headRepo } : {}),
    headRef: typeof value.head.ref === 'string' ? value.head.ref : null,
  };
}

type ListItem = { number: number; state: string; sha: string };

function parseListItem(value: unknown): ListItem {
  if (
    !isRecord(value) ||
    !isCount(value.number) ||
    typeof value.state !== 'string' ||
    !isRecord(value.head) ||
    typeof value.head.sha !== 'string' ||
    !/^[a-f0-9]{40,64}$/.test(value.head.sha)
  ) {
    throw new PullRequestSourceError('UPSTREAM_ERROR');
  }
  return { number: value.number, state: value.state, sha: value.head.sha };
}

type CheckRunPage = { runs: GitHubCheckRun[]; total?: number };

function parseCheckRuns(value: unknown): CheckRunPage {
  if (!isRecord(value) || !Array.isArray(value.check_runs)) {
    throw new PullRequestSourceError('UPSTREAM_ERROR');
  }
  const runs = value.check_runs.map((run) => {
    if (!isRecord(run) || typeof run.status !== 'string') {
      throw new PullRequestSourceError('UPSTREAM_ERROR');
    }
    return {
      status: run.status,
      conclusion: typeof run.conclusion === 'string' ? run.conclusion : null,
    };
  });
  return { runs, ...(isCount(value.total_count) ? { total: value.total_count } : {}) };
}

/**
 * A failed check outranks one still running, so the header turns red as soon as it is known.
 * An `incomplete` rollup (more runs than were read) is never reported as passing.
 */
export function summarizeChecks(
  runs: readonly GitHubCheckRun[],
  incomplete = false,
): PullRequestChecks {
  if (runs.length === 0 && !incomplete) return 'none';
  if (runs.some((run) => run.conclusion != null && FAILING_CONCLUSIONS.has(run.conclusion))) {
    return 'failing';
  }
  if (incomplete || runs.some((run) => run.status !== 'completed')) return 'running';
  return 'passing';
}

function summarizeState(pull: GitHubPull): PullRequestState {
  return pull.merged ? 'merged' : pull.state;
}

/** Conflict state only means something while the pull request is open. */
function summarizeMergeable(pull: GitHubPull): PullRequestMergeable {
  if (summarizeState(pull) !== 'open' || pull.mergeable == null) return 'unknown';
  return pull.mergeable ? 'clean' : 'conflicting';
}

export function toConversationPullRequest(
  pull: GitHubPull,
  runs: readonly GitHubCheckRun[],
  incomplete = false,
): TConversationPullRequest {
  return {
    number: pull.number,
    title: pull.title.slice(0, MAX_TITLE_LENGTH),
    url: pull.url,
    provider: 'github',
    additions: pull.additions,
    deletions: pull.deletions,
    state: summarizeState(pull),
    isDraft: pull.draft,
    mergeable: summarizeMergeable(pull),
    checks: summarizeChecks(runs, incomplete),
  };
}

/** GitHub asks for at least a minute when a secondary limit names no wait. */
const SECONDARY_LIMIT_WAIT_MS = 60_000;
const SECONDARY_LIMIT_MESSAGE = /secondary rate limit|abuse detection/i;
const MAX_ERROR_BODY_CHARS = 2_000;

function isRateLimited(response: Response): boolean {
  return (
    response.status === 429 ||
    (response.status === 403 &&
      (response.headers.get('x-ratelimit-remaining') === '0' ||
        response.headers.get('retry-after') != null))
  );
}

/**
 * A secondary limit can arrive as a 403 with neither limiting header, told apart only by its
 * message. The body is read bounded and only matched, never stored or returned.
 */
async function isSecondaryLimit(response: Response): Promise<boolean> {
  if (response.status !== 403 && response.status !== 429) return false;
  try {
    const text = (await response.text()).slice(0, MAX_ERROR_BODY_CHARS);
    return SECONDARY_LIMIT_MESSAGE.test(text);
  } catch {
    return false;
  }
}

/**
 * Reads pull requests through GitHub's REST API with a token the caller supplies. Only the
 * decision (absent, rate limited, failed) leaves this module; response bodies and error text
 * never do.
 */
export function createGitHubPullRequestSource({
  fetchFn,
}: {
  /** The deployment's HTTP client, so proxying and instrumentation are decided by the caller. */
  fetchFn: PullRequestFetch;
}): PullRequestSource {
  /** One lookup's credential and deadlines, shared by every request it makes. */
  type Lookup = {
    token: string;
    /** The host's REST API root; every request of the lookup goes there and nowhere else. */
    apiBase: string;
    requestTimeoutMs: number;
    /** Aborts when the whole lookup runs out of time. */
    deadline: AbortSignal;
    maxCheckRunPages: number;
    maxCandidatePullRequests: number;
    maxHeadComparisons: number;
    maxCandidatePages: number;
  };

  async function getJson(
    pathname: string,
    lookup: Lookup,
    /** Statuses that mean "nothing there" for this request, besides 404. */
    absent: readonly number[] = [],
    /** Whether a 404 means "nothing there". Check runs for a commit that was just read are not. */
    notFoundIsAbsent = true,
  ): Promise<unknown | null> {
    let response: Response;
    try {
      response = await fetchFn(`${lookup.apiBase}${pathname}`, {
        headers: {
          Accept: 'application/vnd.github+json',
          Authorization: `Bearer ${lookup.token}`,
          'X-GitHub-Api-Version': '2022-11-28',
          'User-Agent': 'LibreChat-Pull-Requests',
        },
        signal: AbortSignal.any([AbortSignal.timeout(lookup.requestTimeoutMs), lookup.deadline]),
      });
    } catch {
      throw new PullRequestSourceError('UPSTREAM_ERROR');
    }
    if (response.ok) {
      try {
        return await response.json();
      } catch {
        throw new PullRequestSourceError('UPSTREAM_ERROR');
      }
    }
    if (isRateLimited(response)) {
      const hinted = retryAfterMs(response, 'x-ratelimit-reset');
      /** A 429 that names no wait may still be a secondary limit, told apart by its message. */
      const wait =
        hinted == null && (await isSecondaryLimit(response)) ? SECONDARY_LIMIT_WAIT_MS : hinted;
      throw new PullRequestSourceError('RATE_LIMITED', wait);
    }
    if (await isSecondaryLimit(response)) {
      throw new PullRequestSourceError('RATE_LIMITED', SECONDARY_LIMIT_WAIT_MS);
    }
    /** A repository the token cannot see is indistinguishable from one without pull requests. */
    if ((notFoundIsAbsent && response.status === 404) || absent.includes(response.status)) {
      return null;
    }
    /** The provider no longer accepts the token: revoked, or expired without a refresh. */
    if (response.status === 401) throw new PullRequestSourceError('CREDENTIAL_REJECTED');
    throw new PullRequestSourceError('UPSTREAM_ERROR');
  }

  /** Reads every page up to a bound, so a failure on a later page still shows. */
  async function readCheckRuns(
    pathname: string,
    lookup: Lookup,
  ): Promise<{ runs: GitHubCheckRun[]; incomplete: boolean }> {
    const runs: GitHubCheckRun[] = [];
    for (let page = 1; page <= lookup.maxCheckRunPages; page++) {
      /** The head was read a moment ago, so a 404 here is a head that vanished, not no checks. */
      const body = await getJson(
        `${pathname}?per_page=${CHECK_RUN_PAGE_SIZE}&page=${page}`,
        lookup,
        [],
        false,
      );
      if (body == null) throw new PullRequestSourceError('UPSTREAM_ERROR');
      const parsed = parseCheckRuns(body);
      runs.push(...parsed.runs);
      const done =
        parsed.total != null
          ? runs.length >= parsed.total || parsed.runs.length === 0
          : parsed.runs.length < CHECK_RUN_PAGE_SIZE;
      if (done) return { runs, incomplete: false };
    }
    return { runs, incomplete: true };
  }

  return {
    async find({
      repo,
      branch,
      head: recordedHead,
      pullRequest,
      host = DEFAULT_GITHUB_HOST,
      token,
      limits,
      allowedRepositories,
    }) {
      /**
       * A lane with a branch and no commit yet has nothing to match a pull request against, and
       * every candidate would be accepted, including an earlier incarnation of a reused branch
       * name. `undefined` is a caller that never recorded a head, which keeps matching by branch.
       */
      if (recordedHead === null) return null;
      const lookup: Lookup = {
        token,
        apiBase: gitHubApiBase(host),
        requestTimeoutMs: limits?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
        deadline: AbortSignal.timeout(limits?.lookupTimeoutMs ?? DEFAULT_LOOKUP_TIMEOUT_MS),
        maxCheckRunPages: limits?.maxCheckRunPages ?? DEFAULT_MAX_CHECK_RUN_PAGES,
        maxCandidatePullRequests:
          limits?.maxCandidatePullRequests ?? DEFAULT_MAX_CANDIDATE_PULL_REQUESTS,
        maxHeadComparisons: limits?.maxHeadComparisons ?? DEFAULT_MAX_HEAD_COMPARISONS,
        maxCandidatePages: limits?.maxCandidatePages ?? DEFAULT_MAX_CANDIDATE_PAGES,
      };
      const match = REPO_PATTERN.exec(repo);
      if (match == null || branch.length === 0) return null;
      const [, owner, name] = match;
      if (!isPathSegment(owner) || !isPathSegment(name)) return null;
      const base = `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}`;
      /**
       * A fork's checks run in the fork. Its repository is read only when an administrator named
       * it, so the server's token is never used against a repository nobody authorized.
       */
      const checksBaseOf = (pull: GitHubPull): string => {
        const headRepo = pull.headRepo;
        if (headRepo == null || headRepo.toLowerCase() === `${owner}/${name}`.toLowerCase()) {
          return base;
        }
        const parts = REPO_PATTERN.exec(headRepo);
        if (parts == null || !isPathSegment(parts[1]) || !isPathSegment(parts[2])) return base;
        if (!isAllowedRepository(headRepo, allowedRepositories)) return base;
        return `/repos/${encodeURIComponent(parts[1])}/${encodeURIComponent(parts[2])}`;
      };
      const report = async (pull: GitHubPull): Promise<TConversationPullRequest> => {
        const checks = await readCheckRuns(
          `${checksBaseOf(pull)}/commits/${pull.headSha}/check-runs`,
          lookup,
        );
        return toConversationPullRequest(pull, checks.runs, checks.incomplete);
      };
      const headFilter = encodeURIComponent(`${owner}:${branch}`);
      /** `undefined`: the repository is not visible. An empty list: it is, and nothing matched. */
      const listPulls = async (state: 'open' | 'closed', perPage: number, page = 1) => {
        const listed = await getJson(
          `${base}/pulls?state=${state}&sort=updated&direction=desc&per_page=${perPage}&page=${page}&head=${headFilter}`,
          lookup,
        );
        if (listed == null) return undefined;
        if (!Array.isArray(listed)) throw new PullRequestSourceError('UPSTREAM_ERROR');
        return listed.map(parseListItem);
      };
      const recorded =
        typeof recordedHead === 'string' && COMMIT_ID.test(recordedHead) ? recordedHead : undefined;
      let comparisons = 0;
      /**
       * A branch name can be deleted and reused, so with a recorded commit a pull request counts
       * only when its head is that commit or builds on it. A commit GitHub no longer knows is no
       * match, and both the candidates listed and the comparisons made are bounded by config.
       */
      /** Every comparison, the first match and the revalidation of a moved head alike, spends the
       *  same configured budget. */
      const contains = async (headSha: string): Promise<boolean> => {
        if (recorded == null || headSha === recorded) return true;
        if (comparisons >= lookup.maxHeadComparisons) {
          logDiscoveryDiagnostic('github', 'COMPARISON_LIMIT');
          return false;
        }
        comparisons += 1;
        const comparison = await getJson(`${base}/compare/${recorded}...${headSha}`, lookup);
        if (comparison == null) {
          logDiscoveryDiagnostic('github', 'COMPARISON_UNAVAILABLE');
          return false;
        }
        if (!isRecord(comparison) || typeof comparison.status !== 'string') {
          throw new PullRequestSourceError('UPSTREAM_ERROR');
        }
        return CONTAINS_RECORDED_STATUSES.has(comparison.status);
      };
      const matches = (candidate: ListItem): Promise<boolean> => contains(candidate.sha);
      const choose = async (candidates: ListItem[]): Promise<ListItem | null> => {
        for (const candidate of candidates) {
          if (await matches(candidate)) return candidate;
        }
        return null;
      };
      /**
       * Reads pages of one state until a short page, the configured page budget, or (without a
       * recorded commit, where only the newest matters) the first page. `undefined` is a
       * repository the token cannot see.
       */
      const listAll = async (state: 'open' | 'closed'): Promise<ListItem[] | undefined> => {
        const perPage =
          state === 'closed' && recorded == null ? 1 : lookup.maxCandidatePullRequests;
        const pages = state === 'closed' && recorded == null ? 1 : lookup.maxCandidatePages;
        const all: ListItem[] = [];
        for (let page = 1; page <= pages; page++) {
          const items = await listPulls(state, perPage, page);
          if (items === undefined) return page === 1 ? undefined : all;
          all.push(...items);
          if (items.length < perPage) break;
        }
        return all;
      };
      /**
       * A pull request the conversation's own output linked to is read directly, so a lane with
       * commits not pushed yet still shows it. Its head must be the lane's branch in the lane's
       * repository. A branch name alone proves nothing for a fork (anyone can open `patch-1`), so
       * a fork's pull request counts only when it carries the recorded commit, as in the search.
       */
      if (pullRequest != null && Number.isSafeInteger(pullRequest) && pullRequest > 0) {
        const linked = await getJson(`${base}/pulls/${pullRequest}`, lookup);
        if (linked != null) {
          const pull = parsePull(linked, host);
          const sameRepository = pull.headRepo?.toLowerCase() === repo.toLowerCase();
          if (
            pull.headRef === branch &&
            (sameRepository || (recorded != null && (await contains(pull.headSha))))
          ) {
            return report(pull);
          }
        }
      }
      /** Open first on its own, so closed history on a reused branch name cannot hide it. */
      const open = await listAll('open');
      if (open === undefined) return null;
      /**
       * The head filter names the repository's own owner, so a pull request from a fork is not in
       * these lists. The recorded commit finds it instead: GitHub lists the pull requests that
       * carry a commit whatever fork they come from, and the branch name narrows them.
       */
      let forks: Promise<ListItem[]> | undefined;
      const forkCandidates = (): Promise<ListItem[]> => (forks ??= findForkCandidates());
      const findForkCandidates = async (): Promise<ListItem[]> => {
        if (recorded == null) return [];
        const found = await getJson(
          `${base}/commits/${recorded}/pulls?per_page=${lookup.maxCandidatePullRequests}`,
          lookup,
          [422],
        );
        if (found == null) return [];
        if (!Array.isArray(found)) throw new PullRequestSourceError('UPSTREAM_ERROR');
        return found
          .filter((item) => isRecord(item) && isRecord(item.head) && item.head.ref === branch)
          .map(parseListItem);
      };
      /**
       * Open pull requests win over closed ones whichever repository they come from, so a closed
       * pull request in the base repository cannot hide an open one from a fork.
       */
      const chosen =
        (await choose(open)) ??
        (await choose((await forkCandidates()).filter((item) => item.state === 'open'))) ??
        (await choose((await listAll('closed')) ?? [])) ??
        (await choose((await forkCandidates()).filter((item) => item.state !== 'open')));
      if (chosen == null) return null;

      /**
       * The detail comes first and its head decides which commit's checks are read. Listing,
       * detail and checks are separate requests, so a push between them could otherwise pair
       * today's line counts and mergeability with yesterday's checks.
       */
      const detail = await getJson(`${base}/pulls/${chosen.number}`, lookup);
      if (detail == null) return null;
      const pull = parsePull(detail, host);
      /** A push between the list and the detail can move the head off the recorded commit. */
      if (pull.headSha !== chosen.sha && !(await contains(pull.headSha))) return null;
      return report(pull);
    },
  };
}