Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

fix(github-org-sync): shrink GraphQL page size and paginate team members to avoid GitHub 502s

Devin AI5db949fc6608
diff --git a/backend/src/ee/services/github-org-sync/github-org-sync-service.test.ts b/backend/src/ee/services/github-org-sync/github-org-sync-service.test.ts
new file mode 100644
index 0000000000..0ac7e07820
--- /dev/null
+++ b/backend/src/ee/services/github-org-sync/github-org-sync-service.test.ts
@@ -0,0 +1,78 @@
+import { Octokit } from "@octokit/core";
+import { describe, expect, test, vi } from "vitest";
+
+import { fetchGithubOrgTeams } from "./github-org-sync-service";
+
+type TVariables = { cursor: string | null; slug?: string; teamsPageSize?: number; membersPageSize?: number };
+
+const page = <T>(nodes: T[], endCursor: string | null) => ({
+ edges: nodes.map((node) => ({ node })),
+ pageInfo: { hasNextPage: endCursor !== null, endCursor }
+});
+
+const team = (slug: string, members: string[], membersCursor: string | null = null) => ({
+ slug,
+ name: slug.toUpperCase(),
+ description: null,
+ members: page(
+ members.map((login) => ({ login })),
+ membersCursor
+ )
+});
+
+const buildOctokit = (handler: (query: string, variables: TVariables) => unknown) => {
+ const graphql = vi.fn(async (query: string, variables: TVariables) => handler(query, variables));
+ return { octokit: { graphql } as unknown as Pick<Octokit, "graphql">, graphql };
+};
+
+describe("fetchGithubOrgTeams", () => {
+ test("follows the team cursor across pages and flattens members", async () => {
+ const { octokit, graphql } = buildOctokit((_query, { cursor }) => {
+ if (cursor === null) {
+ return { organization: { teams: page([team("a", ["u1", "u2"])], "c1") } };
+ }
+ return { organization: { teams: page([team("b", ["u3"])], null) } };
+ });
+
+ const teams = await fetchGithubOrgTeams(octokit, "acme");
+
+ expect(teams).toEqual([
+ { slug: "a", name: "A", description: null, members: ["u1", "u2"] },
+ { slug: "b", name: "B", description: null, members: ["u3"] }
+ ]);
+ expect(graphql).toHaveBeenCalledTimes(2);
+ expect(graphql.mock.calls[1][1]).toMatchObject({ cursor: "c1", org: "acme" });
+ });
+
+ test("paginates members separately for teams whose members exceed one page", async () => {
+ const { octokit, graphql } = buildOctokit((query, { cursor, slug }) => {
+ if (query.includes("query orgTeams")) {
+ return {
+ organization: {
+ teams: page([team("big", ["u1"], "m1"), team("small", ["u9"])], null)
+ }
+ };
+ }
+ expect(slug).toBe("big");
+ if (cursor === "m1") {
+ return { organization: { team: { members: page([{ login: "u2" }], "m2") } } };
+ }
+ return { organization: { team: { members: page([{ login: "u3" }], null) } } };
+ });
+
+ const teams = await fetchGithubOrgTeams(octokit, "acme");
+
+ expect(teams.find((t) => t.slug === "big")?.members).toEqual(["u1", "u2", "u3"]);
+ expect(teams.find((t) => t.slug === "small")?.members).toEqual(["u9"]);
+ expect(graphql).toHaveBeenCalledTimes(3);
+ });
+
+ test("passes a per-request abort signal", async () => {
+ const { octokit, graphql } = buildOctokit(() => ({ organization: { teams: page([], null) } }));
+
+ await fetchGithubOrgTeams(octokit, "acme");
+
+ const options = graphql.mock.calls[0][1] as unknown as { request: { signal: AbortSignal } };
+ expect(options.request.signal).toBeInstanceOf(AbortSignal);
+ });
+});
diff --git a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts
index 5865f66a50..187414bc35 100644
--- a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts
+++ b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts
@@ -37,6 +37,167 @@ import {
const OctokitWithPlugin = Octokit.plugin(paginateGraphql);
+// GitHub's GraphQL API times out (502) when one page resolves too many nodes; nesting members
+// under teams multiplies the per-page node count, so the team page is kept small.
+const GITHUB_TEAMS_PAGE_SIZE = 20;
+const GITHUB_TEAM_MEMBERS_PAGE_SIZE = 100;
+const GITHUB_MAX_PAGES = 500;
+const GITHUB_REQUEST_TIMEOUT_MS = 30_000;
+
+type TGithubPageInfo = { hasNextPage: boolean; endCursor: string | null };
+
+type TGithubTeamMembersConnection = {
+ edges: { node: { login: string } }[];
+ pageInfo: TGithubPageInfo;
+};
+
+type TGithubOrgTeamsResponse = {
+ organization: {
+ teams: {
+ edges: {
+ node: {
+ slug: string;
+ name: string;
+ description: string | null;
+ members: TGithubTeamMembersConnection;
+ };
+ }[];
+ pageInfo: TGithubPageInfo;
+ };
+ };
+};
+
+type TGithubTeamMembersResponse = {
+ organization: {
+ team: { members: TGithubTeamMembersConnection } | null;
+ };
+};
+
+export type TGithubTeam = {
+ slug: string;
+ name: string;
+ description: string | null;
+ members: string[];
+};
+
+const ORG_TEAMS_QUERY = `
+ query orgTeams($org: String!, $cursor: String, $teamsPageSize: Int!, $membersPageSize: Int!) {
+ organization(login: $org) {
+ teams(first: $teamsPageSize, after: $cursor) {
+ edges {
+ node {
+ slug
+ name
+ description
+ members(first: $membersPageSize) {
+ edges {
+ node {
+ login
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ }
+`;
+
+const TEAM_MEMBERS_QUERY = `
+ query teamMembers($org: String!, $slug: String!, $cursor: String, $membersPageSize: Int!) {
+ organization(login: $org) {
+ team(slug: $slug) {
+ members(first: $membersPageSize, after: $cursor) {
+ edges {
+ node {
+ login
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ }
+ }
+`;
+
+const nextCursor = (pageInfo: TGithubPageInfo) => (pageInfo.hasNextPage ? pageInfo.endCursor : null);
+
+export const fetchGithubOrgTeams = async (octokit: Pick<Octokit, "graphql">, org: string): Promise<TGithubTeam[]> => {
+ const graphql = <T>(query: string, variables: Record<string, string | number | null>) =>
+ retryWithBackoff(() =>
+ octokit.graphql<T>(query, {
+ ...variables,
+ request: { signal: AbortSignal.timeout(GITHUB_REQUEST_TIMEOUT_MS) }
+ })
+ );
+
+ const teams: (TGithubTeam & { membersCursor: string | null })[] = [];
+
+ let teamsCursor: string | null = null;
+ let teamsPage = 0;
+ do {
+ const data: TGithubOrgTeamsResponse = await graphql<TGithubOrgTeamsResponse>(ORG_TEAMS_QUERY, {
+ org,
+ cursor: teamsCursor,
+ teamsPageSize: GITHUB_TEAMS_PAGE_SIZE,
+ membersPageSize: GITHUB_TEAM_MEMBERS_PAGE_SIZE
+ });
+ const connection = data.organization.teams;
+ connection.edges.forEach(({ node }) => {
+ teams.push({
+ slug: node.slug,
+ name: node.name,
+ description: node.description,
+ members: node.members.edges.map((edge) => edge.node.login),
+ membersCursor: nextCursor(node.members.pageInfo)
+ });
+ });
+ teamsCursor = nextCursor(connection.pageInfo);
+ teamsPage += 1;
+ } while (teamsCursor && teamsPage < GITHUB_MAX_PAGES);
+
+ if (teamsCursor) {
+ logger.warn({ org, fetchedTeams: teams.length }, "GitHub org team sync hit the page cap; team list truncated");
+ }
+
+ for (const team of teams) {
+ let membersPage = 0;
+ while (team.membersCursor && membersPage < GITHUB_MAX_PAGES) {
+ const data: TGithubTeamMembersResponse = await graphql<TGithubTeamMembersResponse>(TEAM_MEMBERS_QUERY, {
+ org,
+ slug: team.slug,
+ cursor: team.membersCursor,
+ membersPageSize: GITHUB_TEAM_MEMBERS_PAGE_SIZE
+ });
+ const connection = data.organization.team?.members;
+ if (!connection) break;
+ team.members.push(...connection.edges.map((edge) => edge.node.login));
+ team.membersCursor = nextCursor(connection.pageInfo);
+ membersPage += 1;
+ }
+
+ if (team.membersCursor) {
+ logger.warn(
+ { org, team: team.slug, fetchedMembers: team.members.length },
+ "GitHub org team sync hit the page cap; team member list truncated"
+ );
+ }
+ }
+
+ return teams.map(({ slug, name, description, members }) => ({ slug, name, description, members }));
+};
+
// Type definitions for GitHub API errors
interface GitHubApiError extends Error {
status?: number;
@@ -594,121 +755,53 @@ export const githubOrgSyncServiceFactory = ({
const startTime = Date.now();
const syncErrors: string[] = [];
- const octokit = new OctokitWithPlugin({
- auth: orgAccessToken,
- request: {
- signal: AbortSignal.timeout(30000)
- }
- });
+ const octokit = new Octokit({ auth: orgAccessToken });
- const data = await retryWithBackoff(async () => {
- return octokit.graphql
- .paginate<{
- organization: {
- teams: {
- totalCount: number;
- edges: {
- node: {
- name: string;
- description: string;
- members: {
- edges: {
- node: {
- login: string;
- };
- }[];
- };
- };
- }[];
- };
- };
- }>(
- `
- query orgTeams($cursor: String, $org: String!) {
- organization(login: $org) {
- teams(first: 100, after: $cursor) {
- totalCount
- edges {
- node {
- name
- description
- members(first: 100) {
- edges {
- node {
- login
- }
- }
- }
- }
- }
- pageInfo {
- hasNextPage
- endCursor
- }
- }
- }
+ const githubTeams = await fetchGithubOrgTeams(octokit, config.githubOrgName).catch((err) => {
+ logger.error(err, "GitHub GraphQL error for batched team sync");
+
+ const gitHubError = err as GitHubApiError;
+ const statusCode = gitHubError.status || gitHubError.response?.status;
+ if (statusCode) {
+ if (statusCode === 401) {
+ throw new BadRequestError({
+ message: "GitHub access token is invalid or expired. Please provide a new token."
+ });
}
- `,
- {
- org: config.githubOrgName
- }
- )
- .catch((err) => {
- logger.error(err, "GitHub GraphQL error for batched team sync");
-
- const gitHubError = err as GitHubApiError;
- const statusCode = gitHubError.status || gitHubError.response?.status;
- if (statusCode) {
- if (statusCode === 401) {
- throw new BadRequestError({
- message: "GitHub access token is invalid or expired. Please provide a new token."
- });
- }
- if (statusCode === 403) {
- throw new BadRequestError({
- message:
- "GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role."
- });
- }
- if (statusCode === 404) {
- throw new BadRequestError({
- message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.`
- });
- }
- }
+ if (statusCode === 403) {
+ throw new BadRequestError({
+ message:
+ "GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role."
+ });
+ }
+ if (statusCode === 404) {
+ throw new BadRequestError({
+ message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.`
+ });
+ }
+ if (statusCode >= 500) {
+ throw new BadRequestError({
+ message: `GitHub did not respond in time while listing teams for organization ${config.githubOrgName}. Please try again later.`
+ });
+ }
+ }
- if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
- throw new BadRequestError({
- message:
- "Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions."
- });
- }
- throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` });
+ if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
+ throw new BadRequestError({
+ message:
+ "Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions."
});
+ }
+ throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` });
});
- const {
- organization: { teams }
- } = data;
-
- const userTeamMap = new Map<string, string[]>();
- const allGithubUsernamesInTeams = new Set<string>();
-
- teams?.edges?.forEach((teamEdge) => {
- const teamName = teamEdge.node.name.toLowerCase();
-
- teamEdge.node.members.edges.forEach((memberEdge) => {
- const username = memberEdge.node.login.toLowerCase();
- allGithubUsernamesInTeams.add(username);
-
- if (!userTeamMap.has(username)) {
- userTeamMap.set(username, []);
- }
- userTeamMap.get(username)!.push(teamName);
- });
+ const githubTeamMembersByName = new Map<string, string[]>();
+ githubTeams.forEach((team) => {
+ const teamName = team.name.toLowerCase();
+ githubTeamMembersByName.set(teamName, [...(githubTeamMembersByName.get(teamName) ?? []), ...team.members]);
});
- const allGithubTeamNames = Array.from(new Set(teams?.edges?.map((edge) => edge.node.name.toLowerCase()) || []));
+ const allGithubTeamNames = Array.from(githubTeamMembersByName.keys());
const existingTeamsOnInfisical = await groupDAL.find({
orgId: orgPermission.orgId,
@@ -770,44 +863,40 @@ export const githubOrgSyncServiceFactory = ({
)) as GroupMembership[];
const expectedUserIds = new Set<string>();
- teams?.edges?.forEach((teamEdge) => {
- if (teamEdge.node.name.toLowerCase() === teamName) {
- teamEdge.node.members.edges.forEach((memberEdge) => {
- const githubUsername = memberEdge.node.login.toLowerCase();
+ (githubTeamMembersByName.get(teamName) ?? []).forEach((login) => {
+ const githubUsername = login.toLowerCase();
- const matchingMember = activeMembers.find((member) => {
- const email = member.user?.email || member.inviteEmail;
- if (!email) return false;
+ const matchingMember = activeMembers.find((member) => {
+ const email = member.user?.email || member.inviteEmail;
+ if (!email) return false;
- const emailPrefix = email.split("@")[0].toLowerCase();
- const emailDomain = email.split("@")[1].toLowerCase();
+ const emailPrefix = email.split("@")[0].toLowerCase();
+ const emailDomain = email.split("@")[1].toLowerCase();
- if (emailPrefix === githubUsername) {
- return true;
- }
- const domainName = emailDomain.split(".")[0];
- if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) {
- const baseUsername = githubUsername.slice(0, -domainName.length);
- if (emailPrefix === baseUsername) {
- return true;
- }
- }
- const emailSplitRegex = new RE2(/[._-]/);
- const emailParts = emailPrefix.split(emailSplitRegex);
- const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), "");
- if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) {
- return true;
- }
- return false;
- });
-
- if (matchingMember?.user?.id) {
- expectedUserIds.add(matchingMember.user.id);
- logger.info(
- `Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}`
- );
+ if (emailPrefix === githubUsername) {
+ return true;
+ }
+ const domainName = emailDomain.split(".")[0];
+ if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) {
+ const baseUsername = githubUsername.slice(0, -domainName.length);
+ if (emailPrefix === baseUsername) {
+ return true;
}
- });
+ }
+ const emailSplitRegex = new RE2(/[._-]/);
+ const emailParts = emailPrefix.split(emailSplitRegex);
+ const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), "");
+ if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) {
+ return true;
+ }
+ return false;
+ });
+
+ if (matchingMember?.user?.id) {
+ expectedUserIds.add(matchingMember.user.id);
+ logger.info(
+ `Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}`
+ );
}
});