Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

retrieve file content for large diffs

Matheus Nogueiraa296abbf3589
diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.test.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.test.ts
index 277f720f2e..805679756e 100644
--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.test.ts
+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.test.ts
@@ -33,7 +33,8 @@ vi.mock("@app/lib/config/env", async (importOriginal) => ({
...(await importOriginal<typeof import("@app/lib/config/env")>()),
getConfig: () => ({
INF_APP_CONNECTION_GITHUB_RADAR_APP_ID: "1",
- INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY: "key"
+ INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY: "key",
+ SECRET_SCANNING_SCAN_TIMEOUT: 60_000
})
}));
@@ -241,6 +242,126 @@ describe("scanGitHubCommitsAndGetFindings", () => {
expect(getCommit).toHaveBeenCalledTimes(30);
expect(scanContentAndGetFindings).not.toHaveBeenCalled();
});
+
+ describe("a file whose patch GitHub left out", () => {
+ const parentSha = "p".repeat(40);
+ const toContent = (text: string) => ({
+ data: {
+ type: "file",
+ size: text.length,
+ sha: "s",
+ encoding: "base64",
+ content: Buffer.from(text).toString("base64")
+ }
+ });
+ const asOctokitWithContents = (
+ files: unknown[],
+ getContent: unknown,
+ getBlob: unknown = vi.fn()
+ ): Parameters<typeof scanGitHubCommitsAndGetFindings>[0]["octokit"] =>
+ ({
+ repos: { getCommit: vi.fn().mockResolvedValue({ data: { files, parents: [{ sha: parentSha }] } }), getContent },
+ git: { getBlob }
+ }) as unknown as Parameters<typeof scanGitHubCommitsAndGetFindings>[0]["octokit"];
+ const scannedLines = () =>
+ scanContentAndGetFindings.mock.calls.map(([content]) => content.split("\n").filter(Boolean));
+
+ test("scans every line of an added file", async () => {
+ const getContent = vi.fn().mockResolvedValue(toContent("one\ntwo\n"));
+
+ await scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents([{ status: "added", filename: "big.txt", additions: 2 }], getContent),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit]
+ });
+
+ expect(getContent).toHaveBeenCalledTimes(1);
+ expect(getContent).toHaveBeenCalledWith(expect.objectContaining({ path: "big.txt", ref: commit.sha }));
+ expect(scannedLines()).toEqual([["one", "two"]]);
+ });
+
+ test("scans only the lines a modified or renamed file adds", async () => {
+ const getContent = vi
+ .fn()
+ .mockImplementation(({ ref }: { ref: string }) =>
+ Promise.resolve(toContent(ref === parentSha ? "kept\nold\n" : "kept\nnew\n"))
+ );
+
+ await scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents(
+ [{ status: "renamed", filename: "new.txt", previous_filename: "old.txt", additions: 1 }],
+ getContent
+ ),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit]
+ });
+
+ expect(getContent).toHaveBeenCalledWith(expect.objectContaining({ path: "old.txt", ref: parentSha }));
+ expect(scannedLines()).toEqual([["new"]]);
+ });
+
+ test("reads a file over 1 MB from the blob API", async () => {
+ const getContent = vi
+ .fn()
+ .mockResolvedValue({ data: { type: "file", size: 2_000_000, sha: "blob", content: "" } });
+ const getBlob = vi.fn().mockResolvedValue({ data: { content: Buffer.from("large\n").toString("base64") } });
+
+ await scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents([{ status: "added", filename: "big.txt", additions: 1 }], getContent, getBlob),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit]
+ });
+
+ expect(getBlob).toHaveBeenCalledWith(expect.objectContaining({ file_sha: "blob" }));
+ expect(scannedLines()).toEqual([["large"]]);
+ });
+
+ test("skips a binary file, which adds no lines", async () => {
+ const getContent = vi.fn();
+
+ await scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents([{ status: "added", filename: "image.png", additions: 0 }], getContent),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit],
+ requireCompleteCommits: true
+ });
+
+ expect(getContent).not.toHaveBeenCalled();
+ expect(scanContentAndGetFindings).not.toHaveBeenCalled();
+ });
+
+ test("fails a scan that requires every file when the file is too large to fetch, and skips it otherwise", async () => {
+ const getContent = vi
+ .fn()
+ .mockResolvedValue({ data: { type: "file", size: 50_000_000, sha: "blob", content: "" } });
+ const files = [{ status: "added", filename: "huge.txt", additions: 1 }];
+
+ await expect(
+ scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents(files, getContent),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit],
+ requireCompleteCommits: true
+ })
+ ).rejects.toThrow("huge.txt is too large for GitHub to include, and the file couldn't be scanned in full");
+
+ await expect(
+ scanGitHubCommitsAndGetFindings({
+ octokit: asOctokitWithContents(files, getContent),
+ repositoryFullName: "acme/api",
+ resourceName: "acme/api",
+ commits: [commit]
+ })
+ ).resolves.toEqual([]);
+
+ expect(scanContentAndGetFindings).not.toHaveBeenCalled();
+ });
+ });
});
describe("createSkippedPullRequestCheckRun", () => {
diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.ts
index 799c3e752a..8b7ee6204f 100644
--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.ts
+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-fns.ts
@@ -1,4 +1,7 @@
import { Repository } from "@octokit/webhooks-types";
+import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
import { ProbotOctokit } from "probot";
import { z } from "zod";
@@ -22,6 +25,11 @@ import {
TSecretScanningFactoryProcessEvent,
TSecretScanningScanReport
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
+import {
+ execFileBounded,
+ GIT_PROCESS_ENV,
+ SecretScanningExecPhase
+} from "@app/ee/services/secret-scanning/secret-scanning-exec";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { getConfig } from "@app/lib/config/env";
@@ -45,6 +53,8 @@ const COMPARE_COMMITS_MAX_PAGES = 20;
const COMMIT_FILES_PER_PAGE = 100;
// GitHub lists at most 3,000 of a commit's files, across pages.
const COMMIT_MAX_LISTED_FILES = 3000;
+// A file whose patch GitHub leaves out is fetched whole to rebuild it, so its size is capped.
+const MISSING_PATCH_MAX_FILE_BYTES = 10 * 1024 * 1024;
// GitHub rejects a check run update carrying more than 50 annotations; more are sent as further updates.
const CHECK_RUN_ANNOTATIONS_PER_REQUEST = 50;
const CHECK_RUN_MAX_ANNOTATIONS = 500;
@@ -121,23 +131,125 @@ const listCommitFiles = async (
{ owner, repo, sha }: { owner: string; repo: string; sha: string }
) => {
const files: NonNullable<Awaited<ReturnType<TGitHubOctokit["repos"]["getCommit"]>>["data"]["files"]> = [];
+ let parentSha: string | null = null;
for (let page = 1; files.length < COMMIT_MAX_LISTED_FILES; page += 1) {
// eslint-disable-next-line no-await-in-loop
const { data } = await octokit.repos.getCommit({ owner, repo, ref: sha, per_page: COMMIT_FILES_PER_PAGE, page });
const pageFiles = data.files ?? [];
+ // GitHub diffs a merge commit's files against its first parent.
+ if (page === 1) parentSha = data.parents?.length ? data.parents[0].sha : null;
+
files.push(...pageFiles);
if (pageFiles.length < COMMIT_FILES_PER_PAGE) break;
}
- return { files, isComplete: files.length < COMMIT_MAX_LISTED_FILES };
+ return { files, parentSha, isComplete: files.length < COMMIT_MAX_LISTED_FILES };
+};
+
+class MissingPatchFileTooLargeError extends Error {}
+
+const getGitHubFileContent = async (
+ octokit: TGitHubOctokit,
+ { owner, repo, path, ref }: { owner: string; repo: string; path: string; ref: string }
+) => {
+ const { data } = await octokit.repos.getContent({ owner, repo, path, ref });
+
+ if (Array.isArray(data) || data.type !== "file") {
+ throw new Error(`Expected a file at ${path} [ref=${ref}], got ${Array.isArray(data) ? "a directory" : data.type}`);
+ }
+
+ if (data.size > MISSING_PATCH_MAX_FILE_BYTES) throw new MissingPatchFileTooLargeError();
+
+ // The contents API leaves out the content of a file over 1 MB; the blob API returns it.
+ if ("content" in data && data.content && data.encoding === "base64") return Buffer.from(data.content, "base64");
+
+ const { data: blob } = await octokit.git.getBlob({ owner, repo, file_sha: data.sha });
+
+ return Buffer.from(blob.content, "base64");
+};
+
+// Builds the patch GitHub would have sent: a unified diff starting at its first hunk. Context lines are left out, as
+// only added lines are scanned.
+const buildPatchFromContents = async (before: Buffer, after: Buffer) => {
+ const tempFolder = await mkdtemp(join(tmpdir(), "infisical-patch-"));
+
+ try {
+ const beforePath = join(tempFolder, "before");
+ const afterPath = join(tempFolder, "after");
+ const patchPath = join(tempFolder, "patch.diff");
+
+ await Promise.all([writeFile(beforePath, before), writeFile(afterPath, after)]);
+
+ // Exits 1 when the files differ. The diff goes to a file because the process output that is kept is size capped.
+ await execFileBounded(
+ "git",
+ [
+ "diff",
+ "--no-index",
+ "--no-color",
+ "--no-ext-diff",
+ "--text",
+ "--unified=0",
+ `--output=${patchPath}`,
+ beforePath,
+ afterPath
+ ],
+ {
+ phase: SecretScanningExecPhase.Scan,
+ timeoutMs: getConfig().SECRET_SCANNING_SCAN_TIMEOUT,
+ env: GIT_PROCESS_ENV,
+ successExitCodes: [0, 1]
+ }
+ );
+
+ const diff = await readFile(patchPath, "utf8");
+ const firstHunkIndex = diff.indexOf("\n@@");
+
+ return firstHunkIndex === -1 ? "" : diff.slice(firstHunkIndex + 1);
+ } finally {
+ await rm(tempFolder, { recursive: true, force: true });
+ }
+};
+
+// GitHub leaves out the patch of a diff too large to show, so its added lines are diffed here from the file's contents.
+const buildMissingGitHubPatch = async (
+ octokit: TGitHubOctokit,
+ {
+ owner,
+ repo,
+ commitSha,
+ parentSha,
+ file
+ }: {
+ owner: string;
+ repo: string;
+ commitSha: string;
+ parentSha: string | null;
+ file: { status: string; filename: string; previous_filename?: string };
+ }
+) => {
+ const after = await getGitHubFileContent(octokit, { owner, repo, path: file.filename, ref: commitSha });
+
+ const before =
+ file.status === "added" || !parentSha
+ ? Buffer.alloc(0)
+ : await getGitHubFileContent(octokit, {
+ owner,
+ repo,
+ path: file.previous_filename ?? file.filename,
+ ref: parentSha
+ });
+
+ return buildPatchFromContents(before, after);
};
/**
* A commit that changes more files than GitHub lists can't be fully scanned from the API. A pull request check passes
- * `requireCompleteCommits` so it fails rather than passing on part of the commit; a push scan scans what it can.
+ * `requireCompleteCommits` so it fails rather than passing on part of the commit; a push scan scans what it can. The
+ * same goes for a file whose patch GitHub left out and that can't be rebuilt from its contents.
*/
export const scanGitHubCommitsAndGetFindings = async ({
octokit,
@@ -160,7 +272,7 @@ export const scanGitHubCommitsAndGetFindings = async ({
for (const commit of commits) {
// eslint-disable-next-line no-await-in-loop
- const { files, isComplete } = await listCommitFiles(octokit, { owner, repo, sha: commit.sha });
+ const { files, parentSha, isComplete } = await listCommitFiles(octokit, { owner, repo, sha: commit.sha });
if (!isComplete) {
if (requireCompleteCommits) {
@@ -176,16 +288,52 @@ export const scanGitHubCommitsAndGetFindings = async ({
for (const file of files) {
// A renamed, copied or changed file's patch can add lines too; a removed file's adds none.
- if (file.status !== "removed" && file.patch) {
+ // eslint-disable-next-line no-continue
+ if (file.status === "removed") continue;
+
+ let { patch } = file;
+
+ // A binary file has no patch either, but adds no lines.
+ if (!patch && file.additions > 0) {
+ try {
+ // eslint-disable-next-line no-await-in-loop
+ patch = await buildMissingGitHubPatch(octokit, {
+ owner,
+ repo,
+ commitSha: commit.sha,
+ parentSha,
+ file
+ });
+ } catch (error) {
+ const reason =
+ error instanceof MissingPatchFileTooLargeError
+ ? `it is larger than ${MISSING_PATCH_MAX_FILE_BYTES / 1024 / 1024} MB`
+ : "its contents couldn't be read from GitHub";
+
+ logger.warn(
+ error,
+ `secretScanningV2DiffScan: GitHub - file patch missing and couldn't be rebuilt [repository=${repositoryFullName}] [commit=${commit.sha}] [file=${file.filename}]`
+ );
+
+ if (requireCompleteCommits) {
+ throw new Error(
+ `Commit ${commit.sha.slice(0, 7)}'s change to ${file.filename} is too large for GitHub to include, and the file couldn't be scanned in full because ${reason}.`
+ );
+ }
+ }
+ }
+
+ if (patch) {
+ const filePatch = patch;
// eslint-disable-next-line
- const findings = await scanContentAndGetFindings(replaceNonChangesWithNewlines(`\n${file.patch}`), configPath);
+ const findings = await scanContentAndGetFindings(replaceNonChangesWithNewlines(`\n${filePatch}`), configPath);
const adjustedFindings = findings.map((finding) => {
- const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
+ const startLine = convertPatchLineToFileLineNumber(filePatch, finding.StartLine);
const endLine =
finding.StartLine === finding.EndLine
? startLine
- : convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
+ : convertPatchLineToFileLineNumber(filePatch, finding.EndLine);
const startColumn = finding.StartColumn - 1; // subtract 1 for +
const endColumn = finding.EndColumn - 1; // subtract 1 for +