Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

add commit digest to identity commits creeping in

Matheus Nogueira357cabe92bc1
diff --git a/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts b/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts
index 59cc9d3217..7dff60f015 100644
--- a/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts
+++ b/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts
@@ -5,11 +5,16 @@ import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasLastScannedCommit = await knex.schema.hasColumn(TableName.SecretScanningScan, "lastScannedCommit");
const hasProgressUpdatedAt = await knex.schema.hasColumn(TableName.SecretScanningScan, "progressUpdatedAt");
+ const hasLastScannedCommitDigest = await knex.schema.hasColumn(
+ TableName.SecretScanningScan,
+ "lastScannedCommitDigest"
+ );
- if (!hasLastScannedCommit || !hasProgressUpdatedAt) {
+ if (!hasLastScannedCommit || !hasProgressUpdatedAt || !hasLastScannedCommitDigest) {
await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
if (!hasLastScannedCommit) t.string("lastScannedCommit").nullable();
if (!hasProgressUpdatedAt) t.timestamp("progressUpdatedAt").nullable();
+ if (!hasLastScannedCommitDigest) t.string("lastScannedCommitDigest").nullable();
});
}
}
@@ -17,11 +22,16 @@ export async function up(knex: Knex): Promise<void> {
export async function down(knex: Knex): Promise<void> {
const hasLastScannedCommit = await knex.schema.hasColumn(TableName.SecretScanningScan, "lastScannedCommit");
const hasProgressUpdatedAt = await knex.schema.hasColumn(TableName.SecretScanningScan, "progressUpdatedAt");
+ const hasLastScannedCommitDigest = await knex.schema.hasColumn(
+ TableName.SecretScanningScan,
+ "lastScannedCommitDigest"
+ );
- if (hasLastScannedCommit || hasProgressUpdatedAt) {
+ if (hasLastScannedCommit || hasProgressUpdatedAt || hasLastScannedCommitDigest) {
await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
if (hasLastScannedCommit) t.dropColumn("lastScannedCommit");
if (hasProgressUpdatedAt) t.dropColumn("progressUpdatedAt");
+ if (hasLastScannedCommitDigest) t.dropColumn("lastScannedCommitDigest");
});
}
}
diff --git a/backend/src/db/schemas/secret-scanning-scans.ts b/backend/src/db/schemas/secret-scanning-scans.ts
index 3e5c665d1c..51fdc8dcdc 100644
--- a/backend/src/db/schemas/secret-scanning-scans.ts
+++ b/backend/src/db/schemas/secret-scanning-scans.ts
@@ -16,7 +16,8 @@ export const SecretScanningScansSchema = z.object({
createdAt: z.date().nullable().optional(),
scanningStartedAt: z.date().nullable().optional(),
lastScannedCommit: z.string().nullable().optional(),
- progressUpdatedAt: z.date().nullable().optional()
+ progressUpdatedAt: z.date().nullable().optional(),
+ lastScannedCommitDigest: z.string().nullable().optional()
});
export type TSecretScanningScans = z.infer<typeof SecretScanningScansSchema>;
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
index 86fe51eac5..dc04b7a747 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
@@ -1,6 +1,8 @@
-import { mkdtemp, rm } from "fs/promises";
+import { execFile } from "child_process";
+import { mkdtemp, rm, writeFile } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
+import { promisify } from "util";
import { describe, expect, test, vi } from "vitest";
import {
@@ -12,6 +14,7 @@ import {
import {
assertClonedRepositoryWithinSizeLimit,
parseScanErrorMessage,
+ planCommitBatches,
SecretScanningSizeLimitError
} from "./secret-scanning-v2-fns";
@@ -150,3 +153,117 @@ describe("assertClonedRepositoryWithinSizeLimit", () => {
}
});
});
+
+describe("planCommitBatches", () => {
+ const exec = promisify(execFile);
+
+ const git = (repoPath: string, args: string[], date?: string) =>
+ exec("git", args, {
+ cwd: repoPath,
+ env: {
+ ...process.env,
+ GIT_AUTHOR_NAME: "test",
+ GIT_AUTHOR_EMAIL: "[email protected]",
+ GIT_COMMITTER_NAME: "test",
+ GIT_COMMITTER_EMAIL: "[email protected]",
+ ...(date ? { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } : {})
+ }
+ });
+
+ const commit = async (repoPath: string, name: string, date: string) => {
+ await writeFile(join(repoPath, `${name}.txt`), name);
+ await git(repoPath, ["add", "."], date);
+ await git(repoPath, ["commit", "-m", name], date);
+ const { stdout } = await git(repoPath, ["rev-parse", "HEAD"]);
+ return stdout.trim();
+ };
+
+ const withRepo = async (run: (repoPath: string) => Promise<void>) => {
+ const repoPath = await mkdtemp(join(tmpdir(), "plan-commit-batches-"));
+
+ try {
+ await git(repoPath, ["init", "--initial-branch=main"]);
+ await run(repoPath);
+ } finally {
+ await rm(repoPath, { recursive: true, force: true });
+ }
+ };
+
+ test("resumes past the commits it already scanned", async () => {
+ await withRepo(async (repoPath) => {
+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");
+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");
+ await commit(repoPath, "c", "2020-01-03T00:00:00Z");
+ await commit(repoPath, "d", "2020-01-04T00:00:00Z");
+
+ const first = await planCommitBatches({ repoPath, batchSize: 2 });
+
+ expect(first.totalCommits).toBe(4);
+ expect(first.batches).toHaveLength(2);
+
+ const [firstBatch] = first.batches;
+
+ const resumed = await planCommitBatches({
+ repoPath,
+ batchSize: 2,
+ resumeAfterCommit: firstBatch.lastCommit,
+ resumeAfterCommitDigest: firstBatch.prefixDigest
+ });
+
+ expect(resumed.resumed).toBe(true);
+ expect(resumed.prefixChanged).toBe(false);
+ expect(resumed.batches).toEqual(first.batches.slice(1));
+ });
+ });
+
+ test("restarts when a newly reachable ref lands commits ahead of the resume point", async () => {
+ await withRepo(async (repoPath) => {
+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");
+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");
+ await commit(repoPath, "c", "2020-01-03T00:00:00Z");
+ await commit(repoPath, "d", "2020-01-04T00:00:00Z");
+
+ const [firstBatch] = (await planCommitBatches({ repoPath, batchSize: 2 })).batches;
+
+ // A branch rooted before the resume point, carrying a commit older than it. `--all` picks it
+ // up and the date ordering sorts it into the prefix, shifting every index after it.
+ await git(repoPath, ["checkout", "-b", "old", "HEAD~3"]);
+ const smuggled = await commit(repoPath, "smuggled", "2020-01-01T12:00:00Z");
+ await git(repoPath, ["checkout", "main"]);
+
+ const resumed = await planCommitBatches({
+ repoPath,
+ batchSize: 2,
+ resumeAfterCommit: firstBatch.lastCommit,
+ resumeAfterCommitDigest: firstBatch.prefixDigest
+ });
+
+ expect(resumed.prefixChanged).toBe(true);
+ expect(resumed.resumed).toBe(false);
+
+ // Every commit is back in the plan, the never-scanned one included.
+ const planned = resumed.batches.reduce((total, batch) => total + batch.maxCount, 0);
+ expect(resumed.totalCommits).toBe(5);
+ expect(planned).toBe(5);
+ expect(smuggled).toBeTruthy();
+ });
+ });
+
+ test("restarts when the resume point has left the repository", async () => {
+ await withRepo(async (repoPath) => {
+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");
+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");
+
+ const plan = await planCommitBatches({
+ repoPath,
+ batchSize: 1,
+ resumeAfterCommit: "0000000000000000000000000000000000000000",
+ resumeAfterCommitDigest: "stale"
+ });
+
+ expect(plan.resumed).toBe(false);
+ expect(plan.prefixChanged).toBe(false);
+ expect(plan.batches).toHaveLength(2);
+ });
+ });
+});
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
index 493a1efa59..8edba48959 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
@@ -156,9 +156,16 @@ export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository):
};
/**
- * Commits are enumerated oldest-first and scanned in slices of that order, because that is the only
- * ordering a resumed scan can trust: new commits land at the end, so an index into it still means
- * the same commit on the next run, where a newest-first index shifts under every push.
+ * Commits are enumerated oldest-first and scanned in slices of that order, because a newest-first
+ * index shifts under every push while an oldest-first one usually survives: an ordinary push lands
+ * its commits at the end, leaving everything before them where it was.
+ *
+ * "Usually" is not "always", which is what `lastScannedCommitDigest` is for. A ref that
+ * becomes reachable between two runs — a new branch, a fetched tag — brings its commits in at their
+ * own dates, which can be older than the resume point, so they sort into the middle of the ordering
+ * and push every later index up. An index alone cannot tell that apart from the commits it used to
+ * name, so the prefix is digested as it is walked and the resumed run checks the digest it stored
+ * against the prefix it now sees.
*
* `git log` itself only walks newest-first, so a slice is expressed as a skip/count against that
* order. Both halves must enumerate with identical flags for the two orderings to correspond.
@@ -171,6 +178,13 @@ export type TCommitBatch = {
maxCount: number;
/** The batch's newest commit, recorded as the scan's resume point once the batch completes. */
lastCommit: string;
+ /**
+ * Digest of every commit up to and including `lastCommit`, recorded beside it so the next run can
+ * tell if a new commit has creeped into this batch and it needs to be rescanned.
+ *
+ * This scenario can happen if a rebase happens in the repository.
+ */
+ prefixDigest: string;
};
const buildCommitBatchLogOpts = ({ skip, maxCount }: TCommitBatch) =>
@@ -185,16 +199,26 @@ const buildCommitBatchLogOpts = ({ skip, maxCount }: TCommitBatch) =>
export const planCommitBatches = async ({
repoPath,
batchSize,
- resumeAfterCommit
+ resumeAfterCommit,
+ resumeAfterCommitDigest
}: {
repoPath: string;
batchSize: number;
resumeAfterCommit?: string | null;
-}): Promise<{ totalCommits: number; batches: TCommitBatch[]; resumed: boolean }> => {
- const boundaries: { index: number; commit: string }[] = [];
+ resumeAfterCommitDigest?: string | null;
+}): Promise<{
+ totalCommits: number;
+ batches: TCommitBatch[];
+ resumed: boolean;
+ /** Set when the resume point is still present but the history before it is not the history that was scanned. */
+ prefixChanged: boolean;
+}> => {
+ const boundaries: { index: number; commit: string; prefixDigest: string }[] = [];
let totalCommits = 0;
let resumeIndex = -1;
+ let resumePrefixDigest = "";
let newestCommit = "";
+ let prefixDigest = "";
// A repository with hundreds of thousands of commits emits more than the exec layer will buffer,
// and nothing here needs the full list: only the batch edges and the resume point are retained.
@@ -210,28 +234,44 @@ export const planCommitBatches = async ({
const index = totalCommits;
totalCommits += 1;
newestCommit = commit;
+ prefixDigest = crypto.nativeCrypto.createHash("sha256").update(`${prefixDigest}${commit}`).digest("hex");
- if (commit === resumeAfterCommit) resumeIndex = index;
- if ((index + 1) % batchSize === 0) boundaries.push({ index, commit });
+ if (commit === resumeAfterCommit) {
+ resumeIndex = index;
+ resumePrefixDigest = prefixDigest;
+ }
+ if ((index + 1) % batchSize === 0) boundaries.push({ index, commit, prefixDigest });
}
});
const lastIndex = totalCommits - 1;
if (totalCommits && boundaries[boundaries.length - 1]?.index !== lastIndex) {
- boundaries.push({ index: lastIndex, commit: newestCommit });
+ boundaries.push({ index: lastIndex, commit: newestCommit, prefixDigest });
}
+ // Commits that became reachable since the last run can sort in ahead of the resume point, so the
+ // indexes after it no longer name the commits they named then. Skipping to that index would step
+ // over commits this scan has never looked at, so the only safe reading of the plan is a fresh one.
+ const prefixChanged = resumeIndex >= 0 && resumePrefixDigest !== resumeAfterCommitDigest;
+ const resumableIndex = prefixChanged ? -1 : resumeIndex;
+
const batches = boundaries
- .map(({ index, commit }, position) => ({
+ .map(({ index, commit, prefixDigest: boundaryPrefixDigest }, position) => ({
index,
skip: totalCommits - 1 - index,
maxCount: index - position * batchSize + 1,
- lastCommit: commit
+ lastCommit: commit,
+ prefixDigest: boundaryPrefixDigest
}))
- .filter(({ index }) => index > resumeIndex)
- .map(({ skip, maxCount, lastCommit }) => ({ skip, maxCount, lastCommit }));
-
- return { totalCommits, batches, resumed: resumeIndex >= 0 };
+ .filter(({ index }) => index > resumableIndex)
+ .map(({ skip, maxCount, lastCommit, prefixDigest: boundaryPrefixDigest }) => ({
+ skip,
+ maxCount,
+ lastCommit,
+ prefixDigest: boundaryPrefixDigest
+ }));
+
+ return { totalCommits, batches, resumed: resumableIndex >= 0, prefixChanged };
};
export async function scanDirectory(
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
index ba81030685..e78787c96b 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
@@ -330,7 +330,10 @@ export const secretScanningV2QueueServiceFactory = ({
* the scan and its lease — once the reaper has given up on it, or another scan of the resource
* has taken over, there is nothing left to make progress on.
*/
- const persistBatch = async (batchFindings: TFindingsPayload, lastScannedCommit?: string) => {
+ const persistBatch = async (
+ batchFindings: TFindingsPayload,
+ resumePoint?: { lastScannedCommit: string; lastScannedCommitDigest: string }
+ ) => {
const owned = await secretScanningV2DAL.findings.transaction(async (tx) => {
if (batchFindings.length) {
const findings = await secretScanningV2DAL.findings.upsert(
@@ -353,7 +356,7 @@ export const secretScanningV2QueueServiceFactory = ({
const progressed = await secretScanningV2DAL.scans.update(
{ id: scanId, status: SecretScanningScanStatus.Scanning },
- { lastScannedCommit, progressUpdatedAt: new Date() },
+ { ...resumePoint, progressUpdatedAt: new Date() },
tx
);
@@ -394,14 +397,17 @@ export const secretScanningV2QueueServiceFactory = ({
const plan = await planCommitBatches({
repoPath: scanPath,
batchSize,
- resumeAfterCommit: scan.lastScannedCommit
+ resumeAfterCommit: scan.lastScannedCommit,
+ resumeAfterCommitDigest: scan.lastScannedCommitDigest
});
- // A rewritten history (force push, or a rebase landing between runs) can take the commit
- // this scan stopped at out of the repository entirely, leaving nothing to resume from.
+ // Two ways a resume point stops meaning anything: a rewritten history (force push, or a
+ // rebase landing between runs) takes the commit out of the repository entirely, or a newly
+ // reachable ref puts commits ahead of it that this scan has never looked at. Either way the
+ // repository is re-walked from the start rather than resumed past commits nobody scanned.
if (scan.lastScannedCommit && !plan.resumed) {
logger.warn(
- `secretScanningV2Queue: Full Scan resume point is no longer in the repository, restarting ${logDetails} [lastScannedCommit=${scan.lastScannedCommit}]`
+ `secretScanningV2Queue: Full Scan cannot resume, restarting ${logDetails} [lastScannedCommit=${scan.lastScannedCommit}] [reason=${plan.prefixChanged ? "history before the resume point changed" : "resume point is no longer in the repository"}]`
);
}
@@ -421,7 +427,10 @@ export const secretScanningV2QueueServiceFactory = ({
findingsCount += batchFindings.length;
// eslint-disable-next-line no-await-in-loop
- stillOwned = await persistBatch(batchFindings, batch.lastCommit);
+ stillOwned = await persistBatch(batchFindings, {
+ lastScannedCommit: batch.lastCommit,
+ lastScannedCommitDigest: batch.prefixDigest
+ });
if (!stillOwned) break;