Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
add commit digest to identity commits creeping in
Matheus Nogueira357cabe92bc1
diff --git a/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts b/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.tsindex 59cc9d3217..7dff60f015 100644--- a/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts+++ b/backend/src/db/migrations/20260916120000_add-last-scanned-commit-to-secret-scanning-scans.ts@@ -5,11 +5,16 @@ import { TableName } from "../schemas";export async function up(knex: Knex): Promise<void> {const hasLastScannedCommit = await knex.schema.hasColumn(TableName.SecretScanningScan, "lastScannedCommit");const hasProgressUpdatedAt = await knex.schema.hasColumn(TableName.SecretScanningScan, "progressUpdatedAt");+ const hasLastScannedCommitDigest = await knex.schema.hasColumn(+ TableName.SecretScanningScan,+ "lastScannedCommitDigest"+ );- if (!hasLastScannedCommit || !hasProgressUpdatedAt) {+ if (!hasLastScannedCommit || !hasProgressUpdatedAt || !hasLastScannedCommitDigest) {await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {if (!hasLastScannedCommit) t.string("lastScannedCommit").nullable();if (!hasProgressUpdatedAt) t.timestamp("progressUpdatedAt").nullable();+ if (!hasLastScannedCommitDigest) t.string("lastScannedCommitDigest").nullable();});}}@@ -17,11 +22,16 @@ export async function up(knex: Knex): Promise<void> {export async function down(knex: Knex): Promise<void> {const hasLastScannedCommit = await knex.schema.hasColumn(TableName.SecretScanningScan, "lastScannedCommit");const hasProgressUpdatedAt = await knex.schema.hasColumn(TableName.SecretScanningScan, "progressUpdatedAt");+ const hasLastScannedCommitDigest = await knex.schema.hasColumn(+ TableName.SecretScanningScan,+ "lastScannedCommitDigest"+ );- if (hasLastScannedCommit || hasProgressUpdatedAt) {+ if (hasLastScannedCommit || hasProgressUpdatedAt || hasLastScannedCommitDigest) {await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {if (hasLastScannedCommit) t.dropColumn("lastScannedCommit");if (hasProgressUpdatedAt) t.dropColumn("progressUpdatedAt");+ if (hasLastScannedCommitDigest) t.dropColumn("lastScannedCommitDigest");});}}diff --git a/backend/src/db/schemas/secret-scanning-scans.ts b/backend/src/db/schemas/secret-scanning-scans.tsindex 3e5c665d1c..51fdc8dcdc 100644--- a/backend/src/db/schemas/secret-scanning-scans.ts+++ b/backend/src/db/schemas/secret-scanning-scans.ts@@ -16,7 +16,8 @@ export const SecretScanningScansSchema = z.object({createdAt: z.date().nullable().optional(),scanningStartedAt: z.date().nullable().optional(),lastScannedCommit: z.string().nullable().optional(),- progressUpdatedAt: z.date().nullable().optional()+ progressUpdatedAt: z.date().nullable().optional(),+ lastScannedCommitDigest: z.string().nullable().optional()});export type TSecretScanningScans = z.infer<typeof SecretScanningScansSchema>;diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.tsindex 86fe51eac5..dc04b7a747 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts@@ -1,6 +1,8 @@-import { mkdtemp, rm } from "fs/promises";+import { execFile } from "child_process";+import { mkdtemp, rm, writeFile } from "fs/promises";import { tmpdir } from "os";import { join } from "path";+import { promisify } from "util";import { describe, expect, test, vi } from "vitest";import {@@ -12,6 +14,7 @@ import {import {assertClonedRepositoryWithinSizeLimit,parseScanErrorMessage,+ planCommitBatches,SecretScanningSizeLimitError} from "./secret-scanning-v2-fns";@@ -150,3 +153,117 @@ describe("assertClonedRepositoryWithinSizeLimit", () => {}});});++describe("planCommitBatches", () => {+ const exec = promisify(execFile);++ const git = (repoPath: string, args: string[], date?: string) =>+ exec("git", args, {+ cwd: repoPath,+ env: {+ ...process.env,+ GIT_AUTHOR_NAME: "test",+ GIT_AUTHOR_EMAIL: "[email protected]",+ GIT_COMMITTER_NAME: "test",+ GIT_COMMITTER_EMAIL: "[email protected]",+ ...(date ? { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } : {})+ }+ });++ const commit = async (repoPath: string, name: string, date: string) => {+ await writeFile(join(repoPath, `${name}.txt`), name);+ await git(repoPath, ["add", "."], date);+ await git(repoPath, ["commit", "-m", name], date);+ const { stdout } = await git(repoPath, ["rev-parse", "HEAD"]);+ return stdout.trim();+ };++ const withRepo = async (run: (repoPath: string) => Promise<void>) => {+ const repoPath = await mkdtemp(join(tmpdir(), "plan-commit-batches-"));++ try {+ await git(repoPath, ["init", "--initial-branch=main"]);+ await run(repoPath);+ } finally {+ await rm(repoPath, { recursive: true, force: true });+ }+ };++ test("resumes past the commits it already scanned", async () => {+ await withRepo(async (repoPath) => {+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");+ await commit(repoPath, "c", "2020-01-03T00:00:00Z");+ await commit(repoPath, "d", "2020-01-04T00:00:00Z");++ const first = await planCommitBatches({ repoPath, batchSize: 2 });++ expect(first.totalCommits).toBe(4);+ expect(first.batches).toHaveLength(2);++ const [firstBatch] = first.batches;++ const resumed = await planCommitBatches({+ repoPath,+ batchSize: 2,+ resumeAfterCommit: firstBatch.lastCommit,+ resumeAfterCommitDigest: firstBatch.prefixDigest+ });++ expect(resumed.resumed).toBe(true);+ expect(resumed.prefixChanged).toBe(false);+ expect(resumed.batches).toEqual(first.batches.slice(1));+ });+ });++ test("restarts when a newly reachable ref lands commits ahead of the resume point", async () => {+ await withRepo(async (repoPath) => {+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");+ await commit(repoPath, "c", "2020-01-03T00:00:00Z");+ await commit(repoPath, "d", "2020-01-04T00:00:00Z");++ const [firstBatch] = (await planCommitBatches({ repoPath, batchSize: 2 })).batches;++ // A branch rooted before the resume point, carrying a commit older than it. `--all` picks it+ // up and the date ordering sorts it into the prefix, shifting every index after it.+ await git(repoPath, ["checkout", "-b", "old", "HEAD~3"]);+ const smuggled = await commit(repoPath, "smuggled", "2020-01-01T12:00:00Z");+ await git(repoPath, ["checkout", "main"]);++ const resumed = await planCommitBatches({+ repoPath,+ batchSize: 2,+ resumeAfterCommit: firstBatch.lastCommit,+ resumeAfterCommitDigest: firstBatch.prefixDigest+ });++ expect(resumed.prefixChanged).toBe(true);+ expect(resumed.resumed).toBe(false);++ // Every commit is back in the plan, the never-scanned one included.+ const planned = resumed.batches.reduce((total, batch) => total + batch.maxCount, 0);+ expect(resumed.totalCommits).toBe(5);+ expect(planned).toBe(5);+ expect(smuggled).toBeTruthy();+ });+ });++ test("restarts when the resume point has left the repository", async () => {+ await withRepo(async (repoPath) => {+ await commit(repoPath, "a", "2020-01-01T00:00:00Z");+ await commit(repoPath, "b", "2020-01-02T00:00:00Z");++ const plan = await planCommitBatches({+ repoPath,+ batchSize: 1,+ resumeAfterCommit: "0000000000000000000000000000000000000000",+ resumeAfterCommitDigest: "stale"+ });++ expect(plan.resumed).toBe(false);+ expect(plan.prefixChanged).toBe(false);+ expect(plan.batches).toHaveLength(2);+ });+ });+});diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.tsindex 493a1efa59..8edba48959 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts@@ -156,9 +156,16 @@ export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository):};/**- * Commits are enumerated oldest-first and scanned in slices of that order, because that is the only- * ordering a resumed scan can trust: new commits land at the end, so an index into it still means- * the same commit on the next run, where a newest-first index shifts under every push.+ * Commits are enumerated oldest-first and scanned in slices of that order, because a newest-first+ * index shifts under every push while an oldest-first one usually survives: an ordinary push lands+ * its commits at the end, leaving everything before them where it was.+ *+ * "Usually" is not "always", which is what `lastScannedCommitDigest` is for. A ref that+ * becomes reachable between two runs — a new branch, a fetched tag — brings its commits in at their+ * own dates, which can be older than the resume point, so they sort into the middle of the ordering+ * and push every later index up. An index alone cannot tell that apart from the commits it used to+ * name, so the prefix is digested as it is walked and the resumed run checks the digest it stored+ * against the prefix it now sees.** `git log` itself only walks newest-first, so a slice is expressed as a skip/count against that* order. Both halves must enumerate with identical flags for the two orderings to correspond.@@ -171,6 +178,13 @@ export type TCommitBatch = {maxCount: number;/** The batch's newest commit, recorded as the scan's resume point once the batch completes. */lastCommit: string;+ /**+ * Digest of every commit up to and including `lastCommit`, recorded beside it so the next run can+ * tell if a new commit has creeped into this batch and it needs to be rescanned.+ *+ * This scenario can happen if a rebase happens in the repository.+ */+ prefixDigest: string;};const buildCommitBatchLogOpts = ({ skip, maxCount }: TCommitBatch) =>@@ -185,16 +199,26 @@ const buildCommitBatchLogOpts = ({ skip, maxCount }: TCommitBatch) =>export const planCommitBatches = async ({repoPath,batchSize,- resumeAfterCommit+ resumeAfterCommit,+ resumeAfterCommitDigest}: {repoPath: string;batchSize: number;resumeAfterCommit?: string | null;-}): Promise<{ totalCommits: number; batches: TCommitBatch[]; resumed: boolean }> => {- const boundaries: { index: number; commit: string }[] = [];+ resumeAfterCommitDigest?: string | null;+}): Promise<{+ totalCommits: number;+ batches: TCommitBatch[];+ resumed: boolean;+ /** Set when the resume point is still present but the history before it is not the history that was scanned. */+ prefixChanged: boolean;+}> => {+ const boundaries: { index: number; commit: string; prefixDigest: string }[] = [];let totalCommits = 0;let resumeIndex = -1;+ let resumePrefixDigest = "";let newestCommit = "";+ let prefixDigest = "";// A repository with hundreds of thousands of commits emits more than the exec layer will buffer,// and nothing here needs the full list: only the batch edges and the resume point are retained.@@ -210,28 +234,44 @@ export const planCommitBatches = async ({const index = totalCommits;totalCommits += 1;newestCommit = commit;+ prefixDigest = crypto.nativeCrypto.createHash("sha256").update(`${prefixDigest}${commit}`).digest("hex");- if (commit === resumeAfterCommit) resumeIndex = index;- if ((index + 1) % batchSize === 0) boundaries.push({ index, commit });+ if (commit === resumeAfterCommit) {+ resumeIndex = index;+ resumePrefixDigest = prefixDigest;+ }+ if ((index + 1) % batchSize === 0) boundaries.push({ index, commit, prefixDigest });}});const lastIndex = totalCommits - 1;if (totalCommits && boundaries[boundaries.length - 1]?.index !== lastIndex) {- boundaries.push({ index: lastIndex, commit: newestCommit });+ boundaries.push({ index: lastIndex, commit: newestCommit, prefixDigest });}+ // Commits that became reachable since the last run can sort in ahead of the resume point, so the+ // indexes after it no longer name the commits they named then. Skipping to that index would step+ // over commits this scan has never looked at, so the only safe reading of the plan is a fresh one.+ const prefixChanged = resumeIndex >= 0 && resumePrefixDigest !== resumeAfterCommitDigest;+ const resumableIndex = prefixChanged ? -1 : resumeIndex;+const batches = boundaries- .map(({ index, commit }, position) => ({+ .map(({ index, commit, prefixDigest: boundaryPrefixDigest }, position) => ({index,skip: totalCommits - 1 - index,maxCount: index - position * batchSize + 1,- lastCommit: commit+ lastCommit: commit,+ prefixDigest: boundaryPrefixDigest}))- .filter(({ index }) => index > resumeIndex)- .map(({ skip, maxCount, lastCommit }) => ({ skip, maxCount, lastCommit }));-- return { totalCommits, batches, resumed: resumeIndex >= 0 };+ .filter(({ index }) => index > resumableIndex)+ .map(({ skip, maxCount, lastCommit, prefixDigest: boundaryPrefixDigest }) => ({+ skip,+ maxCount,+ lastCommit,+ prefixDigest: boundaryPrefixDigest+ }));++ return { totalCommits, batches, resumed: resumableIndex >= 0, prefixChanged };};export async function scanDirectory(diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.tsindex ba81030685..e78787c96b 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts@@ -330,7 +330,10 @@ export const secretScanningV2QueueServiceFactory = ({* the scan and its lease — once the reaper has given up on it, or another scan of the resource* has taken over, there is nothing left to make progress on.*/- const persistBatch = async (batchFindings: TFindingsPayload, lastScannedCommit?: string) => {+ const persistBatch = async (+ batchFindings: TFindingsPayload,+ resumePoint?: { lastScannedCommit: string; lastScannedCommitDigest: string }+ ) => {const owned = await secretScanningV2DAL.findings.transaction(async (tx) => {if (batchFindings.length) {const findings = await secretScanningV2DAL.findings.upsert(@@ -353,7 +356,7 @@ export const secretScanningV2QueueServiceFactory = ({const progressed = await secretScanningV2DAL.scans.update({ id: scanId, status: SecretScanningScanStatus.Scanning },- { lastScannedCommit, progressUpdatedAt: new Date() },+ { ...resumePoint, progressUpdatedAt: new Date() },tx);@@ -394,14 +397,17 @@ export const secretScanningV2QueueServiceFactory = ({const plan = await planCommitBatches({repoPath: scanPath,batchSize,- resumeAfterCommit: scan.lastScannedCommit+ resumeAfterCommit: scan.lastScannedCommit,+ resumeAfterCommitDigest: scan.lastScannedCommitDigest});- // A rewritten history (force push, or a rebase landing between runs) can take the commit- // this scan stopped at out of the repository entirely, leaving nothing to resume from.+ // Two ways a resume point stops meaning anything: a rewritten history (force push, or a+ // rebase landing between runs) takes the commit out of the repository entirely, or a newly+ // reachable ref puts commits ahead of it that this scan has never looked at. Either way the+ // repository is re-walked from the start rather than resumed past commits nobody scanned.if (scan.lastScannedCommit && !plan.resumed) {logger.warn(- `secretScanningV2Queue: Full Scan resume point is no longer in the repository, restarting ${logDetails} [lastScannedCommit=${scan.lastScannedCommit}]`+ `secretScanningV2Queue: Full Scan cannot resume, restarting ${logDetails} [lastScannedCommit=${scan.lastScannedCommit}] [reason=${plan.prefixChanged ? "history before the resume point changed" : "resume point is no longer in the repository"}]`);}@@ -421,7 +427,10 @@ export const secretScanningV2QueueServiceFactory = ({findingsCount += batchFindings.length;// eslint-disable-next-line no-await-in-loop- stillOwned = await persistBatch(batchFindings, batch.lastCommit);+ stillOwned = await persistBatch(batchFindings, {+ lastScannedCommit: batch.lastCommit,+ lastScannedCommitDigest: batch.prefixDigest+ });if (!stillOwned) break;