Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
import { AccessScope } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { logger } from "@app/lib/logger";
import { resolveCoreMeter } from "@app/lib/telemetry/metrics";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { ActorType } from "../auth/auth-type";
import { TCaKeySourceServiceFactory } from "../certificate-authority/ca-key-source/ca-key-source-service";
import { TCertificateAuthoritySecretDALFactory } from "../certificate-authority/certificate-authority-secret-dal";
import { TKmsServiceFactory } from "../kms/kms-service";
import { TMembershipUserDALFactory } from "../membership-user/membership-user-dal";
import { TUserDALFactory } from "../user/user-dal";
import { TProjectDALFactory } from "./project-dal";
// ── load shaping ──────────────────────────────────────────────────────────────
// The cron tick is NOT the pacer — it only keeps the queue fed. Selecting the oldest
// DISCOVERY_BATCH expired projects (ORDER BY deleteAfter ASC) + jobId dedupe keeps the in-flight
// set bounded at ~DISCOVERY_BATCH regardless of backlog size (we never enqueue 85k at once).
// The worker meters the actual drain RATE via the rate limiter + concurrency, so DB load is a
// flat plateau rather than a per-tick spike.
const DISCOVERY_BATCH = 200;
const WORKER_CONCURRENCY = 3; // ceiling on simultaneous heavy deletes (fleet-wide)
const RATE_LIMIT_MAX = 30; // steady drain rate: 30 projects / minute
const RATE_LIMIT_DURATION_MS = 60 * 1000;
// per-project chunked secret_versions_v2 delete tuning
const SECRET_VERSION_DELETE_BATCH = 5000;
const BATCH_STATEMENT_TIMEOUT_MS = 30 * 1000;
const INTER_BATCH_SLEEP_MS = 10;
// Env count above which env deletion is delegated to the paced env hard-delete worker (one untimed
// cascade per env, per-env retries). At or below it, deleteById cascades the envs inline as today.
const ENV_DELEGATION_THRESHOLD = 100;
// Generous so a large project's chunked delete won't see the lock expire mid-flight and let a
// second worker start; crash recovery happens on the next cron tick after expiry.
const PROJECT_DELETE_LOCK_TTL_MS = 30 * 60 * 1000;
const CRON_HANDLER_TIMEOUT_MS = 5 * 60 * 1000;
type TProjectCleanupQueueFactoryDep = {
projectDAL: Pick<
TProjectDALFactory,
| "findExpiredForHardDelete"
| "findByIdIncludingExpired"
| "findIncludingExpired"
| "findProjectGhostUser"
| "hardDeleteProjectSecretVersionsInBatches"
| "hardDeleteProjectSecretReferencesInBatches"
| "hardDeleteProjectApprovalSecretLinksInBatches"
| "hardDeleteProjectSecretsInBatches"
| "softDeleteProjectEnvironments"
| "countProjectEnvironments"
| "deleteById"
| "transaction"
>;
membershipUserDAL: Pick<TMembershipUserDALFactory, "delete">;
userDAL: Pick<TUserDALFactory, "deleteById">;
kmsService: Pick<TKmsServiceFactory, "deleteInternalKms">;
certificateAuthoritySecretDAL: Pick<TCertificateAuthoritySecretDALFactory, "findExternallyKeyedByProjectId">;
caKeySourceService: Pick<TCaKeySourceServiceFactory, "prepareKeyReleases">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "deleteItem">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
queueService: Pick<TQueueServiceFactory, "queue" | "start">;
cronJob: TCronJobFactory;
};
export type TProjectCleanupQueueFactory = ReturnType<typeof projectCleanupQueueFactory>;
export const projectCleanupQueueFactory = ({
projectDAL,
membershipUserDAL,
userDAL,
kmsService,
certificateAuthoritySecretDAL,
caKeySourceService,
keyStore,
auditLogService,
queueService,
cronJob
}: TProjectCleanupQueueFactoryDep) => {
const appCfg = getConfig();
if (appCfg.isDailyResourceCleanUpDevelopmentMode) {
logger.warn("Project hard-delete cron is in development mode.");
}
// ── observability ─────────────────────────────────────────────────────────────
const meter = resolveCoreMeter();
// Per-pod, last-tick value: only the pod that won the cron redlock updates this; other pods report 0.
// Alarms must aggregate with max() across pods, else a "stuck at cap" backlog gets diluted to ~0.
let lastDiscoveryCount = 0;
const discoveryGauge = meter.createObservableGauge("infisical.project_cleanup.discovered", {
description:
"Expired projects found on last discovery tick (capped at discovery batch). Sustained value at the cap = backlog remains.",
unit: "{project}"
});
discoveryGauge.addCallback((observableResult) => {
if (!getConfig().OTEL_TELEMETRY_COLLECTION_ENABLED) return;
observableResult.observe(lastDiscoveryCount);
});
const processProjectHardDelete = async (projectId: string) => {
let deletedVersions = 0;
let deletedReferences = 0;
let nulledApprovalLinks = 0;
let deletedSecrets = 0;
const lock = await keyStore
.acquireLock([KeyStorePrefixes.ProjectDeleteLock(projectId)], PROJECT_DELETE_LOCK_TTL_MS)
.catch(() => null);
if (!lock) {
logger.info(`project-hard-delete: lock held, will retry next firing [projectId=${projectId}]`);
return;
}
try {
// Read via transaction → primary, not replica. Defeats replica-lag races against a concurrent
// worker that already finished.
const project = await projectDAL.transaction((tx) => projectDAL.findByIdIncludingExpired(projectId, tx));
if (!project || !project.deleteAfter || new Date(project.deleteAfter).getTime() > Date.now()) {
logger.info(`project-hard-delete: skipping (gone/already-removed/not-yet-expired) [projectId=${projectId}]`);
return;
}
// 1) Chunk-delete the largest project-scoped table ahead of the final cascade. secret_versions_v2
// has no folderId/secretId FK and no other FK back to the project tree, so the project-delete
// cascade would otherwise orphan all of these rows. Deleting by folderId is FK-safe and bounds the
// final cascade's transaction size.
await projectDAL.hardDeleteProjectSecretVersionsInBatches(
projectId,
SECRET_VERSION_DELETE_BATCH,
BATCH_STATEMENT_TIMEOUT_MS,
INTER_BATCH_SLEEP_MS,
(deleted) => {
deletedVersions += deleted;
}
);
// 2) Same treatment for the two statements that actually time out inside the final cascade on a
// project with a large secret graph: the secret_references_v2 CASCADE and the
// secret_approval_requests_secrets_v2 SET NULL. Then the secrets themselves, so their remaining
// per-secret CASCADE children run on a batch-sized statement instead of the whole project.
// Order matters: deleting secrets first would just drag both cascades back into one statement.
await projectDAL.hardDeleteProjectSecretReferencesInBatches(
projectId,
SECRET_VERSION_DELETE_BATCH,
BATCH_STATEMENT_TIMEOUT_MS,
INTER_BATCH_SLEEP_MS,
(deleted) => {
deletedReferences += deleted;
}
);
await projectDAL.hardDeleteProjectApprovalSecretLinksInBatches(
projectId,
SECRET_VERSION_DELETE_BATCH,
BATCH_STATEMENT_TIMEOUT_MS,
INTER_BATCH_SLEEP_MS,
(deleted) => {
nulledApprovalLinks += deleted;
}
);
await projectDAL.hardDeleteProjectSecretsInBatches(
projectId,
SECRET_VERSION_DELETE_BATCH,
BATCH_STATEMENT_TIMEOUT_MS,
INTER_BATCH_SLEEP_MS,
(deleted) => {
deletedSecrets += deleted;
}
);
// 3) For big projects: mark envs for the env worker; the cron re-fires this job until drained.
const envCount = await projectDAL.countProjectEnvironments(projectId);
if (envCount > ENV_DELEGATION_THRESHOLD) {
const marked = await projectDAL.softDeleteProjectEnvironments(projectId);
logger.info(
{ projectId, envCount, marked, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
`project-hard-delete: waiting on env drain [projectId=${projectId}] [envsRemaining=${envCount}] [newlyMarked=${marked}]`
);
return;
}
const releaseCaKeys = await caKeySourceService.prepareKeyReleases(
await certificateAuthoritySecretDAL.findExternallyKeyedByProjectId(projectId)
);
// 4) Final cascade in one transaction — handles the remaining (smaller) child tables, including
// deferred / NO ACTION FKs (e.g. secret_rotation_v2_secret_mappings) that PG resolves correctly
// as a single cascade tree (the same proven path the synchronous delete used).
const deleted = await projectDAL.transaction(async (tx) => {
// capture the ghost user before its project membership is removed below
const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx).catch(() => null);
await membershipUserDAL.delete(
{ scopeOrgId: project.orgId, scopeProjectId: projectId, scope: AccessScope.Project },
tx
);
const delProject = await projectDAL.deleteById(projectId, tx);
// a racing worker may have hard-deleted it after our re-read but before we acquired the row
if (!delProject) return false;
// before removing a KMS key, ensure no other (incl. pending-deletion) project references it
if (delProject.kmsCertificateKeyId) {
const linked = await projectDAL.findIncludingExpired(
{ kmsCertificateKeyId: delProject.kmsCertificateKeyId },
{ tx }
);
if (!linked.length) await kmsService.deleteInternalKms(delProject.kmsCertificateKeyId, delProject.orgId, tx);
}
if (delProject.kmsSecretManagerKeyId) {
const linked = await projectDAL.findIncludingExpired(
{ kmsSecretManagerKeyId: delProject.kmsSecretManagerKeyId },
{ tx }
);
if (!linked.length)
await kmsService.deleteInternalKms(delProject.kmsSecretManagerKeyId, delProject.orgId, tx);
}
if (ghostUser) await userDAL.deleteById(ghostUser.id, tx);
return true;
});
if (!deleted) {
logger.info(
`project-hard-delete: hard-deleted project was already removed by a concurrent worker [projectId=${projectId}]`
);
return;
}
await releaseCaKeys();
await keyStore.deleteItem(KeyStorePrefixes.LicenseCloudPlan(project.orgId));
await auditLogService.createAuditLog({
orgId: project.orgId,
projectId,
actor: { type: ActorType.PLATFORM, metadata: {} },
event: {
type: EventType.DELETE_PROJECT,
metadata: { id: projectId, name: project.name, softDelete: false }
}
});
logger.info(
{ projectId, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
`project-hard-delete: hard-deleted project [projectId=${projectId}] [versionsPruned=${deletedVersions}] [referencesPruned=${deletedReferences}] [approvalLinksNulled=${nulledApprovalLinks}] [secretsPruned=${deletedSecrets}]`
);
} catch (err) {
logger.error(
{ err, projectId, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
`project-hard-delete: failed [projectId=${projectId}] [versionsPruned=${deletedVersions}] [referencesPruned=${deletedReferences}] [approvalLinksNulled=${nulledApprovalLinks}] [secretsPruned=${deletedSecrets}]`
);
throw err; // surface to BullMQ so it retries (attempts: 3) and the generic failure metric fires
} finally {
await lock.release();
}
};
const init = () => {
// Worker: meters the actual drain rate. concurrency caps parallelism; the limiter caps throughput
// (fleet-wide via Redis) so a backlog drains as an even plateau instead of a burst.
queueService.start(
QueueName.ProjectHardDelete,
async (job) => {
await processProjectHardDelete(job.data.projectId);
},
{ concurrency: WORKER_CONCURRENCY, limiter: { max: RATE_LIMIT_MAX, duration: RATE_LIMIT_DURATION_MS } }
);
// Cron tick: discovery only (fast). Enqueues the oldest expired projects; jobId dedupe keeps the
// queue bounded and idempotent across ticks.
cronJob.register({
name: CronJobName.ProjectHardDelete,
pattern: appCfg.isDailyResourceCleanUpDevelopmentMode ? "* * * * *" : "*/5 * * * *",
runHashTtlS: 3 * 24 * 60 * 60,
handlerTimeoutMs: CRON_HANDLER_TIMEOUT_MS,
leaseDurationMs: CRON_HANDLER_TIMEOUT_MS,
enabled: !appCfg.isSecondaryInstance,
handler: async () => {
const expiredProjects = await projectDAL.findExpiredForHardDelete(DISCOVERY_BATCH);
lastDiscoveryCount = expiredProjects.length;
if (expiredProjects.length === 0) return;
logger.info(`cron[${CronJobName.ProjectHardDelete}]: enqueuing ${expiredProjects.length} expired project(s)`);
await Promise.all(
expiredProjects.map((project) =>
queueService.queue(
QueueName.ProjectHardDelete,
QueueJobs.ProjectHardDelete,
{ projectId: project.id },
{
jobId: `project-hard-delete-${project.id}`,
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 5000 }
}
)
)
);
}
});
};
return { init };
};