Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

13.7 KB
Raw Blame
import { AccessScope } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { logger } from "@app/lib/logger";
import { resolveCoreMeter } from "@app/lib/telemetry/metrics";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";

import { ActorType } from "../auth/auth-type";
import { TCaKeySourceServiceFactory } from "../certificate-authority/ca-key-source/ca-key-source-service";
import { TCertificateAuthoritySecretDALFactory } from "../certificate-authority/certificate-authority-secret-dal";
import { TKmsServiceFactory } from "../kms/kms-service";
import { TMembershipUserDALFactory } from "../membership-user/membership-user-dal";
import { TUserDALFactory } from "../user/user-dal";
import { TProjectDALFactory } from "./project-dal";

// ── load shaping ──────────────────────────────────────────────────────────────
// The cron tick is NOT the pacer — it only keeps the queue fed. Selecting the oldest
// DISCOVERY_BATCH expired projects (ORDER BY deleteAfter ASC) + jobId dedupe keeps the in-flight
// set bounded at ~DISCOVERY_BATCH regardless of backlog size (we never enqueue 85k at once).
// The worker meters the actual drain RATE via the rate limiter + concurrency, so DB load is a
// flat plateau rather than a per-tick spike.
const DISCOVERY_BATCH = 200;
const WORKER_CONCURRENCY = 3; // ceiling on simultaneous heavy deletes (fleet-wide)
const RATE_LIMIT_MAX = 30; // steady drain rate: 30 projects / minute
const RATE_LIMIT_DURATION_MS = 60 * 1000;

// per-project chunked secret_versions_v2 delete tuning
const SECRET_VERSION_DELETE_BATCH = 5000;
const BATCH_STATEMENT_TIMEOUT_MS = 30 * 1000;
const INTER_BATCH_SLEEP_MS = 10;

// Env count above which env deletion is delegated to the paced env hard-delete worker (one untimed
// cascade per env, per-env retries). At or below it, deleteById cascades the envs inline as today.
const ENV_DELEGATION_THRESHOLD = 100;

// Generous so a large project's chunked delete won't see the lock expire mid-flight and let a
// second worker start; crash recovery happens on the next cron tick after expiry.
const PROJECT_DELETE_LOCK_TTL_MS = 30 * 60 * 1000;
const CRON_HANDLER_TIMEOUT_MS = 5 * 60 * 1000;

type TProjectCleanupQueueFactoryDep = {
  projectDAL: Pick<
    TProjectDALFactory,
    | "findExpiredForHardDelete"
    | "findByIdIncludingExpired"
    | "findIncludingExpired"
    | "findProjectGhostUser"
    | "hardDeleteProjectSecretVersionsInBatches"
    | "hardDeleteProjectSecretReferencesInBatches"
    | "hardDeleteProjectApprovalSecretLinksInBatches"
    | "hardDeleteProjectSecretsInBatches"
    | "softDeleteProjectEnvironments"
    | "countProjectEnvironments"
    | "deleteById"
    | "transaction"
  >;
  membershipUserDAL: Pick<TMembershipUserDALFactory, "delete">;
  userDAL: Pick<TUserDALFactory, "deleteById">;
  kmsService: Pick<TKmsServiceFactory, "deleteInternalKms">;
  certificateAuthoritySecretDAL: Pick<TCertificateAuthoritySecretDALFactory, "findExternallyKeyedByProjectId">;
  caKeySourceService: Pick<TCaKeySourceServiceFactory, "prepareKeyReleases">;
  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "deleteItem">;
  auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
  queueService: Pick<TQueueServiceFactory, "queue" | "start">;
  cronJob: TCronJobFactory;
};

export type TProjectCleanupQueueFactory = ReturnType<typeof projectCleanupQueueFactory>;

export const projectCleanupQueueFactory = ({
  projectDAL,
  membershipUserDAL,
  userDAL,
  kmsService,
  certificateAuthoritySecretDAL,
  caKeySourceService,
  keyStore,
  auditLogService,
  queueService,
  cronJob
}: TProjectCleanupQueueFactoryDep) => {
  const appCfg = getConfig();

  if (appCfg.isDailyResourceCleanUpDevelopmentMode) {
    logger.warn("Project hard-delete cron is in development mode.");
  }

  // ── observability ─────────────────────────────────────────────────────────────
  const meter = resolveCoreMeter();
  // Per-pod, last-tick value: only the pod that won the cron redlock updates this; other pods report 0.
  // Alarms must aggregate with max() across pods, else a "stuck at cap" backlog gets diluted to ~0.
  let lastDiscoveryCount = 0;
  const discoveryGauge = meter.createObservableGauge("infisical.project_cleanup.discovered", {
    description:
      "Expired projects found on last discovery tick (capped at discovery batch). Sustained value at the cap = backlog remains.",
    unit: "{project}"
  });
  discoveryGauge.addCallback((observableResult) => {
    if (!getConfig().OTEL_TELEMETRY_COLLECTION_ENABLED) return;
    observableResult.observe(lastDiscoveryCount);
  });

  const processProjectHardDelete = async (projectId: string) => {
    let deletedVersions = 0;
    let deletedReferences = 0;
    let nulledApprovalLinks = 0;
    let deletedSecrets = 0;

    const lock = await keyStore
      .acquireLock([KeyStorePrefixes.ProjectDeleteLock(projectId)], PROJECT_DELETE_LOCK_TTL_MS)
      .catch(() => null);
    if (!lock) {
      logger.info(`project-hard-delete: lock held, will retry next firing [projectId=${projectId}]`);
      return;
    }

    try {
      // Read via transaction → primary, not replica. Defeats replica-lag races against a concurrent
      // worker that already finished.
      const project = await projectDAL.transaction((tx) => projectDAL.findByIdIncludingExpired(projectId, tx));
      if (!project || !project.deleteAfter || new Date(project.deleteAfter).getTime() > Date.now()) {
        logger.info(`project-hard-delete: skipping (gone/already-removed/not-yet-expired) [projectId=${projectId}]`);
        return;
      }

      // 1) Chunk-delete the largest project-scoped table ahead of the final cascade. secret_versions_v2
      // has no folderId/secretId FK and no other FK back to the project tree, so the project-delete
      // cascade would otherwise orphan all of these rows. Deleting by folderId is FK-safe and bounds the
      // final cascade's transaction size.
      await projectDAL.hardDeleteProjectSecretVersionsInBatches(
        projectId,
        SECRET_VERSION_DELETE_BATCH,
        BATCH_STATEMENT_TIMEOUT_MS,
        INTER_BATCH_SLEEP_MS,
        (deleted) => {
          deletedVersions += deleted;
        }
      );

      // 2) Same treatment for the two statements that actually time out inside the final cascade on a
      // project with a large secret graph: the secret_references_v2 CASCADE and the
      // secret_approval_requests_secrets_v2 SET NULL. Then the secrets themselves, so their remaining
      // per-secret CASCADE children run on a batch-sized statement instead of the whole project.
      // Order matters: deleting secrets first would just drag both cascades back into one statement.
      await projectDAL.hardDeleteProjectSecretReferencesInBatches(
        projectId,
        SECRET_VERSION_DELETE_BATCH,
        BATCH_STATEMENT_TIMEOUT_MS,
        INTER_BATCH_SLEEP_MS,
        (deleted) => {
          deletedReferences += deleted;
        }
      );
      await projectDAL.hardDeleteProjectApprovalSecretLinksInBatches(
        projectId,
        SECRET_VERSION_DELETE_BATCH,
        BATCH_STATEMENT_TIMEOUT_MS,
        INTER_BATCH_SLEEP_MS,
        (deleted) => {
          nulledApprovalLinks += deleted;
        }
      );
      await projectDAL.hardDeleteProjectSecretsInBatches(
        projectId,
        SECRET_VERSION_DELETE_BATCH,
        BATCH_STATEMENT_TIMEOUT_MS,
        INTER_BATCH_SLEEP_MS,
        (deleted) => {
          deletedSecrets += deleted;
        }
      );

      // 3) For big projects: mark envs for the env worker; the cron re-fires this job until drained.
      const envCount = await projectDAL.countProjectEnvironments(projectId);
      if (envCount > ENV_DELEGATION_THRESHOLD) {
        const marked = await projectDAL.softDeleteProjectEnvironments(projectId);
        logger.info(
          { projectId, envCount, marked, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
          `project-hard-delete: waiting on env drain [projectId=${projectId}] [envsRemaining=${envCount}] [newlyMarked=${marked}]`
        );
        return;
      }

      const releaseCaKeys = await caKeySourceService.prepareKeyReleases(
        await certificateAuthoritySecretDAL.findExternallyKeyedByProjectId(projectId)
      );

      // 4) Final cascade in one transaction — handles the remaining (smaller) child tables, including
      // deferred / NO ACTION FKs (e.g. secret_rotation_v2_secret_mappings) that PG resolves correctly
      // as a single cascade tree (the same proven path the synchronous delete used).
      const deleted = await projectDAL.transaction(async (tx) => {
        // capture the ghost user before its project membership is removed below
        const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx).catch(() => null);

        await membershipUserDAL.delete(
          { scopeOrgId: project.orgId, scopeProjectId: projectId, scope: AccessScope.Project },
          tx
        );

        const delProject = await projectDAL.deleteById(projectId, tx);
        // a racing worker may have hard-deleted it after our re-read but before we acquired the row
        if (!delProject) return false;

        // before removing a KMS key, ensure no other (incl. pending-deletion) project references it
        if (delProject.kmsCertificateKeyId) {
          const linked = await projectDAL.findIncludingExpired(
            { kmsCertificateKeyId: delProject.kmsCertificateKeyId },
            { tx }
          );
          if (!linked.length) await kmsService.deleteInternalKms(delProject.kmsCertificateKeyId, delProject.orgId, tx);
        }
        if (delProject.kmsSecretManagerKeyId) {
          const linked = await projectDAL.findIncludingExpired(
            { kmsSecretManagerKeyId: delProject.kmsSecretManagerKeyId },
            { tx }
          );
          if (!linked.length)
            await kmsService.deleteInternalKms(delProject.kmsSecretManagerKeyId, delProject.orgId, tx);
        }

        if (ghostUser) await userDAL.deleteById(ghostUser.id, tx);
        return true;
      });

      if (!deleted) {
        logger.info(
          `project-hard-delete: hard-deleted project was already removed by a concurrent worker [projectId=${projectId}]`
        );
        return;
      }

      await releaseCaKeys();
      await keyStore.deleteItem(KeyStorePrefixes.LicenseCloudPlan(project.orgId));
      await auditLogService.createAuditLog({
        orgId: project.orgId,
        projectId,
        actor: { type: ActorType.PLATFORM, metadata: {} },
        event: {
          type: EventType.DELETE_PROJECT,
          metadata: { id: projectId, name: project.name, softDelete: false }
        }
      });

      logger.info(
        { projectId, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
        `project-hard-delete: hard-deleted project [projectId=${projectId}] [versionsPruned=${deletedVersions}] [referencesPruned=${deletedReferences}] [approvalLinksNulled=${nulledApprovalLinks}] [secretsPruned=${deletedSecrets}]`
      );
    } catch (err) {
      logger.error(
        { err, projectId, deletedVersions, deletedReferences, nulledApprovalLinks, deletedSecrets },
        `project-hard-delete: failed [projectId=${projectId}] [versionsPruned=${deletedVersions}] [referencesPruned=${deletedReferences}] [approvalLinksNulled=${nulledApprovalLinks}] [secretsPruned=${deletedSecrets}]`
      );
      throw err; // surface to BullMQ so it retries (attempts: 3) and the generic failure metric fires
    } finally {
      await lock.release();
    }
  };

  const init = () => {
    // Worker: meters the actual drain rate. concurrency caps parallelism; the limiter caps throughput
    // (fleet-wide via Redis) so a backlog drains as an even plateau instead of a burst.
    queueService.start(
      QueueName.ProjectHardDelete,
      async (job) => {
        await processProjectHardDelete(job.data.projectId);
      },
      { concurrency: WORKER_CONCURRENCY, limiter: { max: RATE_LIMIT_MAX, duration: RATE_LIMIT_DURATION_MS } }
    );

    // Cron tick: discovery only (fast). Enqueues the oldest expired projects; jobId dedupe keeps the
    // queue bounded and idempotent across ticks.
    cronJob.register({
      name: CronJobName.ProjectHardDelete,
      pattern: appCfg.isDailyResourceCleanUpDevelopmentMode ? "* * * * *" : "*/5 * * * *",
      runHashTtlS: 3 * 24 * 60 * 60,
      handlerTimeoutMs: CRON_HANDLER_TIMEOUT_MS,
      leaseDurationMs: CRON_HANDLER_TIMEOUT_MS,
      enabled: !appCfg.isSecondaryInstance,
      handler: async () => {
        const expiredProjects = await projectDAL.findExpiredForHardDelete(DISCOVERY_BATCH);
        lastDiscoveryCount = expiredProjects.length;
        if (expiredProjects.length === 0) return;

        logger.info(`cron[${CronJobName.ProjectHardDelete}]: enqueuing ${expiredProjects.length} expired project(s)`);
        await Promise.all(
          expiredProjects.map((project) =>
            queueService.queue(
              QueueName.ProjectHardDelete,
              QueueJobs.ProjectHardDelete,
              { projectId: project.id },
              {
                jobId: `project-hard-delete-${project.id}`,
                removeOnComplete: true,
                removeOnFail: true,
                attempts: 3,
                backoff: { type: "exponential", delay: 5000 }
              }
            )
          )
        );
      }
    });
  };

  return { init };
};