Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

48.3 KB
Raw Blame
import { AxiosError, isAxiosError } from "axios";
import { Job } from "bullmq";
import { randomUUID } from "crypto";

import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayPoolServiceFactory } from "@app/ee/services/gateway-pool/gateway-pool-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { logger } from "@app/lib/logger";
import {
  highCardinalityMeter,
  recordSecretSyncOutcomeMetric,
  recordSecretSyncStatusChangeMetric
} from "@app/lib/telemetry/metrics";
import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification";
import { TriggerFeature } from "@app/lib/workflow-integrations/types";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { SecretNameSchema } from "@app/server/lib/schemas";
import { decryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns";
import { ActorType } from "@app/services/auth/auth-type";
import { TEventEmitter } from "@app/services/event-outbox/event-outbox-types";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TOidcIssuerServiceFactory } from "@app/services/oidc-issuer/oidc-issuer-service";
import { TOrgDALFactory } from "@app/services/org/org-dal";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TProjectFolderGrantDALFactory } from "@app/services/project-folder-grant/project-folder-grant-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { resolveAzureKeyVaultImportedSecretKey } from "@app/services/secret-sync/azure-key-vault/azure-key-vault-secret-name";
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
import {
  SecretSync,
  SecretSyncImportBehavior,
  SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
  SecretSyncAction,
  SecretSyncStatus,
  TQueueSecretSyncImportSecretsByIdDTO,
  TQueueSecretSyncRemoveSecretsByIdDTO,
  TQueueSecretSyncsByPathDTO,
  TQueueSecretSyncSyncSecretsByIdDTO,
  TQueueSendSecretSyncActionFailedNotificationsDTO,
  TSecretMap,
  TSecretSync,
  TSecretSyncImportSecretsDTO,
  TSecretSyncRaw,
  TSecretSyncRemoveSecretsDTO,
  TSecretSyncSyncSecretsDTO,
  TSecretSyncWithCredentials,
  TSendSecretSyncFailedNotificationsJobDTO,
  TSyncSecretsResult
} from "@app/services/secret-sync/secret-sync-types";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";

import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TGitHubAppDALFactory } from "../github-app/github-app-dal";
import { TMicrosoftTeamsServiceFactory } from "../microsoft-teams/microsoft-teams-service";
import { TProjectMicrosoftTeamsConfigDALFactory } from "../microsoft-teams/project-microsoft-teams-config-dal";
import { TNotificationServiceFactory } from "../notification/notification-service";
import { NotificationType } from "../notification/notification-types";
import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal";

const DEFAULT_SECRET_SYNC_RETRY_CONFIG = {
  attempts: 5,
  backoff: { type: "exponential" as const, delay: 3000 }
};

export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;

type TSecretSyncQueueFactoryDep = {
  queueService: Pick<TQueueServiceFactory, "queue" | "start">;
  cronJob: TCronJobFactory;
  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
  appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "incrementByAndRefreshExpiryIfUnderLimit" | "decrementByOrDelete">;
  gitHubAppDAL: Pick<TGitHubAppDALFactory, "findOne">;
  folderDAL: TSecretFolderDALFactory;
  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
  secretV2BridgeDAL: Pick<
    TSecretV2BridgeDALFactory,
    | "findByFolderId"
    | "findByFolderIds"
    | "find"
    | "insertMany"
    | "upsertSecretReferences"
    | "findBySecretKeys"
    | "bulkUpdate"
    | "deleteMany"
    | "invalidateSecretCacheByProjectId"
  >;
  secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds" | "findByIds">;
  secretSyncDAL: Pick<
    TSecretSyncDALFactory,
    "findById" | "find" | "findConnectionId" | "updateById" | "deleteById" | "update" | "updateAndReturnIds"
  >;
  auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
  projectDAL: TProjectDALFactory;
  smtpService: Pick<TSmtpService, "sendMail">;
  projectBotDAL: TProjectBotDALFactory;
  secretDAL: TSecretDALFactory;
  secretVersionDAL: TSecretVersionDALFactory;
  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
  secretTagDAL: TSecretTagDALFactory;
  secretVersionTagDAL: TSecretVersionTagDALFactory;
  secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
  secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
  resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
  folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
  eventEmitter: TEventEmitter;
  licenseService: Pick<TLicenseServiceFactory, "getPlan">;
  gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">;
  gatewayPoolService: Pick<TGatewayPoolServiceFactory, "resolveEffectiveGatewayId">;
  notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
  projectSlackConfigDAL: Pick<TProjectSlackConfigDALFactory, "getIntegrationDetailsByProject">;
  projectMicrosoftTeamsConfigDAL: Pick<TProjectMicrosoftTeamsConfigDALFactory, "getIntegrationDetailsByProject">;
  microsoftTeamsService: Pick<TMicrosoftTeamsServiceFactory, "sendNotification">;
  telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
  projectFolderGrantDAL: Pick<TProjectFolderGrantDALFactory, "find">;
  orgDAL: Pick<TOrgDALFactory, "findOrgById">;
  oidcIssuerService: Pick<TOidcIssuerServiceFactory, "signToken">;
};

type SecretSyncActionJob = Job<
  TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;

const JITTER_MS = 10 * 1000;
const REQUEUE_MS = 30 * 1000;
const REQUEUE_LIMIT = 120;
const CONNECTION_CONCURRENCY_LIMIT = 5;
const CONNECTION_CONCURRENCY_TTL_SECONDS = (REQUEUE_MS * REQUEUE_LIMIT) / 1000 / 2;

const getRequeueDelay = (failureCount?: number) => {
  const jitter = Math.random() * JITTER_MS;
  if (!failureCount) return jitter;
  return REQUEUE_MS + jitter;
};

export const secretSyncQueueFactory = ({
  queueService,
  cronJob,
  kmsService,
  appConnectionDAL,
  gitHubAppDAL,
  keyStore,
  folderDAL,
  projectEnvDAL,
  secretV2BridgeDAL,
  secretImportDAL,
  secretSyncDAL,
  auditLogService,
  projectMembershipDAL,
  projectDAL,
  smtpService,
  projectBotDAL,
  secretDAL,
  secretVersionDAL,
  secretBlindIndexDAL,
  secretTagDAL,
  secretVersionTagDAL,
  secretVersionV2BridgeDAL,
  secretVersionTagV2BridgeDAL,
  resourceMetadataDAL,
  folderCommitService,
  eventEmitter,
  licenseService,
  gatewayV2Service,
  gatewayPoolService,
  notificationService,
  projectSlackConfigDAL,
  projectMicrosoftTeamsConfigDAL,
  microsoftTeamsService,
  telemetryService,
  projectFolderGrantDAL,
  orgDAL,
  oidcIssuerService
}: TSecretSyncQueueFactoryDep) => {
  const appCfg = getConfig();

  const integrationMeter = highCardinalityMeter("SecretSyncs");
  const syncSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_sync_secrets_errors", {
    description: "Secret Sync - sync secrets errors",
    unit: "1"
  });
  const importSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_import_secrets_errors", {
    description: "Secret Sync - import secrets errors",
    unit: "1"
  });
  const removeSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_remove_secrets_errors", {
    description: "Secret Sync - remove secrets errors",
    unit: "1"
  });

  const $createManySecretsRawFn = createManySecretsRawFnFactory({
    projectDAL,
    projectBotDAL,
    secretDAL,
    secretVersionDAL,
    secretBlindIndexDAL,
    secretTagDAL,
    secretVersionTagDAL,
    folderDAL,
    kmsService,
    secretVersionV2BridgeDAL,
    secretV2BridgeDAL,
    secretVersionTagV2BridgeDAL,
    resourceMetadataDAL,
    folderCommitService,
    eventEmitter
  });

  const $updateManySecretsRawFn = updateManySecretsRawFnFactory({
    projectDAL,
    projectBotDAL,
    secretDAL,
    secretVersionDAL,
    secretBlindIndexDAL,
    secretTagDAL,
    secretVersionTagDAL,
    folderDAL,
    kmsService,
    secretVersionV2BridgeDAL,
    secretV2BridgeDAL,
    secretVersionTagV2BridgeDAL,
    resourceMetadataDAL,
    folderCommitService,
    eventEmitter
  });

  // INCR + cap check + EXPIRE happen atomically inside a Lua script. TTL is refreshed
  // only when a slot is actually claimed; failed probes (over-limit) and releases
  // never touch EXPIRE — so an orphaned counter from a worker that died mid-sync
  // can't have its expiry indefinitely shoved forward by subsequent admit attempts,
  // and ages out within CONNECTION_CONCURRENCY_TTL_SECONDS of the last successful
  // admit.
  const $tryAdmitConnectionConcurrency = async (connectionId: string) => {
    const key = KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId);
    const count = await keyStore.incrementByAndRefreshExpiryIfUnderLimit(
      key,
      CONNECTION_CONCURRENCY_LIMIT,
      CONNECTION_CONCURRENCY_TTL_SECONDS
    );
    return count !== -1;
  };

  const $releaseConnectionConcurrency = async (connectionId: string) => {
    const key = KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId);
    await keyStore.decrementByOrDelete(key);
  };

  const $getInfisicalSecrets = async (
    secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,
    includeImports = true
  ) => {
    const { projectId, folderId, environment, folder } = secretSync;

    if (!folderId || !environment || !folder)
      throw new SecretSyncError({
        message:
          "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path.",
        shouldRetry: false
      });

    const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
      type: KmsDataKey.SecretManager,
      projectId
    });
    const actorOrgId = secretSync.connection.orgId;

    const decryptSecretValue = (value?: Buffer | null) =>
      value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "";

    const { expandSecretReferences } = expandSecretReferencesFactory({
      decryptSecretValue,
      secretDAL: secretV2BridgeDAL,
      folderDAL,
      projectId,
      canExpandValue: () => true,
      actorOrgId,
      orgDAL,
      projectFolderGrantDAL,
      projectDAL,
      kmsService
    });

    return buildSyncPayload(
      {
        projectId,
        environment: environment.slug,
        sourcePath: folder.path,
        sourceFolderId: folderId,
        syncOptions: secretSync.syncOptions as TSecretSync["syncOptions"],
        includeImports
      },
      {
        folderDAL,
        projectEnvDAL,
        secretV2BridgeDAL,
        secretImportDAL,
        expandSecretReferences,
        decryptSecretValue,
        fnSecretsV2FromImportsDeps: {
          projectFolderGrantDAL,
          actorOrgId,
          orgDAL,
          kmsService
        }
      }
    );
  };

  const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {
    const { attempts, backoff } = DEFAULT_SECRET_SYNC_RETRY_CONFIG;
    return queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
      delay: getRequeueDelay(payload.failedToAcquireLockCount),
      attempts,
      backoff,
      removeOnComplete: true,
      removeOnFail: true,
      jobId: randomUUID()
    });
  };

  const queueSecretSyncImportSecretsById = async (payload: TQueueSecretSyncImportSecretsByIdDTO) =>
    queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncImportSecrets, payload, {
      attempts: 1,
      removeOnComplete: true,
      removeOnFail: true,

      jobId: randomUUID()
    });

  const queueSecretSyncRemoveSecretsById = async (payload: TQueueSecretSyncRemoveSecretsByIdDTO) =>
    queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncRemoveSecrets, payload, {
      attempts: 1,
      removeOnComplete: true,
      removeOnFail: true,
      jobId: randomUUID()
    });

  const $queueSendSecretSyncFailedNotifications = async (payload: TQueueSendSecretSyncActionFailedNotificationsDTO) => {
    if (!appCfg.isSmtpConfigured) return;

    await queueService.queue(
      QueueName.AppConnectionSecretSync,
      QueueJobs.SecretSyncSendActionFailedNotifications,
      payload,
      {
        jobId: `secret-sync-${payload.secretSync.id}-failed-notifications`,
        attempts: 5,
        delay: 1000 * 60,
        backoff: {
          type: "exponential",
          delay: 3000
        },
        removeOnFail: true,
        removeOnComplete: true
      }
    );
  };

  const $importSecrets = async (
    secretSync: TSecretSyncWithCredentials,
    importBehavior: SecretSyncImportBehavior
  ): Promise<TSecretMap> => {
    const {
      projectId,
      environment,
      folder,
      destination,
      connection: { orgId }
    } = secretSync;

    await enterpriseSyncCheck(
      licenseService,
      destination,
      orgId,
      "Failed to import secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
    );

    if (!environment || !folder)
      throw new Error(
        "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
      );

    const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
      appConnectionDAL,
      gitHubAppDAL,
      kmsService,
      gatewayV2Service,
      gatewayPoolService,
      oidcIssuerService
    });

    if (!Object.keys(importedSecrets).length) return {};

    let invalidNameCount = 0;
    let errorMessage = "";

    for (const [key] of Object.entries(importedSecrets)) {
      const result = SecretNameSchema.safeParse(key);
      if (!result.success) {
        invalidNameCount += 1;
        if (errorMessage === "") errorMessage = result.error.issues[0]?.message;
      }
    }

    if (invalidNameCount > 0) {
      throw new SecretSyncError({
        message: `Found ${invalidNameCount} invalid secret name${invalidNameCount === 1 ? "" : "s"}. ${errorMessage}`,
        shouldRetry: false
      });
    }

    const importedSecretMap: TSecretMap = {};

    // Import behavior is never combined with including subfolders, so this payload always covers
    // exactly one folder. Comparison is against raw keys as returned by the destination
    // (already schema-stripped), so entries come from `secrets` rather than the schema-applying flatten().
    const payload = await $getInfisicalSecrets(secretSync, false);
    const secretMap: TSecretMap = Object.fromEntries(
      payload.secrets.map((entry) => [
        entry.key,
        { value: entry.value, id: entry.id, comment: entry.comment, secretMetadata: entry.secretMetadata }
      ])
    );

    const secretsToCreate: Parameters<typeof $createManySecretsRawFn>[0]["secrets"] = [];
    const secretsToUpdate: Parameters<typeof $updateManySecretsRawFn>[0]["secrets"] = [];

    Object.entries(importedSecrets).forEach(([key, secretData]) => {
      const { value, comment = "", skipMultilineEncoding } = secretData;
      const importKey =
        destination === SecretSync.AzureKeyVault ? resolveAzureKeyVaultImportedSecretKey(key, secretMap) : key;

      const secret = {
        secretName: importKey,
        secretValue: value,
        type: SecretType.Shared,
        secretComment: comment,
        skipMultilineEncoding: skipMultilineEncoding ?? undefined
      };

      if (Object.hasOwn(secretMap, importKey)) {
        // Only update secrets if the source value is not empty
        if (value && value !== secretMap[importKey].value) {
          secretsToUpdate.push(secret);
          if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination)
            importedSecretMap[importKey] = secretData;
        }
      } else {
        secretsToCreate.push(secret);
        importedSecretMap[importKey] = secretData;
      }
    });

    if (secretsToCreate.length) {
      await $createManySecretsRawFn({
        projectId,
        path: folder.path,
        environment: environment.slug,
        secrets: secretsToCreate
      });
    }

    if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
      await $updateManySecretsRawFn({
        projectId,
        path: folder.path,
        environment: environment.slug,
        secrets: secretsToUpdate
      });
    }

    if (secretsToUpdate.length || secretsToCreate.length)
      await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);

    return importedSecretMap;
  };

  const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO, secretSync: TSecretSyncRaw) => {
    const {
      data: { syncId, auditLogInfo }
    } = job;

    await enterpriseSyncCheck(
      licenseService,
      secretSync.destination as SecretSync,
      secretSync.connection.orgId,
      "Failed to sync secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
    );

    await secretSyncDAL.updateById(
      syncId,
      {
        syncStatus: SecretSyncStatus.Running
      },
      { includeFolderPath: false }
    );

    logger.info(
      `SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
    );

    let isSynced = false;
    let syncMessage: string | null = null;
    let isFinalAttempt = job.attemptsStarted === job.opts.attempts;
    let syncResult: TSyncSecretsResult | undefined;

    try {
      const {
        connection: { orgId, encryptedCredentials, projectId }
      } = secretSync;

      const credentials = await decryptAppConnectionCredentials({
        orgId,
        encryptedCredentials,
        kmsService,
        projectId
      });

      const secretSyncWithCredentials = {
        ...secretSync,
        connection: {
          ...secretSync.connection,
          credentials
        }
      } as TSecretSyncWithCredentials;

      const {
        lastSyncedAt,
        syncOptions: { initialSyncBehavior }
      } = secretSyncWithCredentials;

      let payload = await $getInfisicalSecrets(secretSync);

      if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
        await $importSecrets(
          secretSyncWithCredentials,
          initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource
            ? SecretSyncImportBehavior.PrioritizeSource
            : SecretSyncImportBehavior.PrioritizeDestination
        );

        // $importSecrets writes directly to the source folder, so the payload built above is
        // stale. Re-read rather than merge in memory, since the payload is no longer a plain map.
        payload = await $getInfisicalSecrets(secretSync);
      }

      const result = await SecretSyncFns.syncSecrets(secretSyncWithCredentials, payload, {
        appConnectionDAL,
        gitHubAppDAL,
        kmsService,
        gatewayV2Service,
        gatewayPoolService,
        oidcIssuerService,
        keyStore
      });

      syncResult = result ?? undefined;
      syncMessage = syncResult?.message ?? null;

      isSynced = true;
    } catch (err) {
      const axiosError = err instanceof SecretSyncError && isAxiosError(err.error) ? err.error : err;

      logger.error(
        isAxiosError(axiosError) ? axiosError.response?.data : err,
        `SecretSync Sync Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        syncSecretsErrorHistogram.record(1, {
          version: 1,
          destination: secretSync.destination,
          syncId: secretSync.id,
          projectId: secretSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      syncMessage = parseSyncErrorMessage(err);

      if (err instanceof SecretSyncError && !err.shouldRetry) {
        isFinalAttempt = true;
      } else {
        // re-throw so job fails
        throw err;
      }
    } finally {
      const ranAt = new Date();
      const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;

      recordSecretSyncOutcomeMetric({
        destination: secretSync.destination,
        operation: "sync",
        outcome: isSynced ? "success" : "failure",
        attemptsExhausted: isFinalAttempt
      });

      await auditLogService.createAuditLog({
        projectId: secretSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.SECRET_SYNC_SYNC_SECRETS,
          metadata: {
            syncId: secretSync.id,
            syncOptions: secretSync.syncOptions,
            destination: secretSync.destination,
            destinationConfig: secretSync.destinationConfig,
            folderId: secretSync.folderId,
            connectionId: secretSync.connectionId,
            jobRanAt: ranAt,
            jobId: job.id!,
            syncStatus,
            syncMessage,
            createdSecretKeys: syncResult?.createdSecretKeys,
            updatedSecretKeys: syncResult?.updatedSecretKeys,
            deletedSecretKeys: syncResult?.deletedSecretKeys
          }
        }
      });

      if (isSynced || isFinalAttempt) {
        // lastSyncResult is written only here, so lock and admission failures cannot overwrite the provider's result.
        // The conditional update makes only the run that actually changes it report the change, whatever state this
        // job loaded before taking the lock. A sync that never succeeded (null) failing is a new misconfiguration, not
        // a breakage, so it is not counted.
        const isStatusChange =
          (
            await secretSyncDAL.update(
              {
                id: secretSync.id,
                lastSyncResult: isSynced ? SecretSyncStatus.Failed : SecretSyncStatus.Succeeded
              },
              { lastSyncResult: syncStatus }
            )
          ).length > 0;

        if (isStatusChange) {
          recordSecretSyncStatusChangeMetric({
            destination: secretSync.destination,
            newStatus: isSynced ? "healthy" : "failed"
          });
        }

        const updatedSecretSync = await secretSyncDAL.updateById(
          secretSync.id,
          {
            syncStatus,
            lastSyncJobId: job.id,
            lastSyncMessage: syncMessage,
            lastSyncedAt: isSynced ? ranAt : undefined,
            lastSyncResult: syncStatus
          },
          { includeFolderPath: false }
        );

        if (!isSynced) {
          void telemetryService
            .sendPostHogEvents({
              event: PostHogEventTypes.SecretSyncFailed,
              distinctId: `platform/${secretSync.projectId}`,
              organizationId: secretSync.connection.orgId,
              properties: {
                syncId: secretSync.id,
                syncDestination: secretSync.destination,
                projectId: secretSync.projectId
              }
            })
            .catch(() => {});

          await $queueSendSecretSyncFailedNotifications({
            secretSync: { ...updatedSecretSync, folder: secretSync.folder },
            action: SecretSyncAction.SyncSecrets,
            auditLogInfo
          });
        }
      }
    }

    logger.info("SecretSync Sync Job with ID %s Completed", job.id);
  };

  const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO, secretSync: TSecretSyncRaw) => {
    const {
      data: { syncId, auditLogInfo, importBehavior }
    } = job;

    await secretSyncDAL.update({ id: syncId }, { importStatus: SecretSyncStatus.Running });

    logger.info(
      `SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
    );

    let isSuccess = false;
    let importMessage: string | null = null;
    const isFinalAttempt = job.attemptsStarted === job.opts.attempts;

    try {
      const {
        connection: { orgId, encryptedCredentials, projectId }
      } = secretSync;

      const credentials = await decryptAppConnectionCredentials({
        orgId,
        encryptedCredentials,
        kmsService,
        projectId
      });

      await $importSecrets(
        {
          ...secretSync,
          connection: {
            ...secretSync.connection,
            credentials
          }
        } as TSecretSyncWithCredentials,
        importBehavior
      );

      isSuccess = true;
    } catch (err) {
      logger.error(
        err,
        `SecretSync Import Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        importSecretsErrorHistogram.record(1, {
          version: 1,
          destination: secretSync.destination,
          syncId: secretSync.id,
          projectId: secretSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      importMessage = parseSyncErrorMessage(err);

      // re-throw so job fails
      throw err;
    } finally {
      const ranAt = new Date();
      const importStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;

      recordSecretSyncOutcomeMetric({
        destination: secretSync.destination,
        operation: "import",
        outcome: isSuccess ? "success" : "failure",
        attemptsExhausted: isFinalAttempt
      });

      await auditLogService.createAuditLog({
        projectId: secretSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.SECRET_SYNC_IMPORT_SECRETS,
          metadata: {
            syncId: secretSync.id,
            syncOptions: secretSync.syncOptions,
            destination: secretSync.destination,
            destinationConfig: secretSync.destinationConfig,
            folderId: secretSync.folderId,
            connectionId: secretSync.connectionId,
            jobRanAt: ranAt,
            jobId: job.id!,
            importStatus,
            importMessage,
            importBehavior
          }
        }
      });

      if (isSuccess || isFinalAttempt) {
        const updatedSecretSync = await secretSyncDAL.updateById(
          secretSync.id,
          {
            importStatus,
            lastImportJobId: job.id,
            lastImportMessage: importMessage,
            lastImportedAt: isSuccess ? ranAt : undefined
          },
          { includeFolderPath: false }
        );

        if (!isSuccess) {
          await $queueSendSecretSyncFailedNotifications({
            secretSync: { ...updatedSecretSync, folder: secretSync.folder },
            action: SecretSyncAction.ImportSecrets,
            auditLogInfo
          });
        }
      }
    }

    logger.info("SecretSync Import Job with ID %s Completed", job.id);
  };

  const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO, secretSync: TSecretSyncRaw) => {
    const {
      data: { syncId, auditLogInfo, deleteSyncOnComplete }
    } = job;

    await enterpriseSyncCheck(
      licenseService,
      secretSync.destination as SecretSync,
      secretSync.connection.orgId,
      "Failed to remove secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
    );

    await secretSyncDAL.updateById(
      syncId,
      {
        removeStatus: SecretSyncStatus.Running
      },
      { includeFolderPath: false }
    );

    logger.info(
      `SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
    );

    let isSuccess = false;
    let removeMessage: string | null = null;
    const isFinalAttempt = job.attemptsStarted === job.opts.attempts;

    try {
      const {
        connection: { orgId, encryptedCredentials, projectId }
      } = secretSync;

      const credentials = await decryptAppConnectionCredentials({
        orgId,
        encryptedCredentials,
        kmsService,
        projectId
      });

      const payload = await $getInfisicalSecrets(secretSync, true);

      await SecretSyncFns.removeSecrets(
        {
          ...secretSync,
          connection: {
            ...secretSync.connection,
            credentials
          }
        } as TSecretSyncWithCredentials,
        // We don't care about duplicates when removing secrets: a duplicate name still maps to
        // the same destination key we're deleting either way.
        payload.dedupeConflicts(),
        {
          appConnectionDAL,
          gitHubAppDAL,
          kmsService,
          gatewayV2Service,
          gatewayPoolService,
          oidcIssuerService
        }
      );

      isSuccess = true;
    } catch (err) {
      logger.error(
        err,
        `SecretSync Remove Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        removeSecretsErrorHistogram.record(1, {
          version: 1,
          destination: secretSync.destination,
          syncId: secretSync.id,
          projectId: secretSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      removeMessage = parseSyncErrorMessage(err);

      // re-throw so job fails
      throw err;
    } finally {
      const ranAt = new Date();
      const removeStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;

      recordSecretSyncOutcomeMetric({
        destination: secretSync.destination,
        operation: "remove",
        outcome: isSuccess ? "success" : "failure",
        attemptsExhausted: isFinalAttempt
      });

      await auditLogService.createAuditLog({
        projectId: secretSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.SECRET_SYNC_REMOVE_SECRETS,
          metadata: {
            syncId: secretSync.id,
            syncOptions: secretSync.syncOptions,
            destination: secretSync.destination,
            destinationConfig: secretSync.destinationConfig,
            folderId: secretSync.folderId,
            connectionId: secretSync.connectionId,
            jobRanAt: ranAt,
            jobId: job.id!,
            removeStatus,
            removeMessage
          }
        }
      });

      if (isSuccess || isFinalAttempt) {
        if (isSuccess && deleteSyncOnComplete) {
          await secretSyncDAL.deleteById(secretSync.id);
        } else {
          const updatedSecretSync = await secretSyncDAL.updateById(
            secretSync.id,
            {
              removeStatus,
              lastRemoveJobId: job.id,
              lastRemoveMessage: removeMessage,
              lastRemovedAt: isSuccess ? ranAt : undefined
            },
            { includeFolderPath: false }
          );

          if (!isSuccess) {
            await $queueSendSecretSyncFailedNotifications({
              secretSync: { ...updatedSecretSync, folder: secretSync.folder },
              action: SecretSyncAction.RemoveSecrets,
              auditLogInfo
            });
          }
        }
      }
    }

    logger.info("SecretSync Remove Job with ID %s Completed", job.id);
  };

  const $sendSecretSyncFailedNotifications = async (job: TSendSecretSyncFailedNotificationsJobDTO) => {
    const {
      data: { secretSync, auditLogInfo, action }
    } = job;

    const { projectId, destination, name, folder, lastSyncMessage, lastRemoveMessage, lastImportMessage, environment } =
      secretSync;

    const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
    const project = await projectDAL.findById(projectId);

    let projectAdmins = projectMembers.filter((member) =>
      member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
    );

    const triggeredByUserId =
      auditLogInfo && auditLogInfo.actor.type === ActorType.USER && auditLogInfo.actor.metadata.userId;

    // only notify triggering user if triggered by admin
    if (triggeredByUserId && projectAdmins.map((admin) => admin.userId).includes(triggeredByUserId)) {
      projectAdmins = projectAdmins.filter((admin) => admin.userId === triggeredByUserId);
    }

    const syncDestination = SECRET_SYNC_NAME_MAP[destination as SecretSync];

    let actionLabel: string;
    let failureMessage: string | null | undefined;

    switch (action) {
      case SecretSyncAction.ImportSecrets:
        actionLabel = "Import";
        failureMessage = lastImportMessage;

        break;
      case SecretSyncAction.RemoveSecrets:
        actionLabel = "Remove";
        failureMessage = lastRemoveMessage;

        break;
      case SecretSyncAction.SyncSecrets:
      default:
        actionLabel = `Sync`;
        failureMessage = lastSyncMessage;
        break;
    }

    const baseProjectPath = `/organizations/${project.orgId}/projects/secret-management/${projectId}`;
    const overviewPath = `${baseProjectPath}/overview`;
    const syncPath = `${baseProjectPath}/integrations/secret-syncs/${destination}/${secretSync.id}`;

    const notifications = [
      triggerWorkflowIntegrationNotification({
        input: {
          notification: {
            type: TriggerFeature.SECRET_SYNC_ERROR,
            payload: {
              syncName: name,
              syncDestination,
              failureMessage: failureMessage || "An unknown error occurred",
              syncUrl: `${appCfg.SITE_URL}${syncPath}`,
              syncActionLabel: actionLabel,
              environment: environment?.name || "-",
              secretPath: folder?.path || "-",
              projectName: project.name,
              projectPath: overviewPath
            }
          },
          projectId
        },
        dependencies: {
          projectDAL,
          projectSlackConfigDAL,
          kmsService,
          microsoftTeamsService,
          projectMicrosoftTeamsConfigDAL
        }
      }),
      notificationService.createUserNotifications(
        projectAdmins.map((admin) => ({
          userId: admin.userId,
          orgId: project.orgId,
          type: NotificationType.SECRET_SYNC_FAILED,
          title: `Secret Sync Failed to ${actionLabel} Secrets`,
          body: `Your **${syncDestination}** sync **${name}** failed to complete${failureMessage ? `: \`${failureMessage}\`` : ""}`,
          link: syncPath
        }))
      ),
      smtpService.sendMail({
        recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
        template: SmtpTemplates.SecretSyncFailed,
        subjectLine: `Secret Sync Failed to ${actionLabel} Secrets`,
        substitutions: {
          syncName: name,
          syncDestination,
          content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
          failureMessage,
          secretPath: folder?.path,
          environment: environment?.name,
          projectName: project.name,
          syncUrl: `${appCfg.SITE_URL}${syncPath}`
        }
      })
    ];

    await Promise.allSettled(notifications);
  };

  // Takes a path rather than a folder, so it serves a write inside a folder and the deletion of the
  // folder itself. The path need not still exist: the only thing the folder is needed for is the
  // sync rooted exactly on it, and a deleted folder has no such sync left to find, because
  // secret_syncs.folderId is SET NULL by its foreign key.
  const queueSecretSyncsSyncSecretsByPath = async ({
    secretPath,
    projectId,
    environmentSlug
  }: TQueueSecretSyncsByPathDTO) => {
    const environment = await projectEnvDAL.findOne({ projectId, slug: environmentSlug });
    if (!environment) return;

    const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);

    const ancestorFolders = (
      await folderDAL.findByManySecretPath(
        getAncestorPaths(secretPath).map((path) => ({ envId: environment.id, secretPath: path }))
      )
    ).filter((entry): entry is NonNullable<typeof entry> => Boolean(entry));

    const candidateSyncs = await secretSyncDAL.find(
      {
        $in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },
        isAutoSyncEnabled: true
      },
      undefined,
      { includeFolderPath: false }
    );

    // A sync on the path itself always matches, whether or not it includes subfolders. A sync on an
    // ancestor folder only matches when it includes them, so a sync rooted above this path that does
    // not is never triggered by a change it was never configured to cover.
    // The move and copy warnings in frontend/src/helpers/secretSyncCoverage.ts mirror this rule.
    const secretSyncs = candidateSyncs.filter(
      (sync) =>
        (folder && sync.folderId === folder.id) ||
        Boolean((sync.syncOptions as TSecretSync["syncOptions"])?.includeAllSubFolders)
    );

    await secretSyncDAL.update(
      {
        $in: {
          id: secretSyncs.map((sync) => sync.id)
        }
      },
      {
        syncStatus: SecretSyncStatus.Pending
      }
    );

    await Promise.all(
      secretSyncs.map((secretSync) =>
        queueSecretSyncSyncSecretsById({
          syncId: secretSync.id
        })
      )
    );
  };

  const $handleAcquireLockFailure = async (job: SecretSyncActionJob) => {
    const { syncId, auditLogInfo } = job.data;

    switch (job.name) {
      case QueueJobs.SecretSyncSyncSecrets: {
        const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;

        if (failedToAcquireLockCount < REQUEUE_LIMIT) {
          await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
          return;
        }

        const secretSync = await secretSyncDAL.updateById(syncId, {
          syncStatus: SecretSyncStatus.Failed,
          lastSyncMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastSyncJobId: job.id
        });

        recordSecretSyncOutcomeMetric({
          destination: secretSync.destination,
          operation: "sync",
          outcome: "failure",
          attemptsExhausted: true
        });

        await $queueSendSecretSyncFailedNotifications({
          secretSync,
          action: SecretSyncAction.SyncSecrets,
          auditLogInfo
        });

        break;
      }
      // Scott: the two cases below are unlikely to happen as we check the lock at the API level but including this as a fallback
      case QueueJobs.SecretSyncImportSecrets: {
        const secretSync = await secretSyncDAL.updateById(syncId, {
          importStatus: SecretSyncStatus.Failed,
          lastImportMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastImportJobId: job.id
        });

        recordSecretSyncOutcomeMetric({
          destination: secretSync.destination,
          operation: "import",
          outcome: "failure",
          attemptsExhausted: true
        });

        await $queueSendSecretSyncFailedNotifications({
          secretSync,
          action: SecretSyncAction.ImportSecrets,
          auditLogInfo
        });

        break;
      }
      case QueueJobs.SecretSyncRemoveSecrets: {
        const secretSync = await secretSyncDAL.updateById(syncId, {
          removeStatus: SecretSyncStatus.Failed,
          lastRemoveMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastRemoveJobId: job.id
        });

        recordSecretSyncOutcomeMetric({
          destination: secretSync.destination,
          operation: "remove",
          outcome: "failure",
          attemptsExhausted: true
        });

        await $queueSendSecretSyncFailedNotifications({
          secretSync,
          action: SecretSyncAction.RemoveSecrets,
          auditLogInfo
        });

        break;
      }
      default:
        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
        throw new Error(`Unhandled Secret Sync Job ${job.name}`);
    }
  };

  const $handleDailySecretSyncRetry = async () => {
    const updatedIds = await secretSyncDAL.updateAndReturnIds(
      {
        syncStatus: SecretSyncStatus.Failed,
        isAutoSyncEnabled: true,
        $in: { destination: [...SECRET_SYNC_DAILY_RETRY_DESTINATIONS] }
      },
      { syncStatus: SecretSyncStatus.Pending }
    );

    if (!updatedIds.length) return;

    await Promise.all(
      updatedIds.map((syncId) =>
        queueSecretSyncSyncSecretsById({
          syncId
        })
      )
    );
  };

  queueService.start(QueueName.AppConnectionSecretSync, async (job) => {
    if (job.name === QueueJobs.SecretSyncSendActionFailedNotifications) {
      await $sendSecretSyncFailedNotifications(job as TSendSecretSyncFailedNotificationsJobDTO);
      return;
    }

    if (job.name === QueueJobs.DailySecretSyncRetry) {
      await $handleDailySecretSyncRetry();
      return;
    }

    const { syncId } = job.data as
      | TQueueSecretSyncSyncSecretsByIdDTO
      | TQueueSecretSyncImportSecretsByIdDTO
      | TQueueSecretSyncRemoveSecretsByIdDTO;

    const connectionId = await secretSyncDAL.findConnectionId(syncId);

    if (!connectionId) {
      // skip rather than throw, so it doesn't retry-storm when a sync is deleted.
      logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
      return;
    }

    let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;

    try {
      lock = await keyStore.acquireLock(
        [KeyStorePrefixes.SecretSyncLock(syncId)],
        // scott: not sure on this duration; syncs can take excessive amounts of time so we need to keep it locked,
        // but should always release below...
        5 * 60 * 1000
      );
    } catch (e) {
      logger.info(`SecretSync Failed to acquire lock [syncId=${syncId}] [job=${job.name}]`);

      await $handleAcquireLockFailure(job as SecretSyncActionJob);

      return;
    }

    let admittedConnectionSlot = false;

    try {
      if (job.name === QueueJobs.SecretSyncSyncSecrets) {
        admittedConnectionSlot = await $tryAdmitConnectionConcurrency(connectionId);

        if (!admittedConnectionSlot) {
          logger.info(
            `SecretSync Concurrency limit reached [syncId=${syncId}] [job=${job.name}] [connectionId=${connectionId}]`
          );

          await $handleAcquireLockFailure(job as SecretSyncActionJob);

          return;
        }
      }

      const secretSync = await secretSyncDAL.findById(syncId);

      if (!secretSync) {
        logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
        return;
      }

      if (secretSync.connectionId !== connectionId && job.name === QueueJobs.SecretSyncSyncSecrets) {
        logger.info(
          `SecretSync connection changed, releasing slot and requeueing [syncId=${syncId}] [job=${job.name}] [connectionId=${connectionId}] [currentConnectionId=${secretSync.connectionId}]`
        );

        await $handleAcquireLockFailure(job as SecretSyncActionJob);

        return;
      }

      switch (job.name) {
        case QueueJobs.SecretSyncSyncSecrets: {
          await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO, secretSync);
          break;
        }
        case QueueJobs.SecretSyncImportSecrets:
          await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO, secretSync);
          break;
        case QueueJobs.SecretSyncRemoveSecrets:
          await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO, secretSync);
          break;
        default:
          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
          throw new Error(`Unhandled Secret Sync Job ${job.name}`);
      }
    } finally {
      if (admittedConnectionSlot) {
        await $releaseConnectionConcurrency(connectionId);
      }

      await lock.release();
    }
  });

  const startDailySecretSyncRetryJob = () => {
    cronJob.register({
      name: CronJobName.DailySecretSyncRetry,
      pattern: appCfg.isDailyResourceCleanUpDevelopmentMode ? "*/5 * * * *" : "0 0 * * *",
      runHashTtlS: 3 * 24 * 60 * 60,
      handler: async () => {
        await queueService.queue(
          QueueName.AppConnectionSecretSync,
          QueueJobs.DailySecretSyncRetry,
          undefined as never,
          { jobId: CronJobName.DailySecretSyncRetry }
        );
      }
    });
  };

  return {
    queueSecretSyncSyncSecretsById,
    queueSecretSyncImportSecretsById,
    queueSecretSyncRemoveSecretsById,
    queueSecretSyncsSyncSecretsByPath,
    startDailySecretSyncRetryJob
  };
};