Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
feat: optimized get secrets heap allocation
=8633eddb6363
diff --git a/backend/package-lock.json b/backend/package-lock.jsonindex 894c2b3ccf..325e632f49 100644--- a/backend/package-lock.json+++ b/backend/package-lock.json@@ -93,6 +93,7 @@"dd-trace": "^5.128.0","dns-packet": "^5.6.1","dotenv": "^16.4.1",+ "fast-json-stringify": "^7.1.3","fast-xml-parser": "^5.7.0","fastify": "^5.12.7","fastify-plugin": "^5.0.1",diff --git a/backend/package.json b/backend/package.jsonindex 178cda5cc1..0e030ce29d 100644--- a/backend/package.json+++ b/backend/package.json@@ -238,6 +238,7 @@"dd-trace": "^5.128.0","dns-packet": "^5.6.1","dotenv": "^16.4.1",+ "fast-json-stringify": "^7.1.3","fast-xml-parser": "^5.7.0","fastify": "^5.12.7","fastify-plugin": "^5.0.1",diff --git a/backend/src/ee/services/permission/permission-fns.test.ts b/backend/src/ee/services/permission/permission-fns.test.tsindex eaa955e71a..842b5898a4 100644--- a/backend/src/ee/services/permission/permission-fns.test.ts+++ b/backend/src/ee/services/permission/permission-fns.test.ts@@ -1,4 +1,4 @@-import { createMongoAbility, MongoAbility, RawRuleOf, subject } from "@casl/ability";+import { createMongoAbility, ForbiddenError, MongoAbility, RawRuleOf, subject } from "@casl/ability";import { describe, expect, test } from "vitest";import { conditionsMatcher } from "@app/lib/casl";@@ -235,6 +235,21 @@ describe("throwIfMissingSecretReadValueOrDescribePermission", () => {).toThrow();});+ test("throws a ForbiddenError for the requested action, not the legacy one", () => {+ const ability = buildAbility([]);+ let thrown: unknown;+ try {+ throwIfMissingSecretReadValueOrDescribePermission(ability, ProjectPermissionSecretActions.ReadValue, {+ environment: "prod",+ secretPath: "/"+ });+ } catch (error) {+ thrown = error;+ }+ expect(thrown).toBeInstanceOf(ForbiddenError);+ expect((thrown as ForbiddenError<MongoAbility>).action).toBe(ProjectPermissionSecretActions.ReadValue);+ });+test("respects conditions on the DescribeSecret rule", () => {const ability = buildAbility([allow({diff --git a/backend/src/ee/services/permission/permission-fns.ts b/backend/src/ee/services/permission/permission-fns.tsindex d520dd5e15..9d070c160e 100644--- a/backend/src/ee/services/permission/permission-fns.ts+++ b/backend/src/ee/services/permission/permission-fns.ts@@ -42,25 +42,17 @@ export function throwIfMissingSecretReadValueOrDescribePermission(>,subjectFields?: SecretSubjectFields) {- try {- if (subjectFields) {- ForbiddenError.from(permission).throwUnlessCan(- ProjectPermissionSecretActions.DescribeAndReadValue,- subject(ProjectPermissionSub.Secrets, subjectFields)- );- } else {- ForbiddenError.from(permission).throwUnlessCan(- ProjectPermissionSecretActions.DescribeAndReadValue,- ProjectPermissionSub.Secrets- );- }- } catch {- if (subjectFields) {- ForbiddenError.from(permission).throwUnlessCan(action, subject(ProjectPermissionSub.Secrets, subjectFields));- } else {- ForbiddenError.from(permission).throwUnlessCan(action, ProjectPermissionSub.Secrets);- }+ const secretSubject = subjectFields+ ? subject(ProjectPermissionSub.Secrets, subjectFields)+ : ProjectPermissionSub.Secrets;+ // ForbiddenError captures a stack trace, so only build one when access is actually denied.+ if (+ permission.can(ProjectPermissionSecretActions.DescribeAndReadValue, secretSubject) ||+ permission.can(action, secretSubject)+ ) {+ return;}+ ForbiddenError.from(permission).throwUnlessCan(action, secretSubject);}export function hasSecretReadValueOrDescribePermission(diff --git a/backend/src/lib/casl/index.test.ts b/backend/src/lib/casl/index.test.tsnew file mode 100644index 0000000000..4a439ed9e8--- /dev/null+++ b/backend/src/lib/casl/index.test.ts@@ -0,0 +1,41 @@+import picomatch from "picomatch";+import { describe, expect, test } from "vitest";++import { getGlobMatcher, MAX_CACHED_GLOB_MATCHERS } from "./index";++describe("getGlobMatcher", () => {+ test("matches the same paths as picomatch.isMatch", () => {+ const patterns = ["/svc-*/**", "/**", "/app/*", "/app/{api,web}/**", "/exact"];+ const paths = [+ "/",+ "/svc-001",+ "/svc-001/cmp-1",+ "/svc-001/a/b",+ "/app",+ "/app/api",+ "/app/api/x",+ "/exact",+ "/exact/"+ ];++ for (const pattern of patterns) {+ for (const path of paths) {+ expect(getGlobMatcher(pattern)(path)).toBe(picomatch.isMatch(path, pattern, { strictSlashes: false }));+ }+ }+ });++ test("reuses the compiled matcher for a pattern", () => {+ expect(getGlobMatcher("/reused/**")).toBe(getGlobMatcher("/reused/**"));+ });++ test("evicts the oldest pattern once the cache is full", () => {+ const oldest = getGlobMatcher("/oldest/**");+ for (let i = 0; i < MAX_CACHED_GLOB_MATCHERS; i += 1) getGlobMatcher(`/filler-${i}/**`);++ expect(getGlobMatcher("/oldest/**")).not.toBe(oldest);+ expect(getGlobMatcher(`/filler-${MAX_CACHED_GLOB_MATCHERS - 1}/**`)).toBe(+ getGlobMatcher(`/filler-${MAX_CACHED_GLOB_MATCHERS - 1}/**`)+ );+ });+});diff --git a/backend/src/lib/casl/index.ts b/backend/src/lib/casl/index.tsindex c7305c5ab1..7a8dbf4bd8 100644--- a/backend/src/lib/casl/index.ts+++ b/backend/src/lib/casl/index.ts@@ -12,12 +12,27 @@ const $glob: FieldInstruction<string> = {}};+// Patterns come from customer roles, so the cache is bounded; the oldest entry is evicted first.+export const MAX_CACHED_GLOB_MATCHERS = 1000;+const globMatchers = new Map<string, (path: string) => boolean>();++// picomatch.isMatch compiles the pattern into a regex on every call, and a secret list runs several checks per secret.+export const getGlobMatcher = (pattern: string) => {+ let matcher = globMatchers.get(pattern);+ if (!matcher) {+ matcher = picomatch(pattern, { strictSlashes: false });+ if (globMatchers.size >= MAX_CACHED_GLOB_MATCHERS) globMatchers.delete(globMatchers.keys().next().value as string);+ globMatchers.set(pattern, matcher);+ }+ return matcher;+};+const glob: JsInterpreter<FieldCondition<string>> = (node, object, context) => {const secretPath = context.get(object, node.field) as string;const permissionSecretGlobPath = node.value;if (permissionSecretGlobPath.trim() === "") return false;if (typeof secretPath !== "string") return false;- return picomatch.isMatch(secretPath, permissionSecretGlobPath, { strictSlashes: false });+ return getGlobMatcher(permissionSecretGlobPath)(secretPath);};export const conditionsMatcher = buildMongoQueryMatcher({ $glob }, { glob });diff --git a/backend/src/lib/knex/join.ts b/backend/src/lib/knex/join.tsindex b9043f7429..1a5ea28f5c 100644--- a/backend/src/lib/knex/join.ts+++ b/backend/src/lib/knex/join.ts@@ -125,10 +125,19 @@ export const sqlNestRelationships = <type Cm = MappedRecord<(typeof childrenMapper)[number]>;const recordsGroupedByPk: Record<string, P & Cm> = {};+ const childLabels = childrenMapper.map(({ label }) => label);data.forEach((doc) => {const pk = doc[key];if (!lookupTable.has(pk)) {- recordsGroupedByPk[pk] = parentMapper(doc) as P & Cm;+ const record = parentMapper(doc) as P & Cm;+ // Children are added up front: adding them later by computed key switches every record to slow+ // dictionary-mode objects, which cost several times the memory to hold and to copy.+ recordsGroupedByPk[pk] = {+ ...record,+ ...Object.fromEntries(+ childLabels.filter((label) => !record[label as keyof typeof record]).map((label) => [label, []])+ )+ };recordsOrder.push(pk);lookupTable.add(pk);}diff --git a/backend/src/server/plugins/fastify-zod.test.ts b/backend/src/server/plugins/fastify-zod.test.tsnew file mode 100644index 0000000000..244189a631--- /dev/null+++ b/backend/src/server/plugins/fastify-zod.test.ts@@ -0,0 +1,83 @@+import Fastify from "fastify";+import { describe, expect, test } from "vitest";+import { z } from "zod";++import { fastJsonSerializerCompiler, serializerCompiler, validatorCompiler, ZodTypeProvider } from "./fastify-zod";++const responseSchema = z.object({+ secrets: z+ .object({+ id: z.string(),+ secretValue: z.string(),+ reminderNote: z.string().nullable().optional(),+ skipMultilineEncoding: z.boolean().default(false).nullable().optional(),+ createdAt: z.date(),+ tags: z.object({ id: z.string(), slug: z.string() }).array().optional()+ })+ .array(),+ imports: z.object({ secretPath: z.string() }).array().optional()+});++const handlerData = {+ secrets: [+ {+ id: "a",+ secretValue: "v1",+ reminderNote: null,+ createdAt: new Date("2026-10-30T13:00:00.000Z"),+ tags: [{ id: "t1", slug: "prod", color: "#fff" }],+ encryptedValue: Buffer.from("must not leak"),+ userId: "u1"+ },+ { id: "b", secretValue: "v2", skipMultilineEncoding: true, createdAt: new Date("2026-10-30T13:01:00.000Z") }+ ],+ imports: [],+ internalFlag: true+};++const buildServer = () => {+ const app = Fastify({ logger: false }).withTypeProvider<ZodTypeProvider>();+ app.setValidatorCompiler(validatorCompiler);+ app.setSerializerCompiler(serializerCompiler);++ app.get("/zod", { schema: { response: { 200: responseSchema } } }, async () => handlerData as never);+ app.get(+ "/fast",+ { serializerCompiler: fastJsonSerializerCompiler, schema: { response: { 200: responseSchema } } },+ async () => handlerData as never+ );+ app.get(+ "/fast-not-found",+ {+ serializerCompiler: fastJsonSerializerCompiler,+ schema: { response: { 200: responseSchema, 404: z.object({ message: z.string() }) } }+ },+ async (_, reply) => reply.code(404).send({ message: "not found", stack: "internal" } as never)+ );+ return app;+};++type TBody = { secrets: { tags?: unknown[]; createdAt: string }[] };++describe("fastJsonSerializerCompiler", () => {+ test("produces the same body as the zod serializer, dropping fields the schema does not list", async () => {+ const app = buildServer();+ const [zodRes, fastRes] = await Promise.all([app.inject("/zod"), app.inject("/fast")]);++ expect(fastRes.statusCode).toBe(200);+ expect(fastRes.json()).toEqual(zodRes.json());+ expect(fastRes.body).not.toContain("must not leak");+ expect(fastRes.body).not.toContain("internalFlag");+ const [firstSecret] = fastRes.json<TBody>().secrets;+ expect(firstSecret.tags).toEqual([{ id: "t1", slug: "prod" }]);+ expect(firstSecret.createdAt).toBe("2026-10-30T13:00:00.000Z");+ });++ test("leaves non-2xx responses on the zod serializer", async () => {+ const app = buildServer();+ const res = await app.inject("/fast-not-found");++ expect(res.statusCode).toBe(404);+ expect(res.json()).toEqual({ message: "not found" });+ });+});diff --git a/backend/src/server/plugins/fastify-zod.ts b/backend/src/server/plugins/fastify-zod.tsindex d1cc34c311..dcc71d2f10 100644--- a/backend/src/server/plugins/fastify-zod.ts+++ b/backend/src/server/plugins/fastify-zod.ts@@ -1,3 +1,4 @@+import fastJsonStringify, { type Schema as FastJsonSchema } from "fast-json-stringify";/* eslint-disable */// Code taken from https://www.npmjs.com/package/fastify-type-provider-zod// Full credits goes to https://github.com/turkerdev@@ -304,3 +305,16 @@ export const serializerCompiler: FastifySerializerCompiler<ZodAny | { propertiesthrow new ResponseValidationError(result);};++// For large responses: compiles the schema once instead of zod-parsing every response. fast-json-stringify+// writes only the properties the schema lists, so unlisted fields are still dropped.+// Fastify types a route-level compiler with the whole route schema, but calls it with each status's response schema.+export const fastJsonSerializerCompiler: FastifySerializerCompiler<unknown> = (routeSchema) => {+ const responseRouteSchema = routeSchema as Parameters<typeof serializerCompiler>[0];+ if (!routeSchema.httpStatus?.startsWith("2")) return serializerCompiler(responseRouteSchema);++ const zodSchema = resolveSchema(responseRouteSchema.schema) as ZodAny;+ return fastJsonStringify(+ zodToJsonSchema(zodSchema, { target: "jsonSchema7", $refStrategy: "none" }) as FastJsonSchema+ );+};diff --git a/backend/src/server/routes/v3/deprecated-secret-router.ts b/backend/src/server/routes/v3/deprecated-secret-router.tsindex 758010f170..05605b3344 100644--- a/backend/src/server/routes/v3/deprecated-secret-router.ts+++ b/backend/src/server/routes/v3/deprecated-secret-router.ts@@ -13,6 +13,7 @@ import { BaseSecretNameSchema, SecretNameSchema } from "@app/server/lib/schemas"import { getTelemetryDistinctId } from "@app/server/lib/telemetry";import { getUserAgentType } from "@app/server/plugins/audit-log";import { verifyAuth } from "@app/server/plugins/auth/verify-auth";+import { fastJsonSerializerCompiler } from "@app/server/plugins/fastify-zod";import { ActorType, AuthMode } from "@app/services/auth/auth-type";import { ResourceMetadataWithEncryptionSchema } from "@app/services/resource-metadata/resource-metadata-schema";import { SecretUpdateMode } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";@@ -176,6 +177,7 @@ export const registerDeprecatedSecretRouter = async (server: FastifyZodProvider)config: {rateLimit: secretsLimit},+ serializerCompiler: fastJsonSerializerCompiler,schema: {deprecated: true,description: "Deprecated. Use GET /api/v4/secrets instead.",diff --git a/backend/src/server/routes/v4/secret-router.ts b/backend/src/server/routes/v4/secret-router.tsindex e4e17e0087..dec4044600 100644--- a/backend/src/server/routes/v4/secret-router.ts+++ b/backend/src/server/routes/v4/secret-router.ts@@ -13,6 +13,7 @@ import { BaseSecretNameSchema, SecretNameSchema } from "@app/server/lib/schemas"import { getTelemetryDistinctId } from "@app/server/lib/telemetry";import { getUserAgentType } from "@app/server/plugins/audit-log";import { verifyAuth } from "@app/server/plugins/auth/verify-auth";+import { fastJsonSerializerCompiler } from "@app/server/plugins/fastify-zod";import { ActorType, AuthMode } from "@app/services/auth/auth-type";import { ResourceMetadataWithEncryptionSchema } from "@app/services/resource-metadata/resource-metadata-schema";import { SecretUpdateMode } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";@@ -69,6 +70,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {config: {rateLimit: secretsLimit},+ serializerCompiler: fastJsonSerializerCompiler,schema: {hide: false,operationId: "listSecretsV4",diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.tsindex 188a6db3a0..4875b592b2 100644--- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts+++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts@@ -48,6 +48,16 @@ interface TSecretV2DalArg {export const SECRET_DAL_TTL = () => applyJitter(10 * 60, 2 * 60);export const SECRET_DAL_VERSION_TTL = "15m";export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;++const SECRET_V2_COLUMNS = Object.keys(SecretsV2Schema.shape) as (keyof TSecretsV2)[];++// Same output as SecretsV2Schema.parse for a row selected with selectAllTableCols, without parsing every row.+const pickSecretV2Columns = (row: TSecretsV2) => {+ const secret: Record<string, unknown> = {};+ for (const column of SECRET_V2_COLUMNS) secret[column] = row[column];+ return secret as TSecretsV2;+};+export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {const secretOrm = ormify(db, TableName.SecretV2);@@ -895,7 +905,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {key: "id",parentMapper: (el) => ({_id: el.id,- ...SecretsV2Schema.parse(el),+ ...pickSecretV2Columns(el),isHoneyTokenSecret: Boolean(el.honeyTokenId),rotationId: el.rotationId,isRotatedSecret: Boolean(el.rotationId)diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.tsindex 8d2107b673..19bc0620d5 100644--- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts+++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts@@ -1555,74 +1555,53 @@ export const secretV2BridgeServiceFactory = ({}// scott: if any of this changes it also needs to be mirrored in secret rotation for getting dashboard secrets- const decryptedSecrets = secrets- .filter((el) => {- const canDescribeSecret = hasSecretReadValueOrDescribePermission(+ const decryptedSecrets: ReturnType<typeof reshapeBridgeSecret>[] = [];+ for (const secret of secrets) {+ const secretPath = groupedPaths[secret.folderId][0].path;+ const secretSubjectFields = {+ environment,+ secretPath,+ secretName: secret.key,+ secretTags: secret.tags.map((i) => i.slug)+ };++ const canDescribeSecret = hasSecretReadValueOrDescribePermission(+ permission,+ ProjectPermissionSecretActions.DescribeSecret,+ secretSubjectFields+ );+ if (!canDescribeSecret) continue;++ const canReadValue =+ viewSecretValue &&+ hasSecretReadValueOrDescribePermission(permission,- ProjectPermissionSecretActions.DescribeSecret,- {- environment,- secretPath: groupedPaths[el.folderId][0].path,- secretName: el.key,- secretTags: el.tags.map((i) => i.slug)- }+ ProjectPermissionSecretActions.ReadValue,+ secretSubjectFields);- if (!canDescribeSecret) {- return false;- }+ if (viewSecretValue && !canReadValue) {+ // Recursive secret, should be filtered out+ if (secretPath !== path) continue;- if (viewSecretValue) {- // Recursive secret, should be filtered out- if (groupedPaths[el.folderId][0].path !== path) {- const canReadRecursiveSecretValue = hasSecretReadValueOrDescribePermission(- permission,- ProjectPermissionSecretActions.ReadValue,- {- environment,- secretPath: groupedPaths[el.folderId][0].path,- secretName: el.key,- secretTags: el.tags.map((i) => i.slug)- }- );-- if (!canReadRecursiveSecretValue) {- return false;- }- }-- if (throwOnMissingReadValuePermission) {- throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.ReadValue, {- environment,- secretPath: groupedPaths[el.folderId][0].path,- secretName: el.key,- secretTags: el.tags.map((i) => i.slug)- });- }- // Else, we do nothing. Because we don't want to filter out the secret, OR throw an error.- // If the user doesn't have access to read the value, in the below map function, we mask the secret value and return the secret with a hidden value.+ if (throwOnMissingReadValuePermission) {+ throwIfMissingSecretReadValueOrDescribePermission(+ permission,+ ProjectPermissionSecretActions.ReadValue,+ secretSubjectFields+ );}+ // Otherwise keep the secret with its value masked below.+ }- return canDescribeSecret;- })- .map((secret) => {- const isPersonalSecret = secret.userId === actorId && secret.type === SecretType.Personal;-- const secretValueHidden =- !viewSecretValue ||- !hasSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.ReadValue, {- environment,- secretPath: groupedPaths[secret.folderId][0].path,- secretName: secret.key,- secretTags: secret.tags.map((i) => i.slug)- });-- const isValueMasked = secretValueHidden && !isPersonalSecret;+ const isPersonalSecret = secret.userId === actorId && secret.type === SecretType.Personal;+ const isValueMasked = !canReadValue && !isPersonalSecret;- return reshapeBridgeSecret(+ decryptedSecrets.push(+ reshapeBridgeSecret(projectId,environment,- groupedPaths[secret.folderId][0].path,+ secretPath,{...secret,secretMetadata: secret.secretMetadata?.map((el) => ({@@ -1641,8 +1620,9 @@ export const secretV2BridgeServiceFactory = ({: ""},isValueMasked- );- });+ )+ );+ }// note(daniel): when in relative mode, direct secret references also resolve through imported secrets.// wrap secretDAL with import-aware fetch so the expander sees the full merged view.@@ -1695,7 +1675,8 @@ export const secretV2BridgeServiceFactory = ({if (shouldExpandSecretReferences) {throwIfClientDisconnected(abortSignal);const errors = await expandSecretReferencesGroupedByPath({- secrets: decryptedSecrets,+ // expandSecretReferences returns values without "${" unchanged, so skip them rather than await each one+ secrets: decryptedSecrets.filter((secret) => secret.secretValue.includes("${")),environment,expandSecretReferences});