Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(llma): read evaluation scheduler codec key from temporal env vars only (#129744)

Tom Piccirello9291251cef3f
diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.yml
index 74e13c204c30..7ed0acc2180d 100644
--- a/docker-compose.hobby.yml
+++ b/docker-compose.hobby.yml
@@ -243,6 +243,8 @@ services:
environment:
SITE_URL: https://$DOMAIN
SECRET_KEY: $POSTHOG_SECRET
+ # Django's TEMPORAL_SECRET_KEY defaults to SECRET_KEY, so the evaluation scheduler must encrypt with the same value.
+ TEMPORAL_SECRET_KEY: $POSTHOG_SECRET
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
OBJECT_STORAGE_ENDPOINT: http://objectstorage:19000
diff --git a/nodejs/src/ai-observability/config.ts b/nodejs/src/ai-observability/config.ts
index 0d630c408734..c55b10b88f10 100644
--- a/nodejs/src/ai-observability/config.ts
+++ b/nodejs/src/ai-observability/config.ts
@@ -20,7 +20,8 @@ export function getDefaultAIObservabilityConfig(): AIObservabilityConfig {
TEMPORAL_CLIENT_ROOT_CA: undefined,
TEMPORAL_CLIENT_CERT: undefined,
TEMPORAL_CLIENT_KEY: undefined,
- TEMPORAL_SECRET_KEY: process.env.TEMPORAL_SECRET_KEY ?? process.env.SECRET_KEY,
+ // Unlike Django's setting, this has no SECRET_KEY default, so the scheduler pod never needs that key.
+ TEMPORAL_SECRET_KEY: process.env.TEMPORAL_SECRET_KEY,
TEMPORAL_FALLBACK_SECRET_KEYS: process.env.TEMPORAL_FALLBACK_SECRET_KEYS ?? '',
LLMA_EVAL_SCHEDULER_PROVIDER_KEY_GATING: false,
}
diff --git a/nodejs/src/ai-observability/services/temporal.service.test.ts b/nodejs/src/ai-observability/services/temporal.service.test.ts
index 658fab228cc4..26da44272787 100644
--- a/nodejs/src/ai-observability/services/temporal.service.test.ts
+++ b/nodejs/src/ai-observability/services/temporal.service.test.ts
@@ -1,5 +1,6 @@
import { Client, Connection, WorkflowExecutionAlreadyStartedError } from '@temporalio/client'
+import { getDefaultAIObservabilityConfig } from '~/ai-observability/config'
import { EncryptionCodec } from '~/common/temporal/codec'
import { RawKafkaEvent } from '~/types'
@@ -98,26 +99,78 @@ describe('TemporalService', () => {
})
})
- it('configures an encryption codec when a secret key is set', async () => {
- config.TEMPORAL_SECRET_KEY = 'test-secret-key-for-codec-000000'
- config.TEMPORAL_FALLBACK_SECRET_KEYS = 'fallback-key-one-000000000000000,fallback-key-two-000000000000000'
-
- const newService = new TemporalService(config)
- await newService.startEvaluationRunWorkflow('test', createMockEvent({ uuid: 'test-uuid' }), 'llm_judge')
-
- const clientOptions = (Client as unknown as jest.Mock).mock.calls[0][0]
- expect(clientOptions.dataConverter.payloadCodecs).toHaveLength(1)
- expect(clientOptions.dataConverter.payloadCodecs[0]).toBeInstanceOf(EncryptionCodec)
- })
-
- it('sends payloads unencrypted when no secret key is set', async () => {
- config.TEMPORAL_SECRET_KEY = undefined
-
- const newService = new TemporalService(config)
- await newService.startEvaluationRunWorkflow('test', createMockEvent({ uuid: 'test-uuid' }), 'llm_judge')
+ it.each([
+ {
+ name: 'configures an encryption codec when a temporal secret key is set',
+ env: {
+ CLOUD_DEPLOYMENT: 'US',
+ TEMPORAL_SECRET_KEY: 'test-secret-key-for-codec-000000',
+ TEMPORAL_FALLBACK_SECRET_KEYS: 'fallback-key-one-000000000000000,fallback-key-two-000000000000000',
+ },
+ expected: { encryptionCodecs: [true] },
+ },
+ {
+ name: 'sends payloads unencrypted when no secret key is set',
+ env: {},
+ expected: { encryptionCodecs: undefined },
+ },
+ {
+ name: 'does not use SECRET_KEY as the codec key on self-hosted',
+ env: { NODE_ENV: 'production', SECRET_KEY: 'app-secret-key-not-for-temporal-00' },
+ expected: { encryptionCodecs: undefined },
+ },
+ {
+ name: 'refuses to start on cloud without a temporal secret key',
+ env: { CLOUD_DEPLOYMENT: 'US', SECRET_KEY: 'app-secret-key-not-for-temporal-00' },
+ expected: { error: expect.stringContaining('TEMPORAL_SECRET_KEY is required') },
+ },
+ ])('$name', async ({ env, expected }) => {
+ const names = [
+ 'CLOUD_DEPLOYMENT',
+ 'SECRET_KEY',
+ 'TEMPORAL_SECRET_KEY',
+ 'TEMPORAL_FALLBACK_SECRET_KEYS',
+ ...Object.keys(env),
+ ]
+ const original = Object.fromEntries(names.map((name) => [name, process.env[name]]))
+ for (const name of names) {
+ delete process.env[name]
+ }
+ Object.assign(process.env, env)
+
+ try {
+ const startWorkflow = async (): Promise<boolean[] | undefined> => {
+ const { TEMPORAL_SECRET_KEY, TEMPORAL_FALLBACK_SECRET_KEYS } = getDefaultAIObservabilityConfig()
+ const newService = new TemporalService({
+ ...config,
+ TEMPORAL_SECRET_KEY,
+ TEMPORAL_FALLBACK_SECRET_KEYS,
+ })
+ await newService.startEvaluationRunWorkflow(
+ 'test',
+ createMockEvent({ uuid: 'test-uuid' }),
+ 'llm_judge'
+ )
+ const clientOptions = (Client as unknown as jest.Mock).mock.calls[0][0]
+ return clientOptions.dataConverter?.payloadCodecs.map(
+ (codec: unknown) => codec instanceof EncryptionCodec
+ )
+ }
+ const outcome = await startWorkflow().then(
+ (encryptionCodecs) => ({ encryptionCodecs }),
+ (error: Error) => ({ error: error.message })
+ )
- const clientOptions = (Client as unknown as jest.Mock).mock.calls[0][0]
- expect(clientOptions.dataConverter).toBeUndefined()
+ expect(outcome).toEqual(expected)
+ } finally {
+ for (const name of names) {
+ if (original[name] === undefined) {
+ delete process.env[name]
+ } else {
+ process.env[name] = original[name]
+ }
+ }
+ }
})
it('disconnects client properly', async () => {
diff --git a/nodejs/src/ai-observability/services/temporal.service.ts b/nodejs/src/ai-observability/services/temporal.service.ts
index 15f4843844ce..ecfe434543fc 100644
--- a/nodejs/src/ai-observability/services/temporal.service.ts
+++ b/nodejs/src/ai-observability/services/temporal.service.ts
@@ -13,7 +13,7 @@ import { LRUCache } from 'lru-cache'
import { Counter } from 'prom-client'
import { EncryptionCodec } from '~/common/temporal/codec'
-import { isDevEnv } from '~/common/utils/env-utils'
+import { isCloud, isDevEnv } from '~/common/utils/env-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
@@ -207,7 +207,14 @@ export class TemporalService {
ttl: AGGREGATE_START_DEDUP_TTL_MS,
})
- constructor(private config: TemporalServiceConfig) {}
+ constructor(private config: TemporalServiceConfig) {
+ // Self-hosted runs with NODE_ENV=production and no TEMPORAL_SECRET_KEY, so only Cloud fails closed.
+ if (isCloud() && !config.TEMPORAL_SECRET_KEY) {
+ throw new Error(
+ 'TEMPORAL_SECRET_KEY is required on PostHog Cloud. Without it, evaluation workflow payloads reach Temporal unencrypted.'
+ )
+ }
+ }
private async ensureConnected(): Promise<Client> {
if (this.client) {