🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(prompts): preserve cache epoch across prompt reads (#18752)
Fazlul Karimed7237552aa2
diff --git a/packages/shared/src/server/services/PromptService/index.ts b/packages/shared/src/server/services/PromptService/index.tsindex b0336ab398..58bac03db0 100644--- a/packages/shared/src/server/services/PromptService/index.ts+++ b/packages/shared/src/server/services/PromptService/index.ts@@ -51,8 +51,19 @@ export class PromptService {return this.getRawPrompt(params);}+ let cacheKey: string | null = null;if (this.cacheEnabled) {- const cachedPrompt = await this.getCachedPrompt(params);+ try {+ // Capture the namespace before reading the DB so invalidation cannot+ // move an in-flight result into the new epoch.+ cacheKey = await this.getCacheKey(params);+ } catch (e) {+ this.logError("Error getting cached prompt", e);+ }++ const cachedPrompt = cacheKey+ ? await this.getCachedPrompt(cacheKey)+ : null;this.incrementMetric(cachedPrompt@@ -69,8 +80,8 @@ export class PromptService {const dbPrompt = await this.getDbPrompt(params);- if (this.cacheEnabled && dbPrompt) {- await this.cachePrompt({ ...params, prompt: dbPrompt });+ if (cacheKey && dbPrompt) {+ await this.cachePrompt(cacheKey, dbPrompt);this.logDebug("Successfully cached prompt for params", params);}@@ -146,13 +157,8 @@ export class PromptService {};}- private async getCachedPrompt(- params: PromptParams,- ): Promise<PromptResult | null> {+ private async getCachedPrompt(key: string): Promise<PromptResult | null> {try {- const key = await this.getCacheKey(params);- if (!key) return null;-const value = await this.redis?.get(key);if (value) return JSON.parse(value) as PromptResult;@@ -163,12 +169,9 @@ export class PromptService {return null;}- private async cachePrompt(params: PromptParams & { prompt: PromptResult }) {+ private async cachePrompt(key: string, prompt: PromptResult) {try {- const key = await this.getCacheKey(params);- if (!key) return;-- const value = JSON.stringify(params.prompt);+ const value = JSON.stringify(prompt);await this.redis?.set(key, value, "EX", this.ttlSeconds);} catch (e) {diff --git a/web/src/__tests__/server/promptCache.servertest.ts b/web/src/__tests__/server/promptCache.servertest.tsindex 30c3ded9c9..74b5c7620d 100644--- a/web/src/__tests__/server/promptCache.servertest.ts+++ b/web/src/__tests__/server/promptCache.servertest.ts@@ -10,6 +10,7 @@ describe("PromptService", () => {// Typed handle on the same vi.fn instance wired into mockPrisma, because// Prisma's overloaded findFirst signature survives vitest's Mocked mapping.let promptFindFirstMock: Mock;+ let promptDependencyFindManyMock: Mock;let mockRedis: Mocked<Redis>;let mockMetricIncrementer: Mock;@@ -33,12 +34,13 @@ describe("PromptService", () => {beforeEach(() => {promptFindFirstMock = vi.fn();+ promptDependencyFindManyMock = vi.fn().mockResolvedValue([]);mockPrisma = {prompt: {findFirst: promptFindFirstMock,},promptDependency: {- findMany: vi.fn().mockResolvedValue([]),+ findMany: promptDependencyFindManyMock,},} as unknown as Mocked<PrismaClient>;@@ -102,6 +104,54 @@ describe("PromptService", () => {);});+ it("does not cache an in-flight read under a newer epoch", async () => {+ const cache = new Map<string, string>([+ ["prompt_cache_epoch:project1", "epoch-1"],+ ]);+ mockRedis.get.mockImplementation(+ async (key) => cache.get(key.toString()) ?? null,+ );+ mockRedis.set.mockImplementation(async (key, value) => {+ cache.set(key.toString(), value.toString());+ return "OK";+ });+ promptFindFirstMock.mockResolvedValue(mockPrompt);++ const readStarted = Promise.withResolvers<void>();+ const resumeRead = Promise.withResolvers<void>();+ promptDependencyFindManyMock.mockImplementationOnce(async () => {+ readStarted.resolve();+ await resumeRead.promise;+ return [];+ });+ const params = {+ projectId: "project1",+ promptName: "testPrompt",+ version: undefined,+ label: "test",+ };+ const firstRead = promptService.getPrompt(params);+ await readStarted.promise;++ promptFindFirstMock.mockResolvedValue({+ ...mockPrompt,+ id: "prompt-v2",+ version: 2,+ prompt: "Version 2 content",+ });+ try {+ await promptService.invalidateCache({ projectId: params.projectId });+ } finally {+ resumeRead.resolve();+ }++ expect(await firstRead).toMatchObject({ version: 1 });+ expect(await promptService.getPrompt(params)).toMatchObject({+ version: 2,+ prompt: "Version 2 content",+ });+ });+it("does not share cached prompts between a version and numeric label", async () => {const cache = new Map<string, string>([["prompt_cache_epoch:project1", "epoch-1"],