Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(prompts): preserve cache epoch across prompt reads (#18752)

Fazlul Karimed7237552aa2
diff --git a/packages/shared/src/server/services/PromptService/index.ts b/packages/shared/src/server/services/PromptService/index.ts
index b0336ab398..58bac03db0 100644
--- a/packages/shared/src/server/services/PromptService/index.ts
+++ b/packages/shared/src/server/services/PromptService/index.ts
@@ -51,8 +51,19 @@ export class PromptService {
return this.getRawPrompt(params);
}
+ let cacheKey: string | null = null;
if (this.cacheEnabled) {
- const cachedPrompt = await this.getCachedPrompt(params);
+ try {
+ // Capture the namespace before reading the DB so invalidation cannot
+ // move an in-flight result into the new epoch.
+ cacheKey = await this.getCacheKey(params);
+ } catch (e) {
+ this.logError("Error getting cached prompt", e);
+ }
+
+ const cachedPrompt = cacheKey
+ ? await this.getCachedPrompt(cacheKey)
+ : null;
this.incrementMetric(
cachedPrompt
@@ -69,8 +80,8 @@ export class PromptService {
const dbPrompt = await this.getDbPrompt(params);
- if (this.cacheEnabled && dbPrompt) {
- await this.cachePrompt({ ...params, prompt: dbPrompt });
+ if (cacheKey && dbPrompt) {
+ await this.cachePrompt(cacheKey, dbPrompt);
this.logDebug("Successfully cached prompt for params", params);
}
@@ -146,13 +157,8 @@ export class PromptService {
};
}
- private async getCachedPrompt(
- params: PromptParams,
- ): Promise<PromptResult | null> {
+ private async getCachedPrompt(key: string): Promise<PromptResult | null> {
try {
- const key = await this.getCacheKey(params);
- if (!key) return null;
-
const value = await this.redis?.get(key);
if (value) return JSON.parse(value) as PromptResult;
@@ -163,12 +169,9 @@ export class PromptService {
return null;
}
- private async cachePrompt(params: PromptParams & { prompt: PromptResult }) {
+ private async cachePrompt(key: string, prompt: PromptResult) {
try {
- const key = await this.getCacheKey(params);
- if (!key) return;
-
- const value = JSON.stringify(params.prompt);
+ const value = JSON.stringify(prompt);
await this.redis?.set(key, value, "EX", this.ttlSeconds);
} catch (e) {
diff --git a/web/src/__tests__/server/promptCache.servertest.ts b/web/src/__tests__/server/promptCache.servertest.ts
index 30c3ded9c9..74b5c7620d 100644
--- a/web/src/__tests__/server/promptCache.servertest.ts
+++ b/web/src/__tests__/server/promptCache.servertest.ts
@@ -10,6 +10,7 @@ describe("PromptService", () => {
// Typed handle on the same vi.fn instance wired into mockPrisma, because
// Prisma's overloaded findFirst signature survives vitest's Mocked mapping.
let promptFindFirstMock: Mock;
+ let promptDependencyFindManyMock: Mock;
let mockRedis: Mocked<Redis>;
let mockMetricIncrementer: Mock;
@@ -33,12 +34,13 @@ describe("PromptService", () => {
beforeEach(() => {
promptFindFirstMock = vi.fn();
+ promptDependencyFindManyMock = vi.fn().mockResolvedValue([]);
mockPrisma = {
prompt: {
findFirst: promptFindFirstMock,
},
promptDependency: {
- findMany: vi.fn().mockResolvedValue([]),
+ findMany: promptDependencyFindManyMock,
},
} as unknown as Mocked<PrismaClient>;
@@ -102,6 +104,54 @@ describe("PromptService", () => {
);
});
+ it("does not cache an in-flight read under a newer epoch", async () => {
+ const cache = new Map<string, string>([
+ ["prompt_cache_epoch:project1", "epoch-1"],
+ ]);
+ mockRedis.get.mockImplementation(
+ async (key) => cache.get(key.toString()) ?? null,
+ );
+ mockRedis.set.mockImplementation(async (key, value) => {
+ cache.set(key.toString(), value.toString());
+ return "OK";
+ });
+ promptFindFirstMock.mockResolvedValue(mockPrompt);
+
+ const readStarted = Promise.withResolvers<void>();
+ const resumeRead = Promise.withResolvers<void>();
+ promptDependencyFindManyMock.mockImplementationOnce(async () => {
+ readStarted.resolve();
+ await resumeRead.promise;
+ return [];
+ });
+ const params = {
+ projectId: "project1",
+ promptName: "testPrompt",
+ version: undefined,
+ label: "test",
+ };
+ const firstRead = promptService.getPrompt(params);
+ await readStarted.promise;
+
+ promptFindFirstMock.mockResolvedValue({
+ ...mockPrompt,
+ id: "prompt-v2",
+ version: 2,
+ prompt: "Version 2 content",
+ });
+ try {
+ await promptService.invalidateCache({ projectId: params.projectId });
+ } finally {
+ resumeRead.resolve();
+ }
+
+ expect(await firstRead).toMatchObject({ version: 1 });
+ expect(await promptService.getPrompt(params)).toMatchObject({
+ version: 2,
+ prompt: "Version 2 content",
+ });
+ });
+
it("does not share cached prompts between a version and numeric label", async () => {
const cache = new Map<string, string>([
["prompt_cache_epoch:project1", "epoch-1"],