🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(web): cache prompt list count under the prompt cache epoch (#18348)
Niklas Semmlerb57ada7de757
diff --git a/packages/shared/src/server/services/PromptService/index.ts b/packages/shared/src/server/services/PromptService/index.tsindex 41435b4fa2..b0336ab398 100644--- a/packages/shared/src/server/services/PromptService/index.ts+++ b/packages/shared/src/server/services/PromptService/index.ts@@ -1,5 +1,5 @@import { Redis, Cluster } from "ioredis";-import { randomBytes } from "crypto";+import { createHash, randomBytes } from "crypto";import { env } from "../../../env";import { logger } from "../../logger";import { escapeRegex } from "./utils";@@ -191,6 +191,48 @@ export class PromptService {);}+ /**+ * Counts prompt names matching a list filter, cached under the project's+ * epoch so every prompt write (which rotates the epoch) invalidates it.+ */+ public async getPromptListCount(params: {+ projectId: string;+ filterKey: string;+ computeCount: () => Promise<number>;+ }): Promise<number> {+ if (!this.cacheEnabled) return params.computeCount();++ let key: string | null = null;+ try {+ // Read the epoch before counting: a write committing mid-count rotates+ // the epoch afterwards, so a stale count can only land under a dead key.+ const epoch = await this.getOrCreateEpoch(params);+ if (epoch) {+ const filterHash = createHash("sha256")+ .update(params.filterKey)+ .digest("base64url");+ key = `prompt_list_count:${params.projectId}:${epoch}:${filterHash}`;++ const cached = await this.redis?.get(key);+ if (cached) return Number(cached);+ }+ } catch (e) {+ this.logError("Error getting cached prompt list count", e);+ }++ const count = await params.computeCount();++ if (key) {+ try {+ await this.redis?.set(key, String(count), "EX", this.ttlSeconds);+ } catch (e) {+ this.logError("Error caching prompt list count", e);+ }+ }++ return count;+ }+private async getCacheKey(params: PromptParams): Promise<string | null> {const epoch = await this.getOrCreateEpoch(params);if (!epoch) return null;diff --git a/web/src/__tests__/server/prompts.v2.servertest.ts b/web/src/__tests__/server/prompts.v2.servertest.tsindex 06389b59dd..a7ccfc8a53 100644--- a/web/src/__tests__/server/prompts.v2.servertest.ts+++ b/web/src/__tests__/server/prompts.v2.servertest.ts@@ -21,6 +21,8 @@ import {getObservationById,MAX_PROMPT_NESTING_DEPTH,ChatMessageType,+ PromptService,+ redis,} from "@langfuse/shared/src/server";import {createUserWithOrgRole,@@ -1370,6 +1372,33 @@ describe("/api/public/v2/prompts API Endpoint", () => {});});+ describe("when counting a prompt list across prompt writes", () => {+ it("serves the cached count per filter until the epoch rotates", async () => {+ // CI disables the prompt cache via env, so enable it explicitly.+ const promptService = new PromptService(prisma, redis, undefined, true);+ const projectId = randomUUID();+ const computeCount = vi.fn();+ const count = (filterKey: string) =>+ promptService.getPromptListCount({+ projectId,+ filterKey,+ computeCount,+ });++ computeCount.mockResolvedValueOnce(1).mockResolvedValueOnce(5);+ expect(await count("all")).toBe(1);+ expect(await count("all")).toBe(1);+ expect(await count("tag=a")).toBe(5);+ expect(computeCount).toHaveBeenCalledTimes(2);++ await promptService.invalidateCache({ projectId });++ computeCount.mockResolvedValueOnce(2);+ expect(await count("all")).toBe(2);+ expect(computeCount).toHaveBeenCalledTimes(3);+ });+ });+describe("when fetching a prompt list", () => {const otherProjectPromptName = "prompt-5";let otherProjectId: string;diff --git a/web/src/features/prompts/server/actions/getPromptsMeta.ts b/web/src/features/prompts/server/actions/getPromptsMeta.tsindex 112008ba90..48a6cb7c1b 100644--- a/web/src/features/prompts/server/actions/getPromptsMeta.ts+++ b/web/src/features/prompts/server/actions/getPromptsMeta.ts@@ -5,7 +5,11 @@ import {type PromptType,} from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";-import { tableColumnsToSqlFilterAndPrefix } from "@langfuse/shared/src/server";+import {+ PromptService,+ redis,+ tableColumnsToSqlFilterAndPrefix,+} from "@langfuse/shared/src/server";export type GetPromptsMetaParams = GetPromptsMetaType & { projectId: string };@@ -61,18 +65,25 @@ export const getPromptsMeta = async (ORDER BY n.name`) as PromptsMeta[];- const [{ count: totalItemsCount }] = (await prisma.$queryRaw`- SELECT COUNT(*) AS count- FROM (- SELECT p.name- FROM prompts p- WHERE p."project_id" = ${projectId}- ${getPromptsFilterCondition(params)}- GROUP BY p.name- ) names- `) as { count: BigInt }[];-- const totalItems = Number(totalItemsCount);+ // Page crawls repeat the same full-project count on every page.+ const filterCondition = getPromptsFilterCondition(params);+ const totalItems = await new PromptService(prisma, redis).getPromptListCount({+ projectId,+ filterKey: JSON.stringify([filterCondition.sql, filterCondition.values]),+ computeCount: async () => {+ const [{ count }] = (await prisma.$queryRaw`+ SELECT COUNT(*) AS count+ FROM (+ SELECT p.name+ FROM prompts p+ WHERE p."project_id" = ${projectId}+ ${filterCondition}+ GROUP BY p.name+ ) names+ `) as { count: BigInt }[];+ return Number(count);+ },+ });const totalPages = Math.ceil(totalItems / limit);return {diff --git a/web/src/features/prompts/server/routers/promptRouter.ts b/web/src/features/prompts/server/routers/promptRouter.tsindex ea9cc567f1..41400a9187 100644--- a/web/src/features/prompts/server/routers/promptRouter.ts+++ b/web/src/features/prompts/server/routers/promptRouter.ts@@ -637,7 +637,15 @@ export const promptRouter = createTRPCRouter({},});- promptService.invalidateCache({ projectId });+ // The deletion has committed; a cache failure must not skip webhooks.+ try {+ await promptService.invalidateCache({ projectId });+ } catch (error) {+ logger.error(+ `Failed to invalidate prompt cache after deleting prompts in project ${projectId}`,+ error,+ );+ }// Trigger webhooks for prompt deletionawait Promise.all(