🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Redis, Cluster } from "ioredis";
import { createHash, randomBytes } from "crypto";
import { env } from "../../../env";
import { logger } from "../../logger";
import { escapeRegex } from "./utils";
import { LangfuseConflictError } from "../../../errors/ConflictError";
import { type Prisma, type PrismaClient, type Prompt } from "../../../db";
import {
PromptGraph,
PromptParams,
PartialPrompt,
ResolvedPromptGraph,
PromptServiceMetrics,
PromptResult,
} from "./types";
import { ParsedPromptDependencyTag } from "../../../features/prompts/parsePromptDependencyTags";
export const MAX_PROMPT_NESTING_DEPTH = 5;
// Measured on the JSON-serialized prompt. A dependency tag repeated N times
// expands to N copies of the child, so this bounds the resolved output.
export const MAX_RESOLVED_PROMPT_LENGTH = 5_000_000;
type PrismaClientOrTransaction = PrismaClient | Prisma.TransactionClient;
export class PromptService {
private cacheEnabled: boolean;
private ttlSeconds: number;
// Epoch keys live much longer than cache entries. 7 days gives inactive
// projects plenty of time while still cleaning up eventually.
private epochTtlSeconds = 7 * 24 * 60 * 60;
constructor(
private prisma: PrismaClientOrTransaction,
private redis: Redis | Cluster | null,
private metricIncrementer?: // used for otel metrics
(name: string, value?: number) => void,
cacheEnabled?: boolean, // used for testing
) {
if (cacheEnabled !== undefined) {
this.cacheEnabled = cacheEnabled;
} else {
this.cacheEnabled =
Boolean(redis) && env.LANGFUSE_CACHE_PROMPT_ENABLED === "true";
}
this.ttlSeconds = env.LANGFUSE_CACHE_PROMPT_TTL_SECONDS;
}
public async getPrompt(params: PromptParams): Promise<PromptResult | null> {
if (params.resolve === false) {
return this.getRawPrompt(params);
}
let cacheKey: string | null = null;
if (this.cacheEnabled) {
try {
// Capture the namespace before reading the DB so invalidation cannot
// move an in-flight result into the new epoch.
cacheKey = await this.getCacheKey(params);
} catch (e) {
this.logError("Error getting cached prompt", e);
}
const cachedPrompt = cacheKey
? await this.getCachedPrompt(cacheKey)
: null;
this.incrementMetric(
cachedPrompt
? PromptServiceMetrics.PromptCacheHit
: PromptServiceMetrics.PromptCacheMiss,
);
if (cachedPrompt) {
this.logDebug("Returning cached prompt for params", params);
return cachedPrompt;
}
}
const dbPrompt = await this.getDbPrompt(params);
if (cacheKey && dbPrompt) {
await this.cachePrompt(cacheKey, dbPrompt);
this.logDebug("Successfully cached prompt for params", params);
}
this.logDebug("Returning DB prompt for params", params);
return dbPrompt;
}
private async getDbPrompt(
params: PromptParams,
): Promise<PromptResult | null> {
return this.resolvePrompt(await this.findPrompt(params));
}
private async getRawPrompt(
params: PromptParams,
): Promise<PromptResult | null> {
const prompt = await this.findPrompt(params);
if (!prompt) return null;
return {
...prompt,
resolutionGraph: null,
};
}
private async findPrompt(params: PromptParams): Promise<Prompt | null> {
const { projectId, promptName, version, label } = params;
if (version) {
return this.prisma.prompt.findFirst({
where: {
projectId,
name: promptName,
version,
},
});
}
if (label) {
return this.prisma.prompt.findFirst({
where: {
projectId,
name: promptName,
labels: {
has: label,
},
},
});
}
this.logError("Invalid prompt params", params);
return null;
}
public async resolvePrompt(
prompt: Prompt | null,
): Promise<PromptResult | null> {
if (!prompt) return prompt;
const promptGraph = await this.buildAndResolvePromptGraph({
projectId: prompt.projectId,
parentPrompt: prompt,
});
return {
...prompt,
prompt: promptGraph.resolvedPrompt,
resolutionGraph: promptGraph.graph,
};
}
private async getCachedPrompt(key: string): Promise<PromptResult | null> {
try {
const value = await this.redis?.get(key);
if (value) return JSON.parse(value) as PromptResult;
} catch (e) {
this.logError("Error getting cached prompt", e);
}
return null;
}
private async cachePrompt(key: string, prompt: PromptResult) {
try {
const value = JSON.stringify(prompt);
await this.redis?.set(key, value, "EX", this.ttlSeconds);
} catch (e) {
this.logError("Error caching prompt", e);
}
}
public async invalidateCache(
params: Pick<PromptParams, "projectId">,
): Promise<void> {
if (!this.cacheEnabled) return;
// Rotate the epoch token to move all prompt reads/writes to a fresh namespace.
// Old keys remain untouched and naturally expire via TTL.
await this.redis?.set(
this.getEpochKey(params),
this.newEpochToken(),
"EX",
this.epochTtlSeconds,
);
}
/**
* Counts prompt names matching a list filter, cached under the project's
* epoch so every prompt write (which rotates the epoch) invalidates it.
*/
public async getPromptListCount(params: {
projectId: string;
filterKey: string;
computeCount: () => Promise<number>;
}): Promise<number> {
if (!this.cacheEnabled) return params.computeCount();
let key: string | null = null;
try {
// Read the epoch before counting: a write committing mid-count rotates
// the epoch afterwards, so a stale count can only land under a dead key.
const epoch = await this.getOrCreateEpoch(params);
if (epoch) {
const filterHash = createHash("sha256")
.update(params.filterKey)
.digest("base64url");
key = `prompt_list_count:${params.projectId}:${epoch}:${filterHash}`;
const cached = await this.redis?.get(key);
if (cached) return Number(cached);
}
} catch (e) {
this.logError("Error getting cached prompt list count", e);
}
const count = await params.computeCount();
if (key) {
try {
await this.redis?.set(key, String(count), "EX", this.ttlSeconds);
} catch (e) {
this.logError("Error caching prompt list count", e);
}
}
return count;
}
private async getCacheKey(params: PromptParams): Promise<string | null> {
const epoch = await this.getOrCreateEpoch(params);
if (!epoch) return null;
const prefix = this.getCacheKeyPrefix(params, epoch);
// Numeric labels must not share a cache entry with the same prompt version.
const selector =
typeof params.version === "number"
? `version:${params.version}`
: `label:${params.label}`;
return `${prefix}:${selector}`;
}
private getCacheKeyPrefix(
params: Pick<PromptParams, "projectId" | "promptName">,
epoch: string,
): string {
return `prompt:${params.projectId}:${epoch}:${params.promptName}`;
}
private getEpochKey(params: Pick<PromptParams, "projectId">): string {
// Important: epoch is project-scoped (not prompt-scoped) because resolved prompts
// can include transitive dependencies across multiple prompt names.
return `prompt_cache_epoch:${params.projectId}`;
}
private newEpochToken(): string {
// 48 bits of entropy in a compact URL-safe string (8 chars).
return randomBytes(6).toString("base64url");
}
private async getOrCreateEpoch(
params: Pick<PromptParams, "projectId">,
): Promise<string | null> {
const epochKey = this.getEpochKey(params);
const currentEpoch = await this.redis?.get(epochKey);
if (currentEpoch) return currentEpoch;
const newEpoch = this.newEpochToken();
await this.redis?.set(epochKey, newEpoch, "EX", this.epochTtlSeconds, "NX");
// Return the winner value in case multiple requests initialize concurrently.
return (await this.redis?.get(epochKey)) ?? newEpoch;
}
public async buildAndResolvePromptGraph(params: {
projectId: string;
parentPrompt: PartialPrompt;
dependencies?: ParsedPromptDependencyTag[];
}): Promise<ResolvedPromptGraph> {
try {
const { projectId, parentPrompt, dependencies } = params;
const graph: PromptGraph = {
root: {
name: parentPrompt.name,
version: parentPrompt.version,
id: parentPrompt.id,
},
dependencies: {},
};
const seen = new Set<string>();
const resolve = async (
currentPrompt: PartialPrompt,
deps: ParsedPromptDependencyTag[] | undefined,
level: number,
) => {
// Nesting depth check
if (level >= MAX_PROMPT_NESTING_DEPTH) {
throw new LangfuseConflictError(
`Maximum nesting depth exceeded (${MAX_PROMPT_NESTING_DEPTH})`,
);
}
// Circular dependency check
if (
seen.has(currentPrompt.id) ||
(currentPrompt.name === parentPrompt.name &&
currentPrompt.id !== parentPrompt.id) // ensure that the parent prompt cannot reference a prompt of the same name but different version
) {
throw new LangfuseConflictError(
`Circular dependency detected involving prompt '${currentPrompt.name}' version ${currentPrompt.version}`,
);
}
seen.add(currentPrompt.id);
// deps can be either passed (if a prompt is created and content was scanned) or retrieved from db
let promptDependencies = deps;
if (!deps) {
promptDependencies = (
await this.prisma.promptDependency.findMany({
where: {
projectId,
parentId: currentPrompt.id,
},
select: {
childName: true,
childLabel: true,
childVersion: true,
},
})
).map((dep) => {
if (dep.childVersion !== null) {
return {
name: dep.childName,
type: "version",
version: dep.childVersion,
} as ParsedPromptDependencyTag;
}
if (dep.childLabel !== null) {
return {
name: dep.childName,
type: "label",
label: dep.childLabel,
} as ParsedPromptDependencyTag;
}
throw new LangfuseConflictError(
`Prompt dependency is malformed: ${dep.childName}`,
);
});
}
if (promptDependencies && promptDependencies.length) {
// Instantiate resolved prompt, use stringfied version for regex operations
// Do this inside if clause to skip stringify/parse overhead for prompts without dependencies
let resolvedPrompt = JSON.stringify(currentPrompt.prompt);
for (const dep of promptDependencies) {
const depPrompt = await this.prisma.prompt.findFirst({
where: {
projectId,
name: dep.name,
...(dep.type === "version"
? { version: dep.version }
: { labels: { has: dep.label } }),
},
});
const logName = `${dep.name} - ${dep.type} ${dep.type === "version" ? dep.version : dep.label}`;
if (!depPrompt)
throw new LangfuseConflictError(
`Prompt dependency not found: ${logName}`,
);
if (depPrompt.type !== "text")
throw new LangfuseConflictError(
`Prompt dependency is not a text prompt: ${logName}`,
);
// side-effect: populate adjacency list to return later as well
if (graph.dependencies[currentPrompt.id] === undefined) {
graph.dependencies[currentPrompt.id] = [];
}
graph.dependencies[currentPrompt.id].push({
id: depPrompt.id,
name: depPrompt.name,
version: depPrompt.version,
});
// resolve the prompt content recursively
const resolvedDepPrompt = await resolve(
depPrompt,
undefined,
level + 1,
);
const versionPattern = `@@@langfusePrompt:name=${escapeRegex(depPrompt.name)}\\|version=${escapeRegex(depPrompt.version)}@@@`;
const labelPatterns = depPrompt.labels.map(
(label) =>
`@@@langfusePrompt:name=${escapeRegex(depPrompt.name)}\\|label=${escapeRegex(label)}@@@`,
);
const combinedPattern = [versionPattern, ...labelPatterns].join(
"|",
);
const regex = new RegExp(combinedPattern, "g");
const replaceValue = JSON.stringify(resolvedDepPrompt)
.slice(1, -1) // this is necessary to avoid parsing errors as resolved value is unstringified
.replace(/\$/g, "$$$$"); // Escape dollar signs in replacement string. $ has special meaning in replace(), so we need to escape it with $$. Since we're in a string that will be used as a replacement, we need to double escape it (hence $$$$)
const occurrences = resolvedPrompt.match(regex)?.length ?? 0;
if (
resolvedPrompt.length + occurrences * replaceValue.length >
MAX_RESOLVED_PROMPT_LENGTH
) {
throw new LangfuseConflictError(
`Resolved prompt exceeds maximum size (${MAX_RESOLVED_PROMPT_LENGTH} characters)`,
);
}
resolvedPrompt = resolvedPrompt.replace(regex, replaceValue);
}
seen.delete(currentPrompt.id);
return JSON.parse(resolvedPrompt);
}
seen.delete(currentPrompt.id);
return currentPrompt.prompt;
};
const resolvedPrompt = await resolve(parentPrompt, dependencies, 0);
return {
graph: Object.keys(graph.dependencies).length > 0 ? graph : null,
resolvedPrompt,
};
} catch (err) {
console.error(err);
throw err;
}
}
private logError(message: string, ...args: any[]) {
logger.error(`[PromptService] ${message}`, ...args);
}
private logDebug(message: string, ...args: any[]) {
logger.debug(`[PromptService] ${message}`, ...args);
}
private incrementMetric(name: PromptServiceMetrics, value = 1) {
try {
this.metricIncrementer?.(name, value);
} catch (e) {
this.logError("Error incrementing metric", name, e);
}
}
}