🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { createAnnotationQueueForApi } from "@/src/features/annotation-queues/server/publicAnnotationQueueService";
import {
CreateAnnotationQueueBody,
CreateAnnotationQueueResponse,
} from "@/src/features/public-api/types/annotation-queues";
import { defineTool } from "../../../core/define-tool";
import { buildAnnotationQueueUrl } from "@langfuse/shared/src/server";
import { runMcpTool } from "../../../core/run-mcp-tool";
import { getMcpPublicApiAuth } from "../../publicApi";
import { z } from "zod";
const CreateAnnotationQueueBaseSchema = z
.object({
name: z.string(),
description: z.string().optional(),
scoreConfigIds: z.array(z.string()).min(1),
})
.strict();
export const [createAnnotationQueueTool, handleCreateAnnotationQueue] =
defineTool({
name: "createAnnotationQueue",
description:
"Create an annotation queue, a worklist that collects trace or observation items for human review and scoring.",
action: "annotationQueues:CUD",
baseSchema: CreateAnnotationQueueBaseSchema,
inputSchema: CreateAnnotationQueueBody,
handler: async (input, context) =>
runMcpTool({
spanName: "mcp.annotation_queues.create",
context,
attributes: { "mcp.annotation_queue_name": input.name },
fn: async () => {
const auth = await getMcpPublicApiAuth(context);
const result = await createAnnotationQueueForApi({
projectId: context.projectId,
plan: auth.scope.plan,
input,
auditScope: context,
});
const queue = CreateAnnotationQueueResponse.parse(result);
return {
...queue,
url: buildAnnotationQueueUrl({
projectId: context.projectId,
queueId: queue.id,
}),
};
},
}),
});