🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
createServer,
type IncomingMessage,
type ServerResponse,
} from "node:http";
import type { AddressInfo } from "node:net";
type RecordedRequest = {
method: string;
url: string;
headers: Record<string, string | string[] | undefined>;
body: string;
};
export type LocalLlmServerHandler = (
req: IncomingMessage,
body: string,
res: ServerResponse,
) => void;
export type LocalLlmServer = {
url: string;
host: string;
requests: RecordedRequest[];
close: () => Promise<void>;
};
export async function startLocalLlmServer(
handler: LocalLlmServerHandler,
): Promise<LocalLlmServer> {
const requests: RecordedRequest[] = [];
const server = createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString("utf-8");
requests.push({
method: req.method ?? "",
url: req.url ?? "",
headers: req.headers,
body,
});
handler(req, body, res);
});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = (server.address() as AddressInfo).port;
return {
url: `http://127.0.0.1:${port}`,
host: "127.0.0.1",
requests,
close: () =>
new Promise<void>((resolve, reject) =>
server.close((err) => (err ? reject(err) : resolve())),
),
};
}