World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The usage log says which call it was (#996)
WaylandYange0418353af59
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex 62d11c9d..72ba32e4 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -547,7 +547,7 @@ impl LlmClient {return Err(response_failure("LLM", status, retry_after, resp).await?);}let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;- log_usage(&self.model, &body);+ log_usage(&self.model, &call_label(messages), &body);body["choices"][0]["message"]["content"].as_str().map(|s| strip_reasoning(s).to_string())@@ -666,6 +666,7 @@ impl LlmClient {model = %self.model,prompt = u.prompt_tokens,completion = u.completion_tokens,+ call = %call_label(messages),"llm usage");}@@ -1071,7 +1072,36 @@ impl LlmClient {////// 字段名各家不一:OpenAI 用 prompt_tokens_details.cached_tokens,/// DeepSeek 用 prompt_cache_hit_tokens。两个都读,谁在读谁。-fn log_usage(model: &str, body: &serde_json::Value) {+/// 这一次调用是干什么的:系统消息的头一句,截到六十个字符。+///+/// 用量日志从前只有模型和两个数,一篇文档三万 token 花在哪一种调用上(抽取、两票、提规则、+/// 读数、时间解释)从日志里分不出来,只能按时间窗口猜。系统消息是各处写死的常量,头一句+/// 各不相同,拿它当标签不必改任何一个调用方。没有系统消息的(对话)标 `-`+fn call_label(messages: &[ChatMessage]) -> String {+ let Some(system) = messages.iter().find(|m| m.role == "system") else {+ return "-".into();+ };+ let first = system+ .content+ .split(['.', '。', '\n'])+ .next()+ .unwrap_or("")+ .trim();+ // 标签里不留空格和引号:日志是 `key=value` 的一行,拆的人按空格拆+ first+ .chars()+ .take(60)+ .map(|c| {+ if c.is_whitespace() || c == '"' {+ '_'+ } else {+ c+ }+ })+ .collect()+}++fn log_usage(model: &str, call: &str, body: &serde_json::Value) {let u = &body["usage"];if u.is_null() {return;@@ -1085,6 +1115,8 @@ fn log_usage(model: &str, body: &serde_json::Value) {prompt = n("prompt_tokens"),completion = n("completion_tokens"),cached,+ // 排在后面:读日志的脚本按 `model=… prompt=… completion=…` 的次序认这一行+ call,"llm usage");}@@ -1318,6 +1350,26 @@ mod tests {}use super::*;++ #[test]+ fn a_call_is_labelled_by_the_first_sentence_of_its_system_message() {+ let m = |role: &str, content: &str| ChatMessage {+ role: role.into(),+ content: content.into(),+ };+ assert_eq!(+ call_label(&[+ m(+ "system",+ "You read one passage of a document and write down what it states, in the passage's own words. Output one JSON object."+ ),+ m("user", "Passage: ..."),+ ]),+ "You_read_one_passage_of_a_document_and_write_down_what_it_st"+ );+ assert_eq!(call_label(&[m("user", "hello")]), "-");+ assert!(!call_label(&[m("system", "Say \"ok\" now")]).contains('"'));+ }use tokio::io::AsyncWriteExt;fn client_at(addr: std::net::SocketAddr) -> LlmClient {diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjsindex 2e44c321..1e46c8be 100644--- a/scripts/bench/lib.mjs+++ b/scripts/bench/lib.mjs@@ -172,3 +172,18 @@ export async function askChat(kb, message) {}return { conversation, text, steps, error };}++/// 服务端日志里的用量按调用的种类归并(`llm usage … call=<系统消息的头一句>`)。+/// 返回 [{ call, calls, prompt, completion }],按总量从大到小+export function usageByCall(logText) {+ const by = new Map();+ // eslint-disable-next-line no-control-regex+ const text = logText.replace(/\x1b\[[0-9;]*m/g, "");+ for (const m of text.matchAll(/llm usage .*?prompt=(\d+) completion=(\d+)(?:.*?call=(\S+))?/g)) {+ const call = m[3] ?? "(unlabelled)";+ const a = by.get(call) ?? { call, calls: 0, prompt: 0, completion: 0 };+ a.calls += 1; a.prompt += Number(m[1]); a.completion += Number(m[2]);+ by.set(call, a);+ }+ return [...by.values()].sort((a, b) => b.prompt + b.completion - (a.prompt + a.completion));+}diff --git a/scripts/bench/typed.mjs b/scripts/bench/typed.mjsindex fe43b369..f1b1a01b 100644--- a/scripts/bench/typed.mjs+++ b/scripts/bench/typed.mjs@@ -32,7 +32,7 @@import fs from "node:fs";import path from "node:path";import { fileURLToPath } from "node:url";-import { api, login, parseArgs, sleep, until, log, EMAIL, PASSWORD } from "./lib.mjs";+import { api, login, parseArgs, sleep, until, log, usageByCall, EMAIL, PASSWORD } from "./lib.mjs";import { execFileSync } from "node:child_process";const HERE = path.dirname(fileURLToPath(import.meta.url));@@ -518,6 +518,15 @@ if (SERVER_LOG) {};const t = result.tokens;console.log(`token(服务端日志):抽取 ${t.extract?.total ?? "-"},对齐 ${t.align?.total ?? "-"},勘误 ${t.errata?.total ?? "-"},合计 ${t.total?.total ?? "-"},每篇 ${t.total?.per_document ?? "-"}`);+ // 按调用的种类拆(日志里的 `call=`,系统消息的头一句):阶段是按时间窗口归的,对齐那一段里+ // 两票、提规则、读数混在一起,只有这一栏分得开+ const lines = fs.readFileSync(SERVER_LOG, "utf8").split("\n").filter((l) => {+ const m = /^(?:\x1b\[[0-9;]*m)*(\S+Z)/.exec(l);+ return m && m[1] >= stamps.start && m[1] < stamps.end;+ });+ result.tokens.by_call = usageByCall(lines.join("\n")).map((u) => ({ ...u, per_document: Math.round((u.prompt + u.completion) / docs) }));+ for (const u of result.tokens.by_call)+ console.log(` ${String(u.prompt + u.completion).padStart(9)} ${String(u.calls).padStart(5)} 次 每篇 ${String(u.per_document).padStart(6)} ${u.call}`);}fs.writeFileSync(OUT, JSON.stringify(result, null, 1));console.log(`结果写到 ${OUT}(${result.minutes} 分钟)`);