Skip to content

World's first open-source enterprise world model.

A recording is read with its speakers (#721)

WaylandYangaa0b6bd6921c
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index 3a2ef67a..184dc2de 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -440,6 +440,11 @@ pub struct LlmSettings {
#[serde(skip_serializing)]
pub ocr_api_key: Option<String>,
pub ocr_backend: Option<String>,
+ /// 会标说话人的转写模型(OpenAI `/audio/transcriptions` + `diarized_json`,0040)
+ pub transcribe_base_url: Option<String>,
+ #[serde(skip_serializing)]
+ pub transcribe_api_key: Option<String>,
+ pub transcribe_model: Option<String>,
}
impl LlmSettings {
@@ -452,6 +457,9 @@ impl LlmSettings {
pub fn ocr_ready(&self) -> bool {
self.ocr_base_url.is_some()
}
+ pub fn transcribe_ready(&self) -> bool {
+ self.transcribe_base_url.is_some() && self.transcribe_model.is_some()
+ }
}
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index 77db6f68..8e940fcd 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -168,11 +168,14 @@ pub mod mineru;
pub mod ontology_rdf;
mod parsers;
pub mod provenance;
+mod reading;
+pub mod transcript;
pub use chunker::{chunk_segments, chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};
/// Decode fetched text with the same encoding detection as file ingestion.
pub use parsers::plain_text as decode_text;
pub use provenance::{Origin, Provenance, Segment};
+pub use reading::Reading;
/// 解析产物:纯文本 + 可选结构信息。
#[derive(Debug)]
@@ -219,6 +222,17 @@ pub struct NeedsReader {
pub what: &'static str,
}
+/// 录音转写回来了,却分不出谁说的(0040 决定 5)。
+///
+/// 跟没配转写模型一样对待:不读,文档停下、告警说明原因。分不出说话人的会议记录,「张三说
+/// 他三季度交付」和「李四说张三三季度交付」是同一行字,承诺会记到错的人头上——比什么都
+/// 不抽更糟。换一个会标说话人的模型,存设置时它会重新排队
+#[derive(Debug, Clone, thiserror::Error)]
+#[error(
+ "The transcription model did not say who spoke; a recording is read only with speaker labels"
+)]
+pub struct NoSpeakers;
+
/// 这份文件读不了,换什么模型也读不了(视频、可执行文件、老式二进制格式)。重试没用
#[derive(Debug, Clone, thiserror::Error)]
#[error("{0}")]
diff --git a/crates/utopia-ingest/src/mineru.rs b/crates/utopia-ingest/src/mineru.rs
index 19145cc4..cbc3456a 100644
--- a/crates/utopia-ingest/src/mineru.rs
+++ b/crates/utopia-ingest/src/mineru.rs
@@ -11,63 +11,14 @@
//!
//! 页码从 1 数(`page_idx` + 1):锚点是给人翻页用的,查看器的 `#page=` 也从 1 数。
-use crate::chunker::{chunk_segments, ChunkPiece};
use crate::provenance::{Origin, Provenance, Segment};
+use crate::reading::{Place, Reading, Region};
use serde_json::{json, Value};
-use std::ops::Range;
/// 版面上的辅助区域:页眉、页脚、页码。每页重复一遍,进了正文只会让每块都多一行噪声。
/// 页脚注和旁注不在这里——合同的脚注里写着条款
const CHROME: &[&str] = &["header", "footer", "page_number"];
-/// 拼好的正文,连同每页的出处和每个区域的位置
-#[derive(Debug, Clone)]
-pub struct Reading {
- pub text: String,
- pub segments: Vec<Segment>,
- /// 每个区域在正文里的字节区间、所在页、框
- regions: Vec<Region>,
-}
-
-#[derive(Debug, Clone)]
-struct Region {
- range: Range<usize>,
- page: u64,
- bbox: Option<[f64; 4]>,
-}
-
-impl Reading {
- /// 按页分段切块,再给每块的锚点补上它盖住的区域的外框
- pub fn chunk(&self, budget: usize) -> Vec<ChunkPiece> {
- let mut pieces = chunk_segments(&self.text, &self.segments, budget);
- for p in &mut pieces {
- let span = p.char_start as usize..p.char_end as usize;
- let Some(anchor) = p.provenance.anchor.as_mut() else {
- continue;
- };
- let page = anchor["page"].as_u64();
- let covered = self
- .regions
- .iter()
- .filter(|r| {
- Some(r.page) == page && r.range.start < span.end && span.start < r.range.end
- })
- .filter_map(|r| r.bbox);
- if let Some(b) = covered.reduce(|a, b| {
- [
- a[0].min(b[0]),
- a[1].min(b[1]),
- a[2].max(b[2]),
- a[3].max(b[3]),
- ]
- }) {
- anchor["bbox"] = json!(b);
- }
- }
- pieces
- }
-}
-
/// 读 `content_list`。`model` 记在出处上:哪个版本、哪个后端读的
pub fn reading(content_list: &Value, model: &str) -> Reading {
let mut text = String::new();
@@ -92,17 +43,23 @@ pub fn reading(content_list: &Value, model: &str) -> Reading {
text.push_str(body);
regions.push(Region {
range: start..text.len(),
- page: item["page_idx"].as_u64().unwrap_or(0) + 1,
- bbox: bbox(&item["bbox"]),
+ place: Place::Page {
+ page: item["page_idx"].as_u64().unwrap_or(0) + 1,
+ bbox: bbox(&item["bbox"]),
+ },
});
}
// 一页一段,段首尾相接盖满整份正文:段从这一页第一个区域开始,到下一页第一个区域为止
let mut segments: Vec<Segment> = Vec::new();
for r in &regions {
- if segments.last().is_some_and(|s| {
- s.provenance.anchor.as_ref().map(|a| &a["page"]) == Some(&json!(r.page))
- }) {
+ let Place::Page { page, .. } = r.place else {
+ continue;
+ };
+ if segments
+ .last()
+ .is_some_and(|s| s.provenance.anchor.as_ref().map(|a| &a["page"]) == Some(&json!(page)))
+ {
continue;
}
if let Some(prev) = segments.last_mut() {
@@ -113,7 +70,7 @@ pub fn reading(content_list: &Value, model: &str) -> Reading {
provenance: Provenance {
origin: Origin::Ocr,
model: Some(model.to_string()),
- anchor: Some(json!({ "page": r.page })),
+ anchor: Some(json!({ "page": page })),
},
});
}
@@ -196,8 +153,8 @@ fn bbox(v: &Value) -> Option<[f64; 4]> {
Some(out)
}
-/// 认字认出来的 NUL 跟 PDF 文本层里的一样要剥(#611)
-fn without_nul(s: &str) -> String {
+/// 认字、转写出来的 NUL 跟 PDF 文本层里的一样要剥(#611)
+pub(crate) fn without_nul(s: &str) -> String {
s.replace('\0', "")
}
diff --git a/crates/utopia-ingest/src/reading.rs b/crates/utopia-ingest/src/reading.rs
new file mode 100644
index 00000000..315af3eb
--- /dev/null
+++ b/crates/utopia-ingest/src/reading.rs
@@ -0,0 +1,99 @@
+//! 模型读出来的正文(0040):拼好的一份文字、按出处分的段,以及每一小段在原文件里的位置。
+//!
+//! 扫描件交回的是一个个带页码和框的区域,录音交回的是一句句带起止时刻和说话人的分段。两者
+//! 拼成正文之后按同一套规矩切块;切好之后,每块的锚点从它盖住的那几小段算出来:扫描件取框的
+//! 外接矩形,录音取最早的开始、最晚的结束和出现过的说话人。
+//!
+//! 段比小段粗(扫描件一页一段,录音整段一段),因为一块只装一种出处(决定 2):按小段分,
+//! 一句话就是一块,块碎到抽取看不见上下文。
+
+use crate::chunker::{chunk_segments, ChunkPiece};
+use crate::provenance::Segment;
+use serde_json::json;
+use std::ops::Range;
+
+/// 拼好的正文,连同出处和每一小段的位置
+#[derive(Debug, Clone)]
+pub struct Reading {
+ pub text: String,
+ pub segments: Vec<Segment>,
+ pub(crate) regions: Vec<Region>,
+}
+
+/// 正文里的一小段,和它在原文件里的位置
+#[derive(Debug, Clone)]
+pub(crate) struct Region {
+ pub range: Range<usize>,
+ pub place: Place,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) enum Place {
+ /// 扫描页上的一块:页码从 1 数,框是引擎给的(0–1000 归一化)
+ Page { page: u64, bbox: Option<[f64; 4]> },
+ /// 录音里的一句
+ Time {
+ start_ms: u64,
+ end_ms: u64,
+ speaker: String,
+ },
+}
+
+impl Reading {
+ /// 按段切块,再按每块盖住的小段写锚点
+ pub fn chunk(&self, budget: usize) -> Vec<ChunkPiece> {
+ let mut pieces = chunk_segments(&self.text, &self.segments, budget);
+ for p in &mut pieces {
+ let span = p.char_start as usize..p.char_end as usize;
+ let Some(anchor) = p.provenance.anchor.as_mut() else {
+ continue;
+ };
+ let covered = self
+ .regions
+ .iter()
+ .filter(|r| r.range.start < span.end && span.start < r.range.end);
+ let page = anchor["page"].as_u64();
+ let mut bbox: Option<[f64; 4]> = None;
+ let mut time: Option<(u64, u64)> = None;
+ let mut speakers: Vec<&str> = Vec::new();
+ for r in covered {
+ match &r.place {
+ Place::Page { page: pg, bbox: b } if Some(*pg) == page => {
+ if let Some(b) = b {
+ bbox = Some(bbox.map_or(*b, |a| {
+ [
+ a[0].min(b[0]),
+ a[1].min(b[1]),
+ a[2].max(b[2]),
+ a[3].max(b[3]),
+ ]
+ }));
+ }
+ }
+ Place::Page { .. } => {}
+ Place::Time {
+ start_ms,
+ end_ms,
+ speaker,
+ } => {
+ time = Some(time.map_or((*start_ms, *end_ms), |(s, e)| {
+ (s.min(*start_ms), e.max(*end_ms))
+ }));
+ if !speakers.contains(&speaker.as_str()) {
+ speakers.push(speaker);
+ }
+ }
+ }
+ }
+ if let Some(b) = bbox {
+ anchor["bbox"] = json!(b);
+ }
+ if let Some((start, end)) = time {
+ anchor["start_ms"] = json!(start);
+ anchor["end_ms"] = json!(end);
+ anchor["speaker"] = json!(speakers);
+ }
+ }
+ pieces
+ }
+}
diff --git a/crates/utopia-ingest/src/transcript.rs b/crates/utopia-ingest/src/transcript.rs
new file mode 100644
index 00000000..1abfa424
--- /dev/null
+++ b/crates/utopia-ingest/src/transcript.rs
@@ -0,0 +1,173 @@
+//! 转写服务交回的分段 → 一份正文 + 出处(0040 第三刀)。
+//!
+//! 请求走 OpenAI 的 `/audio/transcriptions`,要 `diarized_json`:`segments` 里每一句带
+//! `speaker`、`text`、`start`、`end`(秒)。
+//!
+//! **说话人写进正文。**抽取只读文字,锚点里的说话人它看不见;「我们三季度交付」谁说的,要在
+//! 那一行字上。连着是同一个人说的几句并成一段(一个发言轮次),换人另起一段,段首写
+//! `Speaker A:`——前缀是结构,标签是服务给的,不猜名字。
+//!
+//! 整段录音是一段出处,块按预算切;每块的锚点取它盖住的那几句的最早开始、最晚结束和
+//! 出现过的说话人(按出场顺序)。一句的时间精确到它自己,一块的时间就是那几句连起来。
+
+use crate::provenance::{Origin, Provenance, Segment};
+use crate::reading::{Place, Reading, Region};
+use crate::{NoSpeakers, Unreadable};
+use serde_json::{json, Value};
+
+/// 读转写结果。有一句带字却没有说话人,整份拒收
+pub fn reading(response: &Value, model: &str) -> anyhow::Result<Reading> {
+ let segments = response["segments"]
+ .as_array()
+ .map(Vec::as_slice)
+ .unwrap_or(&[]);
+ let mut text = String::new();
+ let mut regions: Vec<Region> = Vec::new();
+ let mut last_speaker: Option<String> = None;
+ for seg in segments {
+ let said = crate::mineru::without_nul(seg["text"].as_str().unwrap_or_default());
+ let said = said.split_whitespace().collect::<Vec<_>>().join(" ");
+ if said.is_empty() {
+ continue;
+ }
+ let speaker = match &seg["speaker"] {
+ Value::String(s) if !s.trim().is_empty() => s.trim().to_string(),
+ Value::Number(n) => n.to_string(),
+ _ => return Err(NoSpeakers.into()),
+ };
+ let start = text.len();
+ if last_speaker.as_deref() == Some(speaker.as_str()) {
+ text.push(' ');
+ } else {
+ if !text.is_empty() {
+ text.push_str("\n\n");
+ }
+ text.push_str(&format!("Speaker {speaker}: "));
+ }
+ text.push_str(&said);
+ regions.push(Region {
+ range: start..text.len(),
+ place: Place::Time {
+ start_ms: millis(&seg["start"]),
+ end_ms: millis(&seg["end"]),
+ speaker: speaker.clone(),
+ },
+ });
+ last_speaker = Some(speaker);
+ }
+ if text.is_empty() {
+ // 整段没有一句话;只有一句不分段的全文,也是分不出谁说的
+ if response["text"]
+ .as_str()
+ .is_some_and(|t| !t.trim().is_empty())
+ {
+ return Err(NoSpeakers.into());
+ }
+ return Err(Unreadable("No speech was found in this recording".into()).into());
+ }
+ let (first, last) = (&regions[0], &regions[regions.len() - 1]);
+ let whole = |p: &Place| match p {
+ Place::Time {
+ start_ms, end_ms, ..
+ } => (*start_ms, *end_ms),
+ Place::Page { .. } => (0, 0),
+ };
+ // 段上的锚点先盖住整段录音,切块之后按每块盖住的那几句改写
+ let anchor = json!({
+ "start_ms": whole(&first.place).0,
+ "end_ms": whole(&last.place).1,
+ "speaker": [],
+ });
+ Ok(Reading {
+ segments: vec![Segment {
+ range: 0..text.len(),
+ provenance: Provenance {
+ origin: Origin::Transcribed,
+ model: Some(model.to_string()),
+ anchor: Some(anchor),
+ },
+ }],
+ text,
+ regions,
+ })
+}
+
+fn millis(v: &Value) -> u64 {
+ (v.as_f64().unwrap_or(0.0).max(0.0) * 1000.0).round() as u64
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn meeting() -> Value {
+ json!({
+ "text": "…",
+ "segments": [
+ { "id": "seg_0", "type": "transcript.text.segment", "speaker": "A", "start": 0.0, "end": 2.4, "text": "Let's review the Beta Robotics deal." },
+ { "id": "seg_1", "type": "transcript.text.segment", "speaker": "A", "start": 2.4, "end": 5.1, "text": " Who owns delivery?" },
+ { "id": "seg_2", "type": "transcript.text.segment", "speaker": "B", "start": 5.3, "end": 9.05, "text": "I will deliver the prototype in Q3." },
+ { "id": "seg_3", "type": "transcript.text.segment", "speaker": "A", "start": 9.2, "end": 10.0, "text": " " }
+ ]
+ })
+ }
+
+ #[test]
+ fn a_turn_says_who_spoke_it() {
+ let r = reading(&meeting(), "gpt-4o-transcribe-diarize").unwrap();
+ assert_eq!(
+ r.text,
+ "Speaker A: Let's review the Beta Robotics deal. Who owns delivery?\n\nSpeaker B: I will deliver the prototype in Q3."
+ );
+ assert_eq!(r.segments.len(), 1);
+ assert_eq!(r.segments[0].range, 0..r.text.len());
+ }
+
+ #[test]
+ fn a_chunk_carries_its_times_and_speakers() {
+ let r = reading(&meeting(), "gpt-4o-transcribe-diarize").unwrap();
+ let pieces = r.chunk(300);
+ assert_eq!(pieces.len(), 1);
+ assert_eq!(pieces[0].provenance.origin, Origin::Transcribed);
+ assert_eq!(
+ pieces[0].provenance.anchor,
+ Some(json!({ "start_ms": 0, "end_ms": 9050, "speaker": ["A", "B"] }))
+ );
+ // 块小一点:每块只记它自己那几句的时间和说话人
+ let small = r.chunk(12);
+ assert!(small.len() > 1, "{small:#?}");
+ let last = small.last().unwrap();
+ assert_eq!(
+ last.provenance.anchor.as_ref().unwrap()["speaker"],
+ json!(["B"])
+ );
+ assert_eq!(
+ last.provenance.anchor.as_ref().unwrap()["end_ms"],
+ json!(9050)
+ );
+ assert!(
+ last.provenance.anchor.as_ref().unwrap()["start_ms"]
+ .as_u64()
+ .unwrap()
+ >= 5300
+ );
+ }
+
+ #[test]
+ fn a_transcript_without_speakers_is_refused() {
+ let unlabelled =
+ json!({ "segments": [{ "start": 0.0, "end": 1.0, "text": "We ship in Q3." }] });
+ let err = reading(&unlabelled, "whisper-1").unwrap_err();
+ assert!(err.downcast_ref::<NoSpeakers>().is_some());
+ let plain = json!({ "text": "We ship in Q3." });
+ assert!(reading(&plain, "whisper-1")
+ .unwrap_err()
+ .downcast_ref::<NoSpeakers>()
+ .is_some());
+ let silent = json!({ "text": "", "segments": [] });
+ assert!(reading(&silent, "m")
+ .unwrap_err()
+ .downcast_ref::<Unreadable>()
+ .is_some());
+ }
+}
diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rs
index 377470b3..29dc1da1 100644
--- a/crates/utopia-server/src/alerting.rs
+++ b/crates/utopia-server/src/alerting.rs
@@ -63,7 +63,8 @@ pub async fn observe_document_needs_reader(
kb_id: uuid::Uuid,
document_id: uuid::Uuid,
filename: &str,
- needs: &utopia_ingest::NeedsReader,
+ reader: utopia_ingest::Reader,
+ error: &str,
) {
if let Err(e) = alerts::raise(
&state.pool,
@@ -76,8 +77,8 @@ pub async fn observe_document_needs_reader(
subject_id: Some(document_id),
detail: serde_json::json!({
"name": filename,
- "reader": needs.reader.as_str(),
- "error": needs.to_string(),
+ "reader": reader.as_str(),
+ "error": error,
}),
},
)
diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs
index 00bbf6e9..4d4cd0ba 100644
--- a/crates/utopia-server/src/api/mod.rs
+++ b/crates/utopia-server/src/api/mod.rs
@@ -143,6 +143,10 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router {
"/workspaces/{id}/settings/ocr",
put(settings_routes::put_ocr),
)
+ .route(
+ "/workspaces/{id}/settings/transcribe",
+ put(settings_routes::put_transcribe),
+ )
.route(
"/workspaces/{id}/settings/test",
post(settings_routes::test),
diff --git a/crates/utopia-server/src/api/settings_routes.rs b/crates/utopia-server/src/api/settings_routes.rs
index fc6d9e09..d6822ec8 100644
--- a/crates/utopia-server/src/api/settings_routes.rs
+++ b/crates/utopia-server/src/api/settings_routes.rs
@@ -32,6 +32,9 @@ pub async fn get(
"ocr_base_url": s.ocr_base_url,
"ocr_backend": s.ocr_backend,
"has_ocr_key": s.ocr_api_key.as_deref().is_some_and(|k| !k.is_empty()),
+ "transcribe_base_url": s.transcribe_base_url,
+ "transcribe_model": s.transcribe_model,
+ "has_transcribe_key": s.transcribe_api_key.as_deref().is_some_and(|k| !k.is_empty()),
}),
}))
}
@@ -144,6 +147,55 @@ pub async fn put_ocr(
Ok(Json(json!({ "ok": true, "requeued": requeued })))
}
+#[derive(Deserialize)]
+pub struct PutTranscribeReq {
+ /// 地址或模型空 = 关掉:录音照第一刀降级
+ pub base_url: Option<String>,
+ /// None 或空串 = 保留旧密钥
+ pub api_key: Option<String>,
+ pub model: Option<String>,
+}
+
+/// 转写模型(0040 第三刀)。跟版面识别服务一样单独一个接口;配上的这一刀,这个工作区里
+/// 等着转写的录音重新排队——包括上一个模型分不出说话人、被拒收的那些
+pub async fn put_transcribe(
+ State(state): State<AppState>,
+ AuthUser(user): AuthUser,
+ Path(workspace_id): Path<Uuid>,
+ Json(req): Json<PutTranscribeReq>,
+) -> ApiResult<Json<serde_json::Value>> {
+ utopia_store::workspaces::require_role(&state.pool, user.id, workspace_id, Role::Admin).await?;
+ let nonempty = |v: &Option<String>| -> Option<String> {
+ v.as_deref()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .map(String::from)
+ };
+ let (base_url, model) = (nonempty(&req.base_url), nonempty(&req.model));
+ utopia_store::settings::upsert_transcribe(
+ &state.pool,
+ workspace_id,
+ base_url.as_deref(),
+ nonempty(&req.api_key).as_deref(),
+ model.as_deref(),
+ )
+ .await?;
+ let mut requeued = 0usize;
+ if base_url.is_some() && model.is_some() {
+ let docs = utopia_store::documents::requeue_waiting_for_reader(
+ &state.pool,
+ workspace_id,
+ "transcribe",
+ )
+ .await?;
+ requeued = docs.len();
+ for (document_id, kb_id) in docs {
+ state.emit_document(kb_id, document_id);
+ }
+ }
+ Ok(Json(json!({ "ok": true, "requeued": requeued })))
+}
+
/// 连通性测试:对话发一条最小消息;embedding 试算一条并返回维度;版面识别服务问一声健康。
pub async fn test(
State(state): State<AppState>,
@@ -155,7 +207,8 @@ pub async fn test(
return Ok(Json(
json!({ "chat": { "ok": false, "error": "Not configured" },
"embed": { "ok": false, "error": "Not configured" },
- "ocr": { "ok": false, "error": "Not configured" } }),
+ "ocr": { "ok": false, "error": "Not configured" },
+ "transcribe": { "ok": false, "error": "Not configured" } }),
));
};
@@ -192,7 +245,18 @@ pub async fn test(
},
};
- Ok(Json(
- json!({ "chat": chat_result, "embed": embed_result, "ocr": ocr_result }),
- ))
+ let transcribe_result = match crate::readers::Transcriber::from_settings(&s) {
+ None => json!({ "ok": false, "error": "Not configured" }),
+ Some(t) => match t.check().await {
+ Ok(()) => json!({ "ok": true }),
+ Err(e) => json!({ "ok": false, "error": format!("{e:#}") }),
+ },
+ };
+
+ Ok(Json(json!({
+ "chat": chat_result,
+ "embed": embed_result,
+ "ocr": ocr_result,
+ "transcribe": transcribe_result,
+ })))
}
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 40d0af64..54053f62 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -44,13 +44,21 @@ pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Re
.ok();
// 字要靠模型读、而那种模型没配(0040):**降级**。文件留着,文档停在 failed
// 并记下缺哪一种,报一条库级告警;配上模型会自己重新处理。重试没用——
- // 模型不会在两分钟里自己配上,所以挂 Terminal
- if let Some(needs) = e.downcast_ref::<utopia_ingest::NeedsReader>() {
+ // 模型不会在两分钟里自己配上,所以挂 Terminal。
+ // 转写回来却分不出说话人,跟没配一样对待(决定 5):换个会标说话人的模型再来
+ let waiting = e
+ .downcast_ref::<utopia_ingest::NeedsReader>()
+ .map(|n| n.reader)
+ .or_else(|| {
+ e.downcast_ref::<utopia_ingest::NoSpeakers>()
+ .map(|_| utopia_ingest::Reader::Transcribe)
+ });
+ if let Some(reader) = waiting {
let _ = utopia_store::documents::set_needs_reader(
&state.pool,
document_id,
- needs.reader.as_str(),
- &needs.to_string(),
+ reader.as_str(),
+ &e.to_string(),
)
.await;
if let Some(doc) = &doc {
@@ -59,7 +67,8 @@ pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Re
doc.kb_id,
document_id,
&doc.filename,
- needs,
+ reader,
+ &e.to_string(),
)
.await;
state.emit_document(doc.kb_id, document_id);
@@ -110,26 +119,36 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
(text, pieces)
}
// 没有文本层的扫描件、图片:工作区配了版面识别服务就交给它读(0040 第二刀),
- // 按页分段切块,每块记着页码和框。没配的照旧往上抛,由外面降级
- Err(e) => match (
- e.downcast_ref::<utopia_ingest::NeedsReader>(),
- settings
- .as_ref()
- .and_then(crate::readers::Ocr::from_settings),
- ) {
- (Some(needs), Some(ocr)) if needs.reader == utopia_ingest::Reader::Ocr => {
- let reading = ocr.read(state, &doc, bytes).await?;
- if reading.text.trim().is_empty() {
- return Err(utopia_ingest::Unreadable(
- "The OCR service found no text in this file".into(),
- )
- .into());
+ // 按页分段切块,每块记着页码和框。录音交给会标说话人的转写模型(第三刀),
+ // 每块记着起止时刻和说话人。没配的照旧往上抛,由外面降级
+ Err(e) => {
+ let Some(needs) = e.downcast_ref::<utopia_ingest::NeedsReader>() else {
+ return Err(e);
+ };
+ let reading = match (needs.reader, settings.as_ref()) {
+ (utopia_ingest::Reader::Ocr, Some(s)) => {
+ match crate::readers::Ocr::from_settings(s) {
+ Some(ocr) => ocr.read(state, &doc, bytes).await?,
+ None => return Err(e),
+ }
+ }
+ (utopia_ingest::Reader::Transcribe, Some(s)) => {
+ match crate::readers::Transcriber::from_settings(s) {
+ Some(t) => t.read(&doc, bytes).await?,
+ None => return Err(e),
+ }
}
- let pieces = reading.chunk(state.chunk_tokens);
- (reading.text, pieces)
+ (_, None) => return Err(e),
+ };
+ if reading.text.trim().is_empty() {
+ return Err(utopia_ingest::Unreadable(
+ "The reader found no text in this file".into(),
+ )
+ .into());
}
- _ => return Err(e),
- },
+ let pieces = reading.chunk(state.chunk_tokens);
+ (reading.text, pieces)
+ }
};
let text_len = text.chars().count() as i32;
let chunk_pairs =
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index 6723d685..23ae9719 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -17,6 +17,7 @@
//! 落在 failed。现在走完整的 process_document 到 ready,库里没有一个分块带 NUL。
//!
//! 8. **扫描件等版面识别服务读**(0040 第二刀):配上服务重新排队,交一次、挂回去问、按页切块。
+//! 9. **录音等会标说话人的转写模型读**(第三刀):说话人进正文、时刻进锚点;分不出说话人的降级。
//!
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
@@ -638,6 +639,153 @@ async fn a_failed_read_is_retried_from_a_fresh_submission() -> anyhow::Result<()
f.cleanup().await
}
+/// 假的转写端点:`labels` 时每句带说话人,否则只有时间。记下收到的表单
+#[derive(Clone, Default)]
+struct FakeTranscriber {
+ labels: bool,
+ requests: Arc<Mutex<Vec<String>>>,
+}
+
+impl Respond for FakeTranscriber {
+ fn respond(&self, request: &Request) -> ResponseTemplate {
+ let auth = request
+ .headers
+ .get("authorization")
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or_default()
+ .to_string();
+ self.requests.lock().expect("lock").push(format!(
+ "{auth}\n{}",
+ String::from_utf8_lossy(&request.body)
+ ));
+ let seg = |speaker: &str, start: f64, end: f64, text: &str| {
+ if self.labels {
+ serde_json::json!({ "type": "transcript.text.segment", "speaker": speaker, "start": start, "end": end, "text": text })
+ } else {
+ serde_json::json!({ "start": start, "end": end, "text": text })
+ }
+ };
+ ResponseTemplate::new(200).set_body_json(serde_json::json!({
+ "text": "…",
+ "segments": [
+ seg("A", 0.0, 3.2, "Who delivers the Beta Robotics prototype?"),
+ seg("B", 3.5, 7.25, "I will deliver it in Q3."),
+ ]
+ }))
+ }
+}
+
+async fn with_transcriber(f: &Fx, fake: &FakeTranscriber, model: &str) -> anyhow::Result<()> {
+ f.server.reset().await;
+ Mock::given(method("POST"))
+ .and(path("/embeddings"))
+ .respond_with(f.fake.clone())
+ .mount(&f.server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/asr/audio/transcriptions"))
+ .respond_with(fake.clone())
+ .mount(&f.server)
+ .await;
+ utopia_store::settings::upsert_transcribe(
+ &f.pool,
+ f.ws,
+ Some(&format!("{}/asr", f.server.uri())),
+ Some("asr-secret"),
+ Some(model),
+ )
+ .await?;
+ Ok(())
+}
+
+const MP3: [u8; 12] = [b'I', b'D', b'3', 4, 0, 0, 0, 0, 0, 0, 0xFF, 0xFB];
+
+/// 0040 第三刀:录音交给会标说话人的转写模型。说话人写进正文,每块记着起止时刻和说话人
+#[tokio::test]
+async fn a_recording_is_read_with_who_said_what() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let fake = FakeTranscriber {
+ labels: true,
+ ..Default::default()
+ };
+ with_transcriber(&f, &fake, "gpt-4o-transcribe-diarize").await?;
+ let doc = f.document_with_bytes("board-meeting.mp3", &MP3).await?;
+ super::process_document(&f.state, doc).await?;
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "ready", "{:?}", row.error);
+
+ let req = fake.requests.lock().expect("lock").clone();
+ assert_eq!(req.len(), 1);
+ assert!(req[0].starts_with("Bearer asr-secret\n"));
+ assert!(req[0].contains("diarized_json"));
+ assert!(req[0].contains("gpt-4o-transcribe-diarize"));
+
+ type Stored = (String, String, Option<String>, Option<serde_json::Value>);
+ let chunks: Vec<Stored> = sqlx::query_as(
+ "SELECT text, origin, origin_model, anchor FROM chunks
+ WHERE document_id = $1 AND superseded_at IS NULL ORDER BY seq",
+ )
+ .bind(doc)
+ .fetch_all(&f.pool)
+ .await?;
+ assert_eq!(chunks.len(), 1, "{chunks:#?}");
+ assert_eq!(chunks[0].1, "transcribed");
+ assert_eq!(chunks[0].2.as_deref(), Some("gpt-4o-transcribe-diarize"));
+ assert!(chunks[0].0.contains("Speaker B: I will deliver it in Q3."));
+ assert_eq!(
+ chunks[0].3,
+ Some(serde_json::json!({ "start_ms": 0, "end_ms": 7250, "speaker": ["A", "B"] }))
+ );
+ f.cleanup().await
+}
+
+/// 分不出说话人的转写:跟没配一样降级(决定 5)——不进库、告警说原因、不重试;换一个会标
+/// 说话人的模型存下,录音重新排队读成
+#[tokio::test]
+async fn a_transcript_that_cannot_say_who_spoke_waits_for_one_that_can() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let unlabelled = FakeTranscriber::default();
+ with_transcriber(&f, &unlabelled, "whisper-1").await?;
+ let doc = f.document_with_bytes("board-meeting.mp3", &MP3).await?;
+ let err = super::process_document(&f.state, doc)
+ .await
+ .expect_err("no speakers");
+ assert!(utopia_core::is_terminal(&err));
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "failed");
+ assert_eq!(row.reader_needed.as_deref(), Some("transcribe"));
+ assert!(row
+ .error
+ .as_deref()
+ .is_some_and(|e| e.contains("who spoke")));
+ assert!(
+ f.stored(doc).await?.is_empty(),
+ "nothing unattributed is kept"
+ );
+ let alerts = f.alerts("document.needs_reader").await?;
+ assert_eq!(alerts.len(), 1);
+ assert_eq!(alerts[0]["reader"], "transcribe");
+
+ let labelled = FakeTranscriber {
+ labels: true,
+ ..Default::default()
+ };
+ with_transcriber(&f, &labelled, "gpt-4o-transcribe-diarize").await?;
+ let requeued =
+ utopia_store::documents::requeue_waiting_for_reader(&f.pool, f.ws, "transcribe").await?;
+ assert_eq!(requeued, vec![(doc, f.kb)]);
+ super::process_document(&f.state, doc).await?;
+ assert_eq!(
+ utopia_store::documents::get(&f.pool, doc).await?.status,
+ "ready"
+ );
+ f.cleanup().await
+}
+
/// 读不了的格式(老式 .doc、压缩包):一次失败、不重试,也不进库成乱码;它不缺模型,不报那条告警
#[tokio::test]
async fn a_binary_file_fails_once_instead_of_becoming_text() -> anyhow::Result<()> {
diff --git a/crates/utopia-server/src/readers.rs b/crates/utopia-server/src/readers.rs
index 6af63ccf..9b4cb8bf 100644
--- a/crates/utopia-server/src/readers.rs
+++ b/crates/utopia-server/src/readers.rs
@@ -1,4 +1,5 @@
-//! 读字的服务(0040):扫描件和图片交给版面识别服务 MinerU(`mineru-api`)。
+//! 读字的服务(0040):扫描件和图片交给版面识别服务 MinerU(`mineru-api`),录音交给会标
+//! 说话人的转写模型。
//!
//! 识别一份几百页的扫描件要几分钟到几十分钟。处理任务不在这里干等:第一次来交文件、把
//! 远端任务号记在文档上(`documents.reader_task`),挂 `Deferred` 回队列;之后每次来问
@@ -16,7 +17,7 @@ use anyhow::{anyhow, Context};
use serde_json::{json, Value};
use utopia_core::models::{Document, LlmSettings};
use utopia_core::{Deferred, Terminal};
-use utopia_ingest::mineru::Reading;
+use utopia_ingest::Reading;
use crate::state::AppState;
@@ -218,6 +219,105 @@ impl<'a> Ocr<'a> {
}
}
+/// 转写一段录音最多等多久。OpenAI 一次收 25MB,大约是一小时的压缩音频;本地服务读一小时
+/// 录音在 CPU 上要十几分钟
+const TRANSCRIBE_TIMEOUT: Duration = Duration::from_secs(1800);
+
+/// 工作区配的转写模型:OpenAI 的 `/audio/transcriptions`,要 `diarized_json`
+pub struct Transcriber<'a> {
+ base: &'a str,
+ key: Option<&'a str>,
+ model: &'a str,
+}
+
+impl<'a> Transcriber<'a> {
+ pub fn from_settings(s: &'a LlmSettings) -> Option<Self> {
+ Some(Transcriber {
+ base: s.transcribe_base_url.as_deref()?.trim_end_matches('/'),
+ key: s.transcribe_api_key.as_deref().filter(|k| !k.is_empty()),
+ model: s.transcribe_model.as_deref()?,
+ })
+ }
+
+ async fn transcribe(
+ &self,
+ filename: &str,
+ bytes: Vec<u8>,
+ timeout: Duration,
+ ) -> anyhow::Result<Value> {
+ let client = crate::query_engine::http()?;
+ let part = reqwest::multipart::Part::bytes(bytes)
+ .file_name(filename.to_string())
+ .mime_str("application/octet-stream")?;
+ let form = reqwest::multipart::Form::new()
+ .part("file", part)
+ .text("model", self.model.to_string())
+ .text("response_format", "diarized_json")
+ // 超过 30 秒的录音,diarize 模型要求给切分策略
+ .text("chunking_strategy", "auto");
+ let mut req = client
+ .post(format!("{}/audio/transcriptions", self.base))
+ .multipart(form)
+ .timeout(timeout);
+ if let Some(key) = self.key {
+ req = req.bearer_auth(key);
+ }
+ let resp = req
+ .send()
+ .await
+ .context("the transcription model is unreachable")?;
+ let status = resp.status();
+ if !status.is_success() {
+ let body = resp.text().await.unwrap_or_default();
+ return Err(anyhow!(
+ "The transcription model refused this recording ({status}): {}",
+ body.chars().take(300).collect::<String>()
+ ));
+ }
+ resp.json()
+ .await
+ .context("The transcription model returned something that is not JSON")
+ }
+
+ /// 读这段录音。一次请求读完——没有可以接着问的远端任务;分不出说话人的结果在
+ /// `transcript::reading` 里拒收(`NoSpeakers`)
+ pub async fn read(&self, doc: &Document, bytes: Vec<u8>) -> anyhow::Result<Reading> {
+ let response = self
+ .transcribe(&doc.filename, bytes, TRANSCRIBE_TIMEOUT)
+ .await?;
+ utopia_ingest::transcript::reading(&response, self.model)
+ }
+
+ /// 连通性测试:送一秒静音。端点和模型认这个请求就算通——静音里没有说话人可标,
+ /// 标不标得出要等第一段真录音
+ pub async fn check(&self) -> anyhow::Result<()> {
+ self.transcribe("silence.wav", silent_wav(), Duration::from_secs(60))
+ .await
+ .map(|_| ())
+ }
+}
+
+/// 一秒 16kHz 单声道静音的 WAV
+fn silent_wav() -> Vec<u8> {
+ let (rate, samples): (u32, u32) = (16_000, 16_000);
+ let data_len = samples * 2;
+ let mut w = Vec::with_capacity(44 + data_len as usize);
+ w.extend_from_slice(b"RIFF");
+ w.extend_from_slice(&(36 + data_len).to_le_bytes());
+ w.extend_from_slice(b"WAVEfmt ");
+ w.extend_from_slice(&16u32.to_le_bytes());
+ w.extend_from_slice(&1u16.to_le_bytes());
+ w.extend_from_slice(&1u16.to_le_bytes());
+ w.extend_from_slice(&rate.to_le_bytes());
+ w.extend_from_slice(&(rate * 2).to_le_bytes());
+ w.extend_from_slice(&2u16.to_le_bytes());
+ w.extend_from_slice(&16u16.to_le_bytes());
+ w.extend_from_slice(b"data");
+ w.extend_from_slice(&data_len.to_le_bytes());
+ w.resize(44 + data_len as usize, 0);
+ w
+}
+
fn jstr(v: &Value) -> &str {
v.as_str().unwrap_or_default()
}
diff --git a/crates/utopia-store/src/sealing.rs b/crates/utopia-store/src/sealing.rs
index 36876971..7c835a3b 100644
--- a/crates/utopia-store/src/sealing.rs
+++ b/crates/utopia-store/src/sealing.rs
@@ -1,7 +1,7 @@
//! 凭据补封:升级前落库的明文凭据,在钥匙装好之后的第一次启动补成密文。
//!
//! 幂等,每次启动都跑:判据是「值没有 `enc:v1:` 前缀」,已封的不动。四处凭据
-//! (`llm_settings` 三把 key、`data_sources.conn_string`、`sources.config` 的凭据键、
+//! (`llm_settings` 四把 key、`data_sources.conn_string`、`sources.config` 的凭据键、
//! `sources.ingest_token`)在这里各扫一遍——加了新的存放处要同时加到这里,否则那一处
//! 会一直是明文而没有任何地方报警。
@@ -22,27 +22,37 @@ pub async fn backfill(pool: &PgPool) -> AppResult<usize> {
/// `only` = 只补这一个工作区(测试用;启动时传 None 扫全部)
pub async fn seal_llm_settings(pool: &PgPool, only: Option<Uuid>) -> AppResult<usize> {
- type Keys = (Uuid, Option<String>, Option<String>, Option<String>);
+ type Keys = (
+ Uuid,
+ Option<String>,
+ Option<String>,
+ Option<String>,
+ Option<String>,
+ );
let rows: Vec<Keys> = sqlx::query_as(
- "SELECT workspace_id, chat_api_key, embed_api_key, ocr_api_key FROM llm_settings
+ "SELECT workspace_id, chat_api_key, embed_api_key, ocr_api_key, transcribe_api_key
+ FROM llm_settings
WHERE ($1::uuid IS NULL OR workspace_id = $1)
AND ((chat_api_key IS NOT NULL AND chat_api_key NOT LIKE 'enc:v1:%')
OR (embed_api_key IS NOT NULL AND embed_api_key NOT LIKE 'enc:v1:%')
- OR (ocr_api_key IS NOT NULL AND ocr_api_key NOT LIKE 'enc:v1:%'))",
+ OR (ocr_api_key IS NOT NULL AND ocr_api_key NOT LIKE 'enc:v1:%')
+ OR (transcribe_api_key IS NOT NULL AND transcribe_api_key NOT LIKE 'enc:v1:%'))",
)
.bind(only)
.fetch_all(pool)
.await?;
let n = rows.len();
- for (ws, chat, embed, ocr) in rows {
+ for (ws, chat, embed, ocr, transcribe) in rows {
sqlx::query(
- "UPDATE llm_settings SET chat_api_key = $2, embed_api_key = $3, ocr_api_key = $4
+ "UPDATE llm_settings SET chat_api_key = $2, embed_api_key = $3, ocr_api_key = $4,
+ transcribe_api_key = $5
WHERE workspace_id = $1",
)
.bind(ws)
.bind(secrets::seal_opt(chat.as_deref()))
.bind(secrets::seal_opt(embed.as_deref()))
.bind(secrets::seal_opt(ocr.as_deref()))
+ .bind(secrets::seal_opt(transcribe.as_deref()))
.execute(pool)
.await?;
}
diff --git a/crates/utopia-store/src/settings.rs b/crates/utopia-store/src/settings.rs
index 25e8a9d5..91e9b7c4 100644
--- a/crates/utopia-store/src/settings.rs
+++ b/crates/utopia-store/src/settings.rs
@@ -3,12 +3,14 @@ use utopia_core::models::LlmSettings;
use utopia_core::{secrets, AppError, AppResult};
use uuid::Uuid;
-/// 出库即开封:三把 API key 在库里是封印的(`utopia_core::secrets`)。
+/// 出库即开封:四把 API key 在库里是封印的(`utopia_core::secrets`)。
/// 任何返回 `LlmSettings` 的查询都从这里过
fn opened(mut s: LlmSettings) -> AppResult<LlmSettings> {
s.chat_api_key = secrets::open_opt(s.chat_api_key.as_deref()).map_err(AppError::Other)?;
s.embed_api_key = secrets::open_opt(s.embed_api_key.as_deref()).map_err(AppError::Other)?;
s.ocr_api_key = secrets::open_opt(s.ocr_api_key.as_deref()).map_err(AppError::Other)?;
+ s.transcribe_api_key =
+ secrets::open_opt(s.transcribe_api_key.as_deref()).map_err(AppError::Other)?;
Ok(s)
}
@@ -107,3 +109,32 @@ pub async fn upsert_ocr(
.await?;
opened(row)
}
+
+/// 转写模型的设置,跟版面识别服务一样单独存(管理页上各是一张卡片)
+pub async fn upsert_transcribe(
+ pool: &PgPool,
+ workspace_id: Uuid,
+ base_url: Option<&str>,
+ api_key: Option<&str>,
+ model: Option<&str>,
+) -> AppResult<LlmSettings> {
+ let api_key = secrets::seal_opt(api_key);
+ let row: LlmSettings = sqlx::query_as(
+ "INSERT INTO llm_settings
+ (workspace_id, transcribe_base_url, transcribe_api_key, transcribe_model, updated_at)
+ VALUES ($1, $2, $3, $4, now())
+ ON CONFLICT (workspace_id) DO UPDATE SET
+ transcribe_base_url = EXCLUDED.transcribe_base_url,
+ transcribe_api_key = COALESCE(EXCLUDED.transcribe_api_key, llm_settings.transcribe_api_key),
+ transcribe_model = EXCLUDED.transcribe_model,
+ updated_at = now()
+ RETURNING *",
+ )
+ .bind(workspace_id)
+ .bind(base_url)
+ .bind(api_key)
+ .bind(model)
+ .fetch_one(pool)
+ .await?;
+ opened(row)
+}
diff --git a/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md b/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
index f29119d2..62c814c1 100644
--- a/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
+++ b/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
@@ -12,7 +12,11 @@
service (`llm_settings.ocr_*`, migration 0059), one segment per page with the covered regions'
box in the anchor; the processing job waits on the service with `Deferred` and remembers the
remote task on the document; saving the service queues the waiting documents again · the
- settings card is its own interface cut · recordings are cut 3
+ settings cards are their own interface cut · cut 3 implemented (2026-09-16): recordings are read
+ by a workspace's transcription model (`llm_settings.transcribe_*`, migration 0060) through
+ `/audio/transcriptions` with `diarized_json`; speakers are written into the text and a chunk's
+ anchor carries its times and speakers; a transcript without speaker labels degrades like a
+ missing model
- **Written**: 2026-09-13 (conventions in the [README](README.md))
- **Related**: [0039](0039-a-chunk-is-what-extraction-sees.md) (#633, not merged yet) puts Docling
behind the block model as its cut 2 and leaves "evidence that points at a table cell or an image
@@ -93,7 +97,7 @@ the chunk, and the description would borrow the credibility of the sentence besi
|---|---|
| stated | null — `char_start` / `char_end` already place it in the parsed text |
| ocr | `{"page": n}`, with `"bbox": [x0, y0, x1, y1]` when the engine gives one |
-| transcribed | `{"start_ms": n, "end_ms": n}`, with `"speaker"` when the endpoint gives one |
+| transcribed | `{"start_ms": n, "end_ms": n, "speaker": ["A", "B"]}` — the speakers heard in the chunk, in order (revised with cut 3: required, and a list, because a chunk spans turns) |
| described | `{"page": n, "image": i}` for a PDF, `{"part": "word/media/image3.png"}` for an Office file, `{}` for a standalone image file |
The original is already kept: the document's blob, content-addressed and versioned in
@@ -193,6 +197,21 @@ the boxes of the regions it covers on its page. Page headers, footers and page n
dropped; tables become Markdown tables, so a long scanned table repeats its header across
chunks like any other.
+*Cut 3, as built (2026-09-16).* The request is OpenAI's `/audio/transcriptions` with
+`response_format=diarized_json` and `chunking_strategy=auto`; each returned segment carries a
+speaker, its text, and its start and end in seconds. Speakers go into the text, not only the
+anchor: extraction reads words, and who promised delivery has to be on the line that says it.
+Consecutive segments from one speaker join into a turn, and each turn starts `Speaker A:` with
+the label the model gave — the prefix is structure, and no name is guessed. The whole recording
+is one segment; after packing, a chunk's anchor carries the earliest start, the latest end and
+the speakers heard in it. A segment with text and no speaker refuses the whole transcript
+(`NoSpeakers`), which degrades like a missing model: the document waits with `reader_needed =
+'transcribe'`, the alert carries the reason, and saving a model queues it again.
+
+A recording is read in one request, so it is not resumable within itself: a failure retries the
+whole file. Splitting audio needs a decoder the server does not carry; the endpoint's file limit
+(25 MB on OpenAI, about an hour of compressed audio) bounds what one request loses.
+
### 7. The read contract says it
The evidence API, the MCP tool results (`quote`, `document_id` and `filename` today) and the RDF
@@ -210,7 +229,7 @@ the recording at `start_ms` — is a separate cut after the capability, not part
2. **Scans and document images through MinerU** (revised from Docling): `ocr`. The highest value —
scanned contracts, stamped approvals, invoices, all of which fail today with no text layer —
and the modality that keeps the verbatim contract.
-3. **Recordings**: `transcribed`, segment times required, speakers where the endpoint labels them.
+3. **Recordings**: `transcribed`, segment times and speakers required.
4. **Charts, photos, diagrams**: `described`, under the ceiling.
5. **Video**: its audio track through 3, sampled frames through 4.
diff --git a/docs/decisions/README.md b/docs/decisions/README.md
index df304110..8dca1b8e 100644
--- a/docs/decisions/README.md
+++ b/docs/decisions/README.md
@@ -63,7 +63,7 @@ The test for writing one: if someone (including us) looks at a piece of code in
| 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | Written · cut 1 in progress: qualifier tables (0049), declaration, extraction, panel, export · entity-valued qualifier reserved, conflict-as-two-rows and the canvas label next |
| 0038 | [The interface has a light side](0038-the-interface-has-a-light-side.md) | Implemented (#599) · `data-theme` on `<html>`, chosen by the reader and stored in the browser, `system` resolved to one of the two before first paint · a second token block in `styles.css` and not one rule rewritten: alpha stays, the triplet swaps · the canvas reads its colours from the tokens through one reader file and rebuilds on a switch; edges are flattened over the ground on light because the WebGL shader can only brighten · a `raw-colour` guard rule over `.ts` and `.tsx` · the entity palette is the same in both themes; a paper-tuned set is open |
| 0039 | [A chunk is what extraction sees](0039-a-chunk-is-what-extraction-sees.md) | Implemented, cut 1 · the parser's Markdown is read into blocks and packed by a token budget (1000, cl100k) · a table travels with its caption and header and a continuation repeats both · headings become a breadcrumb and fill `chunks.heading` · a page-break rule is not a boundary and a table it split is joined back · not measured on its own: the budget · cut 2: an external parser behind the block model, evidence that names a cell or a region |
-| 0040 | [A chunk says where its words came from](0040-a-chunk-says-where-its-words-came-from.md) | Cuts 1–2 implemented (the ledger shape, and a file that needs a reader degrades with an alert instead of becoming garbage text; scans and images read by a workspace's MinerU service, one segment per page, waiting on the service without spending retries) · scans and images through a MinerU service, recordings only with speaker labels (revised 2026-09-15) · a chunk carries an **origin** (`stated`, `ocr`, `transcribed`, `described`) and the model that produced it, and an **anchor** back into the original bytes (a page and region, a time range, an image inside the file) — decided before any media reader, because a transcript stored without its times can never be tied to the recording again · the packer never mixes origins in a chunk · facts from a description enter below `AUTO_CLOSE_MIN_CONFIDENCE`, so a misread chart opens a conflict instead of closing a correct fact · per-modality model settings, media reading as a resumable job, origin and anchor in the API, MCP and RDF · cuts: ledger shape, scans via MinerU, recordings, descriptions, video |
+| 0040 | [A chunk says where its words came from](0040-a-chunk-says-where-its-words-came-from.md) | Cuts 1–3 implemented (the ledger shape, and a file that needs a reader degrades with an alert instead of becoming garbage text; scans and images read by a workspace's MinerU service, one segment per page, waiting on the service without spending retries; recordings read by a diarizing transcription model, speakers in the text and times in the anchor, an unlabelled transcript degrades) · scans and images through a MinerU service, recordings only with speaker labels (revised 2026-09-15) · a chunk carries an **origin** (`stated`, `ocr`, `transcribed`, `described`) and the model that produced it, and an **anchor** back into the original bytes (a page and region, a time range, an image inside the file) — decided before any media reader, because a transcript stored without its times can never be tied to the recording again · the packer never mixes origins in a chunk · facts from a description enter below `AUTO_CLOSE_MIN_CONFIDENCE`, so a misread chart opens a conflict instead of closing a correct fact · per-modality model settings, media reading as a resumable job, origin and anchor in the API, MCP and RDF · cuts: ledger shape, scans via MinerU, recordings, descriptions, video |
| 0041 | [A name is a claim about an entity](0041-a-name-is-a-claim-about-an-entity.md) | Cut 0 built (identity bench) · cut 1 implemented (#670): names are value facts on `known_as`, the extractor reports other names, a shared name goes to the adjudicator; forward/reverse F1 0.43/0.54 → 0.68/0.68 · cuts 2–4 (name vectors and neighbours, evidence decides, re-evaluation) not started |
| 0042 | [The chat loop is a runner with hooks](0042-the-chat-loop-is-a-runner-with-hooks.md) | Implemented (#548) · the loop is rig's runner and every policy is a hook with a typed result · the wire stays `LlmClient` behind `RigModel` · a turn cannot end before a tool has run, `no_evidence_needed` is the exit for questions not about the base · RAG fallback only on a 400/422 to the first request with tools · an empty reply is asked again once · the skip rate is the model's (DeepSeek-V3 1–3 of 12, Qwen2.5-72B 0) and recorded, not prevented |
diff --git a/migrations/0060_a_recording_is_read_with_its_speakers.sql b/migrations/0060_a_recording_is_read_with_its_speakers.sql
new file mode 100644
index 00000000..e26e1ba4
--- /dev/null
+++ b/migrations/0060_a_recording_is_read_with_its_speakers.sql
@@ -0,0 +1,9 @@
+-- 录音交给会标说话人的转写模型读(0040 第三刀)。
+--
+-- 走 OpenAI 的 `/audio/transcriptions` 形状、要 `diarized_json`:每一句带说话人和起止时刻。
+-- 按工作区配、跟对话分开,理由与读扫描件的服务相同:一段董事会录音比一段文字敏感。
+-- 交回的分段没有说话人就当没配——不读、告警(决定 5)。地址或模型为空 = 关掉。
+ALTER TABLE llm_settings
+ ADD COLUMN transcribe_base_url TEXT,
+ ADD COLUMN transcribe_api_key TEXT,
+ ADD COLUMN transcribe_model TEXT;