Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
feat(observation): compact storage and add 20 GB capacity retention (#172)
澄潭38d04ccea22b
diff --git a/Cargo.lock b/Cargo.lockindex a46c4544..3013dd18 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2576,7 +2576,10 @@ dependencies = [name = "hiroute-observation"version = "0.1.0"dependencies = [+ "fs2",+ "getrandom 0.3.4","hiroute-domain",+ "hiroute-platform","hmac","parking_lot","rusqlite",@@ -2586,6 +2589,7 @@ dependencies = ["tempfile","thiserror 2.0.20","zeroize",+ "zstd",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 79abeb9a..c360e3b7 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -47,7 +47,8 @@ http = "1.3.1"nix = { version = "0.31.3", features = ["fs", "socket", "user"] }parking_lot = "0.12.4"semver = "1.0.28"-rusqlite = { version = "0.32.1", features = ["backup", "bundled"] }+rusqlite = { version = "0.32.1", features = ["backup", "bundled", "hooks"] }+zstd = "0.13.3"rustix = { version = "1.1.4", features = ["process"] }serde = { version = "1.0.219", features = ["derive", "rc"] }serde_json = "1.0.143"diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rsindex dae7e1ae..13243e02 100644--- a/apps/desktop/src-tauri/build.rs+++ b/apps/desktop/src-tauri/build.rs@@ -20,6 +20,8 @@ fn main() {"desktop_snapshot","observation_read","observation_delete",+ "observation_capacity_get",+ "observation_capacity_set","test_classifier_decision","decision_services","save_decision_service",diff --git a/apps/desktop/src-tauri/capabilities/main.json b/apps/desktop/src-tauri/capabilities/main.jsonindex 9f5a3d1b..a1bc42c3 100644--- a/apps/desktop/src-tauri/capabilities/main.json+++ b/apps/desktop/src-tauri/capabilities/main.json@@ -48,6 +48,8 @@"allow-plan-editor-options","allow-observation-read","allow-observation-delete",+ "allow-observation-capacity-get",+ "allow-observation-capacity-set","allow-test-classifier-decision","allow-decision-services","allow-save-decision-service",diff --git a/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_get.toml b/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_get.tomlnew file mode 100644index 00000000..73559e66--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_get.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-observation-capacity-get"+description = "Enables the observation_capacity_get command without any pre-configured scope."+commands.allow = ["observation_capacity_get"]++[[permission]]+identifier = "deny-observation-capacity-get"+description = "Denies the observation_capacity_get command without any pre-configured scope."+commands.deny = ["observation_capacity_get"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_set.toml b/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_set.tomlnew file mode 100644index 00000000..d945bf72--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/observation_capacity_set.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-observation-capacity-set"+description = "Enables the observation_capacity_set command without any pre-configured scope."+commands.allow = ["observation_capacity_set"]++[[permission]]+identifier = "deny-observation-capacity-set"+description = "Denies the observation_capacity_set command without any pre-configured scope."+commands.deny = ["observation_capacity_set"]diff --git a/apps/desktop/src-tauri/src/bootstrap/startup_progress.rs b/apps/desktop/src-tauri/src/bootstrap/startup_progress.rsindex 630ad497..22208bad 100644--- a/apps/desktop/src-tauri/src/bootstrap/startup_progress.rs+++ b/apps/desktop/src-tauri/src/bootstrap/startup_progress.rs@@ -8,7 +8,9 @@ struct Progress {schema: String,phase: StorageUpgradePhase,}-const UPGRADE_TIMEOUT: Duration = Duration::from_secs(15 * 60);+// Complete source publication and verified multi-GB conversion are I/O work.+// Keep one finite, cancellable overall budget; phase frames never renew it.+const UPGRADE_TIMEOUT: Duration = Duration::from_secs(2 * 60 * 60);pub(super) fn read_ready(reader: &mut impl Read,diff --git a/apps/desktop/src-tauri/src/bridge.rs b/apps/desktop/src-tauri/src/bridge.rsindex d6be7350..ede58e41 100644--- a/apps/desktop/src-tauri/src/bridge.rs+++ b/apps/desktop/src-tauri/src/bridge.rs@@ -768,6 +768,8 @@ pub fn run() {.invoke_handler(tauri::generate_handler![observation_read,observation_delete,+ observation_capacity_get,+ observation_capacity_set,test_classifier_decision,decision_services,save_decision_service,diff --git a/apps/desktop/src-tauri/src/bridge/observation.rs b/apps/desktop/src-tauri/src/bridge/observation.rsindex 0b00a78d..545cc351 100644--- a/apps/desktop/src-tauri/src/bridge/observation.rs+++ b/apps/desktop/src-tauri/src/bridge/observation.rs@@ -31,3 +31,69 @@ pub(super) async fn observation_delete();hiroute_product_session::commands::observation_delete(&context, &product_state, input).await}++#[tauri::command]+pub async fn observation_capacity_get(+ window: WebviewWindow,+ state: State<'_, DesktopState>,+) -> Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure> {+ main_window(&window)?;+ let broker = window.state::<WebConfirmationState>();+ let active = || window.is_visible().unwrap_or(false);+ let notify = |request: &WebConfirmationRequest| {+ window+ .emit_to(+ tauri::EventTarget::webview(window.label()),+ "hiroute-web-confirmation",+ request,+ )+ .map_err(|_| "CONFIRMATION_UNAVAILABLE".to_owned())+ };+ let context = hiroute_product_session::commands::ClientContext {+ id: window.label(),+ confirmations: &broker,+ active: &active,+ notify: Some(¬ify),+ };+ let product_state = hiroute_product_session::commands::CommandState(+ &state.0,+ &state.1.error,+ &state.2,+ &state.3,+ );+ hiroute_product_session::commands::observation_capacity_get(&context, &product_state).await+}++#[tauri::command]+pub async fn observation_capacity_set(+ window: WebviewWindow,+ state: State<'_, DesktopState>,+ input: hiroute_application_api::ObservationCapacityUpdateV1,+) -> Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure> {+ main_window(&window)?;+ let broker = window.state::<WebConfirmationState>();+ let active = || window.is_visible().unwrap_or(false);+ let notify = |request: &WebConfirmationRequest| {+ window+ .emit_to(+ tauri::EventTarget::webview(window.label()),+ "hiroute-web-confirmation",+ request,+ )+ .map_err(|_| "CONFIRMATION_UNAVAILABLE".to_owned())+ };+ let context = hiroute_product_session::commands::ClientContext {+ id: window.label(),+ confirmations: &broker,+ active: &active,+ notify: Some(¬ify),+ };+ let product_state = hiroute_product_session::commands::CommandState(+ &state.0,+ &state.1.error,+ &state.2,+ &state.3,+ );+ hiroute_product_session::commands::observation_capacity_set(&context, &product_state, input)+ .await+}diff --git a/apps/desktop/src-tauri/src/bridge/startup.rs b/apps/desktop/src-tauri/src/bridge/startup.rsindex dc5d3ea4..8a93e83d 100644--- a/apps/desktop/src-tauri/src/bridge/startup.rs+++ b/apps/desktop/src-tauri/src/bridge/startup.rs@@ -65,10 +65,62 @@ pub struct StartupReport {}fn complete_upgrade_backup(root: &std::path::Path) -> Option<PathBuf> {- // The storage startup boundary publishes the source outside storage, under the host's- // existing private data root. No path from JSON or from the WebView is used for opening.- let backup = root.join("storage.upgrade-backups/migration-set/source");+ // Select only known backup names and a portable owner-written batch identity.+ // The WebView never supplies a filesystem path or restoration authority.+ let parent = root.join("storage.upgrade-backups");+ if let Ok(storage) =+ hiroute_diagnostics::files::PrivateDir::open_existing(&root.join("storage"))+ && let Ok(Some(mut file)) = storage.open_read("observation-source-backup.json")+ {+ if file.len() > 16 * 1024 {+ return None;+ }+ let link: serde_json::Value =+ serde_json::from_slice(&file.read_prefix(16 * 1024).ok()?).ok()?;+ file.recheck().ok()?;+ let name = link["backup_name"].as_str()?;+ let id = link["set_id"].as_str()?;+ if link["schema"] != 1+ || !matches!(name, "migration-set" | "observation-format-5")+ || id.is_empty()+ || id.len() > 128+ || id.contains("..")+ || !id+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.'))+ {+ return None;+ }+ for candidate in [+ parent.join(name),+ parent.join(format!("migration-set.{id}")),+ ] {+ if let Some(path) = published_source(&candidate, Some(id)) {+ return Some(path);+ }+ }+ return None;+ }+ published_source(&parent.join("migration-set"), None)+}++fn published_source(batch: &std::path::Path, expected_id: Option<&str>) -> Option<PathBuf> {+ let backup = batch.join("source");let directory = hiroute_diagnostics::files::PrivateDir::open_existing(&backup).ok()?;+ let batch_directory = hiroute_diagnostics::files::PrivateDir::open_existing(batch).ok()?;+ if let Some(mut file) = batch_directory.open_read("source-location.json").ok()? {+ if file.len() > 16 * 1024 {+ return None;+ }+ let hint: serde_json::Value =+ serde_json::from_slice(&file.read_prefix(16 * 1024).ok()?).ok()?;+ file.recheck().ok()?;+ return (hint["schema"] == 1+ && hint["complete"] == true+ && expected_id.is_none_or(|id| hint["batch_id"] == id))+ .then_some(backup);+ }+ // Supported old small manifests predate the bounded directory hint.let mut manifest_file = directory.open_read("source-backup.json").ok()??;if manifest_file.len() > 4 * 1024 * 1024 {return None;@@ -76,8 +128,10 @@ fn complete_upgrade_backup(root: &std::path::Path) -> Option<PathBuf> {let manifest: serde_json::Value =serde_json::from_slice(&manifest_file.read_prefix(4 * 1024 * 1024).ok()?).ok()?;manifest_file.recheck().ok()?;- (manifest["schema"] == "hiroute.upgrade-source-backup/v1" && manifest["complete"] == true)- .then_some(backup)+ (manifest["schema"] == "hiroute.upgrade-source-backup/v1"+ && manifest["complete"] == true+ && expected_id.is_none_or(|id| manifest["source_info"]["batch_id"] == id))+ .then_some(backup)}#[tauri::command]@@ -146,6 +200,45 @@ mod tests {use std::sync::atomic::Ordering;use std::time::Duration;+ #[test]+ fn observation_source_locator_handles_large_manifests_and_rejects_another_batch() {+ let temporary = tempfile::tempdir().unwrap();+ let root = temporary.path();+ let storage = root.join("storage");+ let batch = root.join("storage.upgrade-backups/observation-format-5");+ let source = batch.join("source");+ hiroute_diagnostics::files::PrivateDir::open_or_create(&storage).unwrap();+ hiroute_diagnostics::files::PrivateDir::open_or_create(&source).unwrap();+ std::fs::write(+ storage.join("observation-source-backup.json"),+ br#"{"schema":1,"backup_name":"observation-format-5","set_id":"retained-batch"}"#,+ )+ .unwrap();+ std::fs::write(+ source.join("source-backup.json"),+ vec![b' '; 5 * 1024 * 1024],+ )+ .unwrap();+ std::fs::write(+ batch.join("source-location.json"),+ br#"{"schema":1,"complete":true,"batch_id":"retained-batch"}"#,+ )+ .unwrap();+ assert_eq!(complete_upgrade_backup(root), Some(source));+ std::fs::write(+ batch.join("source-location.json"),+ br#"{"schema":1,"complete":true,"batch_id":"different-batch"}"#,+ )+ .unwrap();+ assert_eq!(complete_upgrade_backup(root), None);+ std::fs::write(+ storage.join("observation-source-backup.json"),+ br#"{"schema":1,"backup_name":"../outside","set_id":"retained-batch"}"#,+ )+ .unwrap();+ assert_eq!(complete_upgrade_backup(root), None);+ }+#[test]#[cfg(unix)]fn recovery_directory_requires_a_private_complete_published_source() {diff --git a/apps/desktop/src/features/ObservationStorageSettings.tsx b/apps/desktop/src/features/ObservationStorageSettings.tsxnew file mode 100644index 00000000..ddd6dbb1--- /dev/null+++ b/apps/desktop/src/features/ObservationStorageSettings.tsx@@ -0,0 +1,47 @@+import { useEffect, useState } from 'react';+import { invoke } from '../host/index.ts';+import { safeDiagnosticCode } from '../error-code';+import type { Language } from '../ui/preferences';++type Capacity = { max_active_bytes: number | null; active_bytes: number | null; measured_at_ms: number | null; state: 'idle' | 'pending' | 'blocked'; reason: string };+export function ObservationStorageSettings({ active, language }: { active: boolean; language: Language }) {+ const text = (zh: string, en: string) => language === 'zh' ? zh : en;+ const [value, setValue] = useState<Capacity | null>(null);+ const [draft, setDraft] = useState('');+ const [busy, setBusy] = useState(false);+ const [error, setError] = useState('');+ useEffect(() => {+ if (!active) return;+ let current = true;+ void invoke<Capacity>('observation_capacity_get').then(value => {+ if (!current) return;+ setValue(value); setDraft(value.max_active_bytes === null ? '' : String(value.max_active_bytes / 1_000_000_000));+ }).catch(error => { if (current) setError(safeDiagnosticCode(error, 'STORAGE_UNAVAILABLE')); });+ return () => { current = false; };+ }, [active]);+ const limit = draft.trim() === '' ? null : Number(draft) * 1_000_000_000;+ const valid = limit === null || (Number.isSafeInteger(limit) && limit > 0);+ async function save() {+ if (busy || !valid) return;+ setBusy(true); setError('');+ try {+ const saved = await invoke<Capacity>('observation_capacity_set', { input: { max_active_bytes: limit } });+ setValue(saved); setDraft(saved.max_active_bytes === null ? '' : String(saved.max_active_bytes / 1_000_000_000));+ } catch (error) { setError(safeDiagnosticCode(error, 'STORAGE_UNAVAILABLE')); }+ finally { setBusy(false); }+ }+ return <>+ <div className="settings-row">+ <div className="settings-copy"><strong>{text('记录容量上限', 'Record storage limit')}</strong>+ <span>{text('默认上限 20GB,记录通常保留 7 天。容量超限会分批清理最旧的已完成请求,可能不足 7 天。留空并保存可关闭容量回收。', 'Default: 20 GB. Records normally last 7 days; exceeding the limit gradually clears the oldest completed requests sooner. Clear the field and save to disable capacity retention.')}</span>+ <span>{text('按间隔回收,空间释放可能延后;封存的旧历史与恢复副本不计入此上限。', 'Reclamation is spaced and space release may lag. Archived history and recovery copies are outside this limit.')}</span>+ {value?.active_bytes !== null && value?.active_bytes !== undefined && <span>{text('最近测量', 'Last measured')}: {(value.active_bytes / 1_000_000_000).toFixed(2)} GB · {value.state === 'blocked' ? text('等待空间释放或可回收记录', 'Waiting for space release or eligible records') : value.state === 'pending' ? text('分批处理中', 'Processing gradually') : text('正常', 'Ready')}</span>}+ </div>+ <div className="worker-settings-control">+ <input className="input worker-settings-input" type="number" min="0.000000001" step="any" inputMode="decimal" aria-label={text('记录容量上限(GB)', 'Record storage limit (GB)')} placeholder={value === null ? text('读取中…', 'Loading…') : text('已关闭', 'Disabled')} value={draft} disabled={busy || value === null} onChange={event => setDraft(event.target.value)} />+ <span>GB</span><button className="btn" type="button" disabled={busy || value === null || !valid || limit === value.max_active_bytes} onClick={() => void save()}>{busy ? text('保存中…', 'Saving…') : text('保存', 'Save')}</button>+ </div>+ </div>+ {(!valid || error) && <div className="settings-inline-error" role="status">{!valid ? text('请输入正数,或留空关闭。', 'Enter a positive value, or leave empty to disable.') : text(`容量设置暂时不可用(${error})`, `Storage settings unavailable (${error})`)}</div>}+ </>;+}diff --git a/apps/desktop/src/product/SettingsPage.tsx b/apps/desktop/src/product/SettingsPage.tsxindex 38703dee..11fd0367 100644--- a/apps/desktop/src/product/SettingsPage.tsx+++ b/apps/desktop/src/product/SettingsPage.tsx@@ -1,6 +1,7 @@import { hostCapabilities } from '../host/index.ts';import { invoke } from '../host/index.ts';import { useEffect, useState } from 'react';+import { ObservationStorageSettings } from '../features/ObservationStorageSettings';import { DiagnosticsSettings } from '../features/diagnostics/DiagnosticsSettings';import { UpdatesSettings } from '../features/UpdatesSettings';import { ShellHostSettings } from '../features/ShellHostSettings';@@ -190,7 +191,7 @@ export function SettingsPage(props: SettingsPageProps) {<section className="settings-group"><h2>{text('本地记录', 'Local records')}</h2><div className="settings-list"><div className="settings-row"><div className="settings-copy"><strong>{text('会话内容', 'Session content')}</strong><span>{text('在会话详情中,可以清理该会话的正文或记录。', 'Clear content or records from each session’s detail view.')}</span></div><button className="btn" type="button" onClick={() => props.onOpenSessions()}>{text('查看会话', 'View sessions')}</button>- </div></div></section>+ </div><ObservationStorageSettings active={props.active} language={props.language} /></div></section><DiagnosticsSettings active={props.active} language={props.language} />{hostCapabilities().updates && <UpdatesSettings active={props.active} language={props.language} onServiceChanged={props.onServiceChanged} />}{hostCapabilities().cli_entry && <section className="settings-group"><h2>CLI</h2><div className="settings-list"><div className="settings-row">diff --git a/apps/desktop/tests/v3/browser/presentation-interactions.mjs b/apps/desktop/tests/v3/browser/presentation-interactions.mjsindex 047ca0dd..c95973da 100644--- a/apps/desktop/tests/v3/browser/presentation-interactions.mjs+++ b/apps/desktop/tests/v3/browser/presentation-interactions.mjs@@ -37,6 +37,37 @@ function styledButtons(label) {}export const presentationScenarios = [+ scenario('desktop.settings.storage-capacity', ['presentation', 'storage'], 'Capacity save, disable and failure preserve the effective policy and usable local-record controls', async () => {+ for (const language of ['zh', 'en']) for (const theme of ['light', 'dark']) {+ await ready(language, theme);+ let effective = { max_active_bytes: 20_000_000_000, active_bytes: 18_000_000_000, measured_at_ms: 1000, state: 'idle', reason: 'within_limit' };+ control().handlers.observation_capacity_get = () => structuredClone(effective);+ control().handlers.observation_capacity_set = payload => { effective = { ...effective, ...payload.input }; return structuredClone(effective); };+ await navigate(language === 'zh' ? '设置' : 'Settings');+ const field = () => document.querySelector('input[aria-label="' + (language === 'zh' ? '记录容量上限(GB)' : 'Record storage limit (GB)') + '"]');+ await until(() => field() && !field().disabled, 'Capacity read did not finish');+ const set = value => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(field(), value); field().dispatchEvent(new Event('input', { bubbles: true })); };+ const save = () => field().closest('.settings-row').querySelector('button');+ assert(field().value === '20' && save().disabled, 'Default capacity does not show 20 GB');+ set('25'); await pause(); save().click();+ await until(() => effective.max_active_bytes === 25_000_000_000 && !field().disabled, 'Custom capacity did not save as decimal bytes');+ assert(control().commands.some(c => c.command === 'observation_capacity_set' && c.payload.input.max_active_bytes === 25_000_000_000), 'Save bypassed the shared capacity boundary');+ set('0'); await pause(); assert(save().disabled, 'Zero capacity can be saved');+ set('21'); await pause(); control().handlers.observation_capacity_set = () => { throw new Error('STORAGE_UNAVAILABLE'); }; save().click();+ await until(() => !field().disabled && document.querySelector('.settings-inline-error')?.textContent.includes('STORAGE_UNAVAILABLE'), 'Failed save did not expose retry');+ assert(effective.max_active_bytes === 25_000_000_000 && field().value === '21', 'Failed save changed the effective policy or discarded the draft');+ control().handlers.observation_capacity_set = payload => { effective = { ...effective, ...payload.input }; return structuredClone(effective); };+ set(''); await pause(); save().click();+ await until(() => effective.max_active_bytes === null && !field().disabled, 'Empty capacity did not disable early expiry');+ control().remount(); await pause();+ await navigate(language === 'zh' ? '设置' : 'Settings');+ await until(() => field() && !field().disabled && field().value === '', 'Disabled policy was not retained when reopening settings');+ field().scrollIntoView({ block: 'center' }); await pause();+ const row = field().closest('.settings-row');+ assert(row.scrollWidth <= row.clientWidth + 1, 'Capacity row overflows');+ bounded(field(), 'Capacity input'); bounded(save(), 'Capacity save');+ }+ }),scenario('desktop.models.view-switch', ['model-connections', 'presentation'], 'Model/connection grouping has visible selection, usable controls and stable selected models', async () => {for (const language of ['zh', 'en']) for (const theme of ['light', 'dark']) {await ready(language, theme); await navigate(language === 'zh' ? '模型' : 'Models');diff --git a/apps/desktop/tests/v3/browser/product-fixtures.ts b/apps/desktop/tests/v3/browser/product-fixtures.tsindex f77d011b..3a61312a 100644--- a/apps/desktop/tests/v3/browser/product-fixtures.ts+++ b/apps/desktop/tests/v3/browser/product-fixtures.ts@@ -616,6 +616,8 @@ export function mockProductInvoke(command: string, payload: Record<string, any>};}if (command === 'worker_dependencies_select_cancel') { preparedWorkerSelection = null; return undefined; }+ if (command === 'observation_capacity_get') return { max_active_bytes: 20_000_000_000, active_bytes: null, measured_at_ms: null, state: 'pending', reason: 'measuring' };+ if (command === 'observation_capacity_set') return { max_active_bytes: payload?.input?.max_active_bytes ?? null, active_bytes: null, measured_at_ms: null, state: 'pending', reason: 'measuring' };if (command === 'worker_settings_get') {return { schema: 'hiroute.worker-settings/v1', max_concurrent: 10 };}diff --git a/apps/desktop/tests/v3/browser/product-shell.tsx b/apps/desktop/tests/v3/browser/product-shell.tsxindex c3c58f82..6f1bd8dd 100644--- a/apps/desktop/tests/v3/browser/product-shell.tsx+++ b/apps/desktop/tests/v3/browser/product-shell.tsx@@ -54,6 +54,7 @@ mockIPC((command, payload) => {const args = (payload ?? {}) as Record<string, unknown>;control.commands.push({ command, payload: args });if (control.handlers[command]) return control.handlers[command](args);+ if (command === 'plugin:window|set_theme') return undefined;if (command === 'startup_status') return { state: 'ready', recovery_available: false };if (command === 'desktop_snapshot') return structuredClone(control.desktop);if (command === 'agent_snapshot') return structuredClone(control.agents);diff --git a/apps/desktop/tests/v3/browser/scenario-requirements.mjs b/apps/desktop/tests/v3/browser/scenario-requirements.mjsindex 6a988905..1a5df5b8 100644--- a/apps/desktop/tests/v3/browser/scenario-requirements.mjs+++ b/apps/desktop/tests/v3/browser/scenario-requirements.mjs@@ -59,6 +59,7 @@ export const focusedRequirements = {};export const productShellRequired = [+ 'desktop.settings.storage-capacity','desktop.routing.native-default','desktop.decisions.connectivity-test','desktop.routing.custom-assessment',diff --git a/apps/desktop/tests/v3/visual/agent-trust-active.mjs b/apps/desktop/tests/v3/visual/agent-trust-active.mjsindex 2f99b183..a15e47fd 100644--- a/apps/desktop/tests/v3/visual/agent-trust-active.mjs+++ b/apps/desktop/tests/v3/visual/agent-trust-active.mjs@@ -134,7 +134,7 @@ try {const results = [];for (const { id } of selected) {const viewport = ['desktop.models.subscription-scan.recommended-login',- 'desktop.models.view-switch', 'desktop.presentation.control-layout',+ 'desktop.models.view-switch', 'desktop.presentation.control-layout', 'desktop.settings.storage-capacity','desktop.routing.additional-budget-consumers','desktop.models.subscription-login.callback-modes'].includes(id)? { width: 755, height: 600 } : { width: 1280, height: 900 };@@ -145,6 +145,9 @@ try {}assertScenarioResults(selected, results);checks.push(...results);+ if (selected.some(s => s.id === 'desktop.settings.storage-capacity')) {+ checks.push(...await (await import('./storage-capacity-captures.mjs')).runStorageCapacityCaptures(client, outputRoot));+ }}if (checks.length === 0) throw new Error('No scenarios executed');const report = {diff --git a/apps/desktop/tests/v3/visual/storage-capacity-captures.mjs b/apps/desktop/tests/v3/visual/storage-capacity-captures.mjsnew file mode 100644index 00000000..3e319d22--- /dev/null+++ b/apps/desktop/tests/v3/visual/storage-capacity-captures.mjs@@ -0,0 +1,23 @@+import { writeFile } from 'node:fs/promises';+import path from 'node:path';+import { evaluate, waitFor } from './cdp-client.mjs';+export async function runStorageCapacityCaptures(client, output) {+ const checks = [];+ for (const language of ['zh', 'en']) for (const theme of ['light', 'dark']) for (const [width, height] of [[1200, 813], [755, 600]]) {+ await client.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false });+ await evaluate(client, `window.productShell.reset(); localStorage.setItem('hiroute.language',${JSON.stringify(language)}); localStorage.setItem('hiroute.theme',${JSON.stringify(theme)}); window.productShell.remount()`);+ const label = language === 'zh' ? '设置' : 'Settings';+ await waitFor(client, `Boolean(document.querySelector('button[aria-label="${label}"]'))`);+ await evaluate(client, `window.productShell.handlers.observation_capacity_get=()=>({max_active_bytes:20_000_000_000,active_bytes:21_000_000_000,measured_at_ms:1000,state:'pending',reason:'physical_release'});document.querySelector('button[aria-label="${label}"]').click()`);+ const field = `document.querySelector('input[aria-label="${language === 'zh' ? '记录容量上限(GB)' : 'Record storage limit (GB)'}"]')`;+ await waitFor(client, `Boolean(${field} && !${field}.disabled && ${field}.value==='20')`);+ await evaluate(client, `${field}.scrollIntoView({block:'center'})`);+ const geometry = await evaluate(client, `(()=>{const field=${field},row=field.closest('.settings-row'),copy=row.querySelector('.settings-copy'),button=row.querySelector('button');return{overflow:row.scrollWidth>row.clientWidth+1,copy:copy.getBoundingClientRect().width,controls:[field,button].map(e=>{const b=e.getBoundingClientRect();return{left:b.left,right:b.right,top:b.top,bottom:b.bottom}})}})()`);+ if (geometry.overflow || geometry.copy < 160 || geometry.controls.some(b => b.left < 0 || b.right > width + 1 || b.top < 0 || b.bottom > height + 1)) throw new Error('Storage settings geometry failed: ' + JSON.stringify(geometry));+ const screenshot = `storage-capacity-${language}-${theme}-${width}x${height}.png`;+ const shot = await client.send('Page.captureScreenshot', { format: 'png' });+ await writeFile(path.join(output, screenshot), Buffer.from(shot.data, 'base64'));+ checks.push({ id: `desktop.settings.storage-capacity.${language}.${theme}.${width}`, state: 'green', screenshot, geometry });+ }+ return checks;+}diff --git a/contracts/cli/contract-set.v1.json b/contracts/cli/contract-set.v1.jsonindex 6782fc22..804e0854 100644--- a/contracts/cli/contract-set.v1.json+++ b/contracts/cli/contract-set.v1.json@@ -1,5 +1,5 @@{- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","files": {"agent-check-request.v1.schema.json": "sha256:fbd9e9b19282a5ed8c3e30b5ae16538bc01318a70e4bb7c7a55e8e0472553feb","agent-launch-descriptor-request.v1.schema.json": "sha256:28f2f9d2d7f9a1618bea0f13383002429bc9e0ff56166cbee86958a245d0fd48",@@ -10,7 +10,7 @@"console-launch-request.v1.schema.json": "sha256:bb201709ef64e99036a97d5879a32a8e4e3a9517010e1041def223e723e7f385","console-token-create.v1.schema.json": "sha256:10cc9b80ed0bec4a5ca80e098cf0ee94710eedd7dfce292cf4a8770a11be78ec","console-token-revoke.v1.schema.json": "sha256:fbec29753f6effd3c854bff57984b3c3932ac5b6f2345e52151e353c15933b36",- "coverage-manifest.v1.json": "sha256:1660122311b4831139e8aeae8d79e16fa27e232b24c24f65a4a6d3df573ce73f",+ "coverage-manifest.v1.json": "sha256:9a8816c88dc42142af44f4d4db54dde52e4259da330d3daadff14d14a7659961","delegation-cancel-request.v1.schema.json": "sha256:0822c53fc16acfcd618fbef66410939f33f634313389a6df4705f414a468b878","delegation-continue-request.v1.schema.json": "sha256:da2f81607f37d154a0fa698a107b37d4b0e9d5a03693dcbcc41e0b7e65db5b93","delegation-get-request.v1.schema.json": "sha256:85b29839c1c508018ebe40e4b2bb5c1dce7c3a87c27efa30b196b2d1b6658471",@@ -24,14 +24,14 @@"local-control-hello.v2.schema.json": "sha256:32fbce882152a886b79bd139d3b4e459b48d94e0634e56ccb189b2c9971ad772","local-control.v2.schema.json": "sha256:4b654de90cd6e333b3953379e5b2e1c5125f1dcbe307b7c7c73f12bb2032d18c","machine-envelope.v2.schema.json": "sha256:23e019a3d024ae302a44105caff4ec95d3fb409a29f936dea1360e543ae6abd6",- "manifest.v1.json": "sha256:64af54a17623c0a84bc43d3ce366ffcb923ee3c8b47777a271fc16414ff15059",+ "manifest.v1.json": "sha256:5569945e5ed7ce3ad327dc7937215e626dd1da276b6083e46dcc6c556c0f9178","operation-cancel-request.v1.schema.json": "sha256:0071cbcf27de3bfb7f6644478fe2b73bb5387ebe33db93629ff5672959e8495f","operation-lookup.v1.schema.json": "sha256:0fbaaa15b0e75b4186979bdd856bed0485d55a4bd33a33f3caf3bd92e01a7850","plan-content-change.v2.schema.json": "sha256:b67a57ca076002e341c7a74a1b8d505819ed20ee6a028354c8e6d2c5ff475f47","plan-draft-change.v1.schema.json": "sha256:cbf6f1ff62fbf04e659c69342a509bbf2b2acc21b3426c3a8e4e138e26cf58c3","plan-editor.v2.schema.json": "sha256:85084c42de1e8d7be3b84401a47ec5e15406431dfad983cd3b48718e93533fe1","plan-lifecycle-change.v1.schema.json": "sha256:23700c0ea1fbc9834e653c4edc8268e346912270c92be4904f9759426f641562",- "planned-manifest.v1.json": "sha256:596204593333a199466a8bb6dcc4c76627dfc737231e2a2f5c3ace9c80a8b525",+ "planned-manifest.v1.json": "sha256:67db007bc5d0519eb5a7d8ebf1e21386c86b8d2243d82262b7793038bfd1a14f","prepare-discovered-model-connection-request.v1.schema.json": "sha256:4451d8d7d06160f399d683bc3987ac2bd458af09994e4e0d989687667f6af1bd","publication-checkpoint-change.v1.schema.json": "sha256:b130577cdae0f1e5dc9f736b8b6f40f22555b4750b4604fdc76c4b65d5b96aa3","session-list-query.v1.schema.json": "sha256:1f71a89fc019b59c3499708c0760b694b8cb8532028b56d98702cd3b2721cd44",diff --git a/contracts/cli/coverage-manifest.v1.json b/contracts/cli/coverage-manifest.v1.jsonindex cacb233a..bb252e80 100644--- a/contracts/cli/coverage-manifest.v1.json+++ b/contracts/cli/coverage-manifest.v1.json@@ -1274,6 +1274,34 @@"state": "executable"}},+ {+ "command_id": "observation.storage.show",+ "lifecycle": "released",+ "negative": {+ "polarity": "negative",+ "scenario_id": "p0.observation.storage.show.negative",+ "state": "executable"+ },+ "positive": {+ "polarity": "positive",+ "scenario_id": "p0.observation.storage.show.positive",+ "state": "executable"+ }+ },+ {+ "command_id": "observation.storage.set",+ "lifecycle": "released",+ "negative": {+ "polarity": "negative",+ "scenario_id": "p0.observation.storage.set.negative",+ "state": "executable"+ },+ "positive": {+ "polarity": "positive",+ "scenario_id": "p0.observation.storage.set.positive",+ "state": "executable"+ }+ },{"command_id": "observation.plan-quality.samples","lifecycle": "released",@@ -1289,7 +1317,7 @@}}],- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","schema_version": {"major": 1,"minor": 0diff --git a/contracts/cli/manifest.v1.json b/contracts/cli/manifest.v1.jsonindex 1d0337e8..daec3cb1 100644--- a/contracts/cli/manifest.v1.json+++ b/contracts/cli/manifest.v1.json@@ -5,7 +5,7 @@},"product_contract_revision": "487a807c8dc25cfb24e8b097882cd3bb9716a009","proposal_map_revision": "ab4da93cc49c6ec56a01e046230411ff1ca82300",- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","visibility": "release","commands": [{@@ -2824,6 +2824,104 @@"state": "executable"}},+ {+ "command_id": "observation.storage.show",+ "path": [+ "observation",+ "storage",+ "show"+ ],+ "operation_id": "GetObservationCapacity",+ "trace_ids": [+ "CLI-059",+ "APP-059"+ ],+ "scope": "p0",+ "lifecycle": "released",+ "kind": "query",+ "request_schema": "hiroute.empty-request/v1",+ "response_schema": "hiroute.command-result/v1",+ "effect": "none",+ "network": "local_control_only",+ "model_call": "never",+ "stdin_channels": [],+ "idempotency": "not_applicable",+ "help": {+ "purpose": "Read the local observation capacity policy and measured reclamation status.",+ "usage": "hiroute observation storage show [typed options] --non-interactive --output json",+ "arguments_and_options": "Only the typed options declared by the generated request schema are accepted; arbitrary patches, URLs, shell, or plaintext Secret argv are forbidden.",+ "effects": "Read-only; no product side effect.",+ "network_and_model_use": "No model use; any external dependency is declared by the typed Application operation.",+ "automation": "Use stable command_id, status, typed data, warnings[].code, and next_actions[].command_id; never parse display text.",+ "output": "One hiroute.machine-envelope/v2 JSON object on stdout; diagnostics belong on stderr.",+ "exit_status": "0 success/accepted; 2 usage; 3 conflict/blocker; 4 capability denied; 5 not found; 6 unavailable; 7 action required; 8 needs attention.",+ "examples": [+ "hiroute observation storage show --non-interactive --output json"+ ],+ "see_also": [+ "hiroute schema show --command-id observation.storage.show"+ ]+ },+ "positive": {+ "scenario_id": "p0.observation.storage.show.positive",+ "polarity": "positive",+ "state": "executable"+ },+ "negative": {+ "scenario_id": "p0.observation.storage.show.negative",+ "polarity": "negative",+ "state": "executable"+ }+ },+ {+ "command_id": "observation.storage.set",+ "path": [+ "observation",+ "storage",+ "set"+ ],+ "operation_id": "SetObservationCapacity",+ "trace_ids": [+ "CLI-060",+ "APP-060"+ ],+ "scope": "p0",+ "lifecycle": "released",+ "kind": "action",+ "request_schema": "hiroute.empty-request/v1",+ "response_schema": "hiroute.command-result/v1",+ "effect": "typed_action",+ "network": "local_control_only",+ "model_call": "never",+ "stdin_channels": [],+ "idempotency": "not_applicable",+ "help": {+ "purpose": "Save a positive byte threshold or disable early observation expiry.",+ "usage": "hiroute observation storage set [typed options] --non-interactive --output json",+ "arguments_and_options": "Only the typed options declared by the generated request schema are accepted; arbitrary patches, URLs, shell, or plaintext Secret argv are forbidden.",+ "effects": "Application-owned typed operation; side effects are recorded by the Operation and external ledgers.",+ "network_and_model_use": "No model inference; external access, when required, is limited to registered typed ports.",+ "automation": "Use stable command_id, status, typed data, warnings[].code, and next_actions[].command_id; never parse display text.",+ "output": "One hiroute.machine-envelope/v2 JSON object on stdout; diagnostics belong on stderr.",+ "exit_status": "0 success/accepted; 2 usage; 3 conflict/blocker; 4 capability denied; 5 not found; 6 unavailable; 7 action required; 8 needs attention.",+ "examples": [+ "hiroute observation storage set --non-interactive --output json"+ ],+ "see_also": [+ "hiroute schema show --command-id observation.storage.set"+ ]+ },+ "positive": {+ "scenario_id": "p0.observation.storage.set.positive",+ "polarity": "positive",+ "state": "executable"+ },+ "negative": {+ "scenario_id": "p0.observation.storage.set.negative",+ "polarity": "negative",+ "state": "executable"+ }+ },{"command_id": "observation.plan-quality.samples","path": [diff --git a/contracts/cli/planned-manifest.v1.json b/contracts/cli/planned-manifest.v1.jsonindex cbd3223e..b6dee614 100644--- a/contracts/cli/planned-manifest.v1.json+++ b/contracts/cli/planned-manifest.v1.json@@ -5,7 +5,7 @@},"product_contract_revision": "487a807c8dc25cfb24e8b097882cd3bb9716a009","proposal_map_revision": "ab4da93cc49c6ec56a01e046230411ff1ca82300",- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","visibility": "planned","commands": [{@@ -4420,6 +4420,104 @@"state": "executable"}},+ {+ "command_id": "observation.storage.show",+ "path": [+ "observation",+ "storage",+ "show"+ ],+ "operation_id": "GetObservationCapacity",+ "trace_ids": [+ "CLI-059",+ "APP-059"+ ],+ "scope": "p0",+ "lifecycle": "released",+ "kind": "query",+ "request_schema": "hiroute.empty-request/v1",+ "response_schema": "hiroute.command-result/v1",+ "effect": "none",+ "network": "local_control_only",+ "model_call": "never",+ "stdin_channels": [],+ "idempotency": "not_applicable",+ "help": {+ "purpose": "Read the local observation capacity policy and measured reclamation status.",+ "usage": "hiroute observation storage show [typed options] --non-interactive --output json",+ "arguments_and_options": "Only the typed options declared by the generated request schema are accepted; arbitrary patches, URLs, shell, or plaintext Secret argv are forbidden.",+ "effects": "Read-only; no product side effect.",+ "network_and_model_use": "No model use; any external dependency is declared by the typed Application operation.",+ "automation": "Use stable command_id, status, typed data, warnings[].code, and next_actions[].command_id; never parse display text.",+ "output": "One hiroute.machine-envelope/v2 JSON object on stdout; diagnostics belong on stderr.",+ "exit_status": "0 success/accepted; 2 usage; 3 conflict/blocker; 4 capability denied; 5 not found; 6 unavailable; 7 action required; 8 needs attention.",+ "examples": [+ "hiroute observation storage show --non-interactive --output json"+ ],+ "see_also": [+ "hiroute schema show --command-id observation.storage.show"+ ]+ },+ "positive": {+ "scenario_id": "p0.observation.storage.show.positive",+ "polarity": "positive",+ "state": "executable"+ },+ "negative": {+ "scenario_id": "p0.observation.storage.show.negative",+ "polarity": "negative",+ "state": "executable"+ }+ },+ {+ "command_id": "observation.storage.set",+ "path": [+ "observation",+ "storage",+ "set"+ ],+ "operation_id": "SetObservationCapacity",+ "trace_ids": [+ "CLI-060",+ "APP-060"+ ],+ "scope": "p0",+ "lifecycle": "released",+ "kind": "action",+ "request_schema": "hiroute.empty-request/v1",+ "response_schema": "hiroute.command-result/v1",+ "effect": "typed_action",+ "network": "local_control_only",+ "model_call": "never",+ "stdin_channels": [],+ "idempotency": "not_applicable",+ "help": {+ "purpose": "Save a positive byte threshold or disable early observation expiry.",+ "usage": "hiroute observation storage set [typed options] --non-interactive --output json",+ "arguments_and_options": "Only the typed options declared by the generated request schema are accepted; arbitrary patches, URLs, shell, or plaintext Secret argv are forbidden.",+ "effects": "Application-owned typed operation; side effects are recorded by the Operation and external ledgers.",+ "network_and_model_use": "No model inference; external access, when required, is limited to registered typed ports.",+ "automation": "Use stable command_id, status, typed data, warnings[].code, and next_actions[].command_id; never parse display text.",+ "output": "One hiroute.machine-envelope/v2 JSON object on stdout; diagnostics belong on stderr.",+ "exit_status": "0 success/accepted; 2 usage; 3 conflict/blocker; 4 capability denied; 5 not found; 6 unavailable; 7 action required; 8 needs attention.",+ "examples": [+ "hiroute observation storage set --non-interactive --output json"+ ],+ "see_also": [+ "hiroute schema show --command-id observation.storage.set"+ ]+ },+ "positive": {+ "scenario_id": "p0.observation.storage.set.positive",+ "polarity": "positive",+ "state": "executable"+ },+ "negative": {+ "scenario_id": "p0.observation.storage.set.negative",+ "polarity": "negative",+ "state": "executable"+ }+ },{"command_id": "observation.plan-quality.samples","path": [diff --git a/contracts/compatibility-support.v1.json b/contracts/compatibility-support.v1.jsonindex db7527e3..84956afc 100644--- a/contracts/compatibility-support.v1.json+++ b/contracts/compatibility-support.v1.json@@ -410,6 +410,7 @@"hiroute.upgrade-installer-decision/v1","hiroute.upgrade-installer-started/v1","hiroute.upgrade-source-backup/v1",+ "hiroute.upgrade-source-backup/v2","hiroute.value-ledger-entry/v1","hiroute.value-query/v1","hiroute.value-view/v1",@@ -1189,6 +1190,17 @@"owner": "application-api/daemon/model-connections","reason": "The shared scanner retains the currently published empty-request V1 projection and one-model discovery handles. An explicit V2 request selects connection-point projection; credentials and source semantics have one private owner. Current CLI/Desktop consume the V2 shape without expanding old handles.","removal_condition": "Remove the V1 request/projection adapter only when the supported public caller policy retires the released V1 API and its one-model handles."+ },+ {+ "subject": "upgrade_source_backup",+ "audited_prefixes": [+ "hiroute.upgrade-source-backup"+ ],+ "current_contracts": [+ "hiroute.upgrade-source-backup/v1",+ "hiroute.upgrade-source-backup/v2"+ ],+ "legacy_support": []}],"forbidden_contracts": [@@ -1264,7 +1276,7 @@"field": "context_window_tokens","mode": "recovery_read_only","owner": "agent-connections",- "reason": "A descriptor reconstructed from a pre-window immutable settings operation has no managed context policy. It retains that operation’s behavior until settings are saved again.",+ "reason": "A descriptor reconstructed from a pre-window immutable settings operation has no managed context policy. It retains that operation\u2019s behavior until settings are saved again.","removal_condition": "Retire with the pre-window settings operation recovery reader.","allowed_paths": ["crates/domain/src/agents/managed_launch.rs",@@ -1809,6 +1821,24 @@"crates/domain/src/routing/authoring.rs","crates/domain/src/operation/routing_content.rs"]+ },+ {+ "contract": "hiroute.observation.query/v2",+ "field": "persisted Observation schema_version 1 through 4 and original absolute owned object locators",+ "mode": "recovery_read_only",+ "owner": "observation-storage",+ "reason": "Supported old histories are sealed unchanged by the startup coordinator with the complete original group, original keys, auxiliary state and matching old package. Startup does not traverse or convert old observations. Old absolute object locators stay unchanged and are read only through whole-group matching-version recovery at the recorded original locations. Current format 5 is the sole writer and current history/statistics start at cutover; already-current stores never reset on reopen. Preserve partial older converter generations for actionable original-group recovery.",+ "removal_condition": "Retire only through an explicit supported-release decision that preserves verified whole-group recovery for every retained supported source.",+ "allowed_paths": [+ "crates/observation/src/store/upgrade.rs",+ "crates/observation/src/store/upgrade/tests.rs",+ "crates/observation/src/store/schema.rs",+ "crates/local-storage/src/migrations/observation_source.rs",+ "crates/local-storage/src/backup/source_snapshot.rs",+ "crates/local-storage/src/backup/source_snapshot/seal.rs",+ "crates/local-storage/src/backup/source_snapshot/seal/tests.rs",+ "crates/daemon/src/control/runtime/history_cutover_tests.rs"+ ]}]}diff --git a/crates/application-api/src/command_schemas.rs b/crates/application-api/src/command_schemas.rsindex c0c2691c..c8890ec7 100644--- a/crates/application-api/src/command_schemas.rs+++ b/crates/application-api/src/command_schemas.rs@@ -169,6 +169,14 @@ pub fn command_schemas(command_id: &str) -> Option<CommandSchemasV1> {request!(ClientEmptyRequestV1),response!(hiroute_domain::ObservationStatusV1),),+ "observation.storage.show" => (+ schema::<ClientEmptyRequestV1>(false),+ schema::<hiroute_domain::ObservationCapacityStatusV1>(true),+ ),+ "observation.storage.set" => (+ schema::<ObservationCapacityUpdateV1>(false),+ schema::<hiroute_domain::ObservationCapacityStatusV1>(true),+ ),"observation.plan-quality.samples" => (observation_request(&["plan_quality"]),response!(PlanQualitySamplesPage),@@ -367,6 +375,11 @@ fn examples(command_id: &str) -> Vec<Value> {json!({"schema_version":hiroute_domain::CHANGE_SPEC_SCHEMA_V1,"spec":spec,"accept_digest":format!("sha256:{}", "0".repeat(64)),"expected_revisions":{"target":0},"idempotency_key":"replace-with-one-stable-key"}),]}+ "observation.storage.show" => vec![json!({})],+ "observation.storage.set" => vec![+ json!({"max_active_bytes":20_000_000_000u64}),+ json!({"max_active_bytes":null}),+ ],"routing.options" => vec![json!({}), json!({"authoring":{}})],"compute.scan" => vec![json!({}), json!({"schema":COMPUTE_SCAN_REQUEST_SCHEMA_V2})],"compute.connection.options"diff --git a/crates/application-api/src/commands.rs b/crates/application-api/src/commands.rsindex f16aeda1..7b6c3286 100644--- a/crates/application-api/src/commands.rs+++ b/crates/application-api/src/commands.rs@@ -1038,6 +1038,24 @@ const COMMANDS: &[StaticCommandDescriptor] = &[Query,"Read recorded token, cache and valuation totals with coverage."),+ command!(+ "observation.storage.show",+ ["observation", "storage", "show"],+ "GetObservationCapacity",+ ["CLI-059", "APP-059"],+ Released,+ Query,+ "Read the local observation capacity policy and measured reclamation status."+ ),+ command!(+ "observation.storage.set",+ ["observation", "storage", "set"],+ "SetObservationCapacity",+ ["CLI-060", "APP-060"],+ Released,+ Action,+ "Save a positive byte threshold or disable early observation expiry."+ ),command!("observation.plan-quality.samples",["observation", "plan-quality", "samples"],diff --git a/crates/application-api/src/lib.rs b/crates/application-api/src/lib.rsindex 25a657cc..966f77f5 100644--- a/crates/application-api/src/lib.rs+++ b/crates/application-api/src/lib.rs@@ -79,8 +79,9 @@ pub use hiroute_domain::{DecisionServiceChangeV1, HIDDEN_AGENT_GRANT_HELPER_VERB_V1,MANAGED_CLAUDE_SETTING_SOURCES_ARGUMENT_V1, MANAGED_CLAUDE_SETTINGS_ARGUMENT_V1,ManagedClaudeLaunchDescriptorV2, ManagedLaunchProfileV1, MaterializationState, ModelAlias,- ModelSwitchFilter, PRODUCT_CONTRACT_REVISION, PROPOSAL_MAP_REVISION, RevisionSetV1,- SchemaVersion, SessionId, SessionListQueryV1, UpstreamProtocol, ValueGroupByV1, ValueQueryV1,+ ModelSwitchFilter, ObservationCapacityStatusV1, PRODUCT_CONTRACT_REVISION,+ PROPOSAL_MAP_REVISION, RevisionSetV1, SchemaVersion, SessionId, SessionListQueryV1,+ UpstreamProtocol, ValueGroupByV1, ValueQueryV1,};pub use protocol::{AgentCheckRequestV1, AgentCheckScopeV1, AgentCheckSuiteV1, AgentModelCheckTargetV2,@@ -266,7 +267,7 @@ mod protocol_tests {#[test]fn p0_registry_and_staging_invariants_remain_closed() {let commands = planned_commands();- assert_eq!(commands.len(), 92);+ assert_eq!(commands.len(), 94);assert_eq!(commands.iter()@@ -314,7 +315,7 @@ mod protocol_tests {CommandLifecycle::Released);let release = release_manifest();- assert_eq!(release.commands.len(), 59);+ assert_eq!(release.commands.len(), 61);assert!(release.commands.iter().all(|command| {command.lifecycle == CommandLifecycle::Released&& command.positive.state == CoverageState::Executablediff --git a/crates/application-api/src/observation_v2.rs b/crates/application-api/src/observation_v2.rsindex 31909ea9..4c7462e1 100644--- a/crates/application-api/src/observation_v2.rs+++ b/crates/application-api/src/observation_v2.rs@@ -166,3 +166,59 @@ pub struct ApiEquivalentValueOptionsV1 {pub report: ObservationValueReportOptionsV2,pub reference_binding_id: String,}++/// Explicit null disables early capacity expiry; an omitted value is invalid.+#[derive(schemars::JsonSchema, Clone, Debug, serde::Serialize, serde::Deserialize)]+#[serde(deny_unknown_fields)]+pub struct ObservationCapacityUpdateV1 {+ #[serde(deserialize_with = "deserialize_capacity_limit")]+ #[schemars(required)]+ pub max_active_bytes: Option<u64>,+}+fn deserialize_capacity_limit<'de, D: serde::Deserializer<'de>>(+ d: D,+) -> Result<Option<u64>, D::Error> {+ let value = Option::<u64>::deserialize(d)?;+ if value == Some(0) {+ return Err(serde::de::Error::custom(+ "capacity must be positive or null",+ ));+ }+ Ok(value)+}++#[cfg(test)]+mod capacity_tests {+ use super::*;+ #[test]+ fn capacity_requires_explicit_positive_integer_or_null() {+ for input in [+ "{}",+ "{\"max_active_bytes\":0}",+ "{\"max_active_bytes\":-1}",+ "{\"max_active_bytes\":18446744073709551616}",+ "{\"max_active_bytes\":1.5}",+ "{\"max_active_bytes\":\"20G\"}",+ "{\"max_active_bytes\":20,\"extra\":true}",+ ] {+ assert!(+ serde_json::from_str::<ObservationCapacityUpdateV1>(input).is_err(),+ "{input}"+ );+ }+ assert_eq!(+ serde_json::from_str::<ObservationCapacityUpdateV1>("{\"max_active_bytes\":null}")+ .unwrap()+ .max_active_bytes,+ None+ );+ assert_eq!(+ serde_json::from_str::<ObservationCapacityUpdateV1>(+ "{\"max_active_bytes\":18446744073709551615}"+ )+ .unwrap()+ .max_active_bytes,+ Some(u64::MAX)+ );+ }+}diff --git a/crates/application/src/lib.rs b/crates/application/src/lib.rsindex c9849501..b6362e18 100644--- a/crates/application/src/lib.rs+++ b/crates/application/src/lib.rs@@ -199,6 +199,12 @@ impl ApplicationService {"GetRoutingReceipt" => self.observation(request, ObservationOperation::Receipt),"GetObservationStatus" => self.observation(request, ObservationOperation::Status),"GetValue" => self.observation(request, ObservationOperation::Value),+ "GetObservationCapacity" => {+ self.observation(request, ObservationOperation::CapacityRead)+ }+ "SetObservationCapacity" => {+ self.observation(request, ObservationOperation::CapacityUpdate)+ }"GetPlanQualitySamples" => self.observation(request, ObservationOperation::PlanQuality),"PreviewSessionDeletion" => {self.observation(request, ObservationOperation::DeletePreview)@@ -683,6 +689,13 @@ impl ApplicationService {.and_then(Value::as_str)== Some("analysis");let result = match operation {+ ObservationOperation::CapacityRead | ObservationOperation::CapacityUpdate => control+ .capacity(+ request.principal.kind,+ request.protected_grant.as_ref(),+ request.payload,+ matches!(operation, ObservationOperation::CapacityUpdate),+ ),ObservationOperation::List => control.list(request.principal.kind,request.protected_grant.as_ref(),@@ -745,6 +758,8 @@ impl ApplicationService {#[derive(Clone, Copy)]enum ObservationOperation {+ CapacityRead,+ CapacityUpdate,DeletePreview,DeleteApply,List,diff --git a/crates/application/src/observation_query/control.rs b/crates/application/src/observation_query/control.rsindex 1550102e..b6acf83c 100644--- a/crates/application/src/observation_query/control.rs+++ b/crates/application/src/observation_query/control.rs@@ -135,6 +135,47 @@ impl ObservationControl {serde_json::to_value(page).map_err(|_| ObservationQueryError::Corrupt)}+ pub(crate) fn capacity(+ &self,+ ambient: PrincipalKind,+ grant: Option<&ProtectedClientGrantV2>,+ payload: Value,+ update: bool,+ ) -> Result<Value, ObservationQueryError> {+ if ambient == PrincipalKind::SealedCollaboration+ || grant.is_some_and(|g| g.principal_kind.is_collaboration())+ {+ return Err(ObservationQueryError::Unauthorized);+ }+ let operation = if update {+ "SetObservationCapacity"+ } else {+ "GetObservationCapacity"+ };+ // The local transport authenticates the OS owner, but deliberately maps+ // an ambient CLI/Desktop peer to Skill rather than claiming a human+ // identity. This narrow configuration operation admits that local peer;+ // an explicitly supplied grant must still authorize this exact payload.+ let principal = self.local_or_protected_principal(grant, operation, &payload, true)?;+ super::require(+ &principal,+ hiroute_domain::ObservationCapabilityV1::ManageRetention,+ )?;+ let status = if update {+ let input: hiroute_application_api::ObservationCapacityUpdateV1 =+ serde_json::from_value(payload).map_err(|_| ObservationQueryError::InvalidQuery)?;+ self.service+ .port+ .set_observation_capacity(input.max_active_bytes)?+ } else {+ if payload != json!({}) {+ return Err(ObservationQueryError::InvalidQuery);+ }+ self.service.port.observation_capacity()?+ };+ serde_json::to_value(status).map_err(|_| ObservationQueryError::Corrupt)+ }+pub(crate) fn plan_quality(&self,grant: Option<&ProtectedClientGrantV2>,@@ -412,6 +453,7 @@ mod tests {struct FixedPorts {now_ms: AtomicI64,+ capacity_limit: Mutex<Option<u64>>,queries: Mutex<Vec<SessionListQueryV1>>,content_modes: Mutex<Vec<ContentMode>>,plans: Vec<AgentPlanId>,@@ -421,6 +463,7 @@ mod tests {fn new(now_ms: i64, plans: &[&str]) -> Self {Self {now_ms: AtomicI64::new(now_ms),+ capacity_limit: Mutex::new(None),queries: Mutex::new(Vec::new()),content_modes: Mutex::new(Vec::new()),plans: plans@@ -539,6 +582,22 @@ mod tests {}impl ObservationQueryPort for FixedPorts {+ fn observation_capacity(+ &self,+ ) -> Result<hiroute_domain::ObservationCapacityStatusV1, ObservationQueryError> {+ Ok(hiroute_domain::ObservationCapacityStatusV1 {+ max_active_bytes: *self.capacity_limit.lock().unwrap(),+ ..Default::default()+ })+ }+ fn set_observation_capacity(+ &self,+ limit: Option<u64>,+ ) -> Result<hiroute_domain::ObservationCapacityStatusV1, ObservationQueryError> {+ *self.capacity_limit.lock().unwrap() = limit;+ self.observation_capacity()+ }+fn list_sessions(&self,_workspace_id: &WorkspaceId,@@ -648,6 +707,77 @@ mod tests {ObservationControl::new(ports.clone(), ports.clone(), ports.clone(), ports)}+ #[test]+ fn capacity_is_global_owner_authority_and_failed_or_delegated_input_cannot_change_it() {+ let ports = Arc::new(FixedPorts::new(1000, &[]));+ let controller = control(ports.clone());+ assert_eq!(+ controller+ .capacity(+ hiroute_application_api::PrincipalV1::ambient_local_peer().kind,+ None,+ json!({"max_active_bytes":20_000_000_000u64}),+ true+ )+ .unwrap()["max_active_bytes"],+ 20_000_000_000u64+ );+ assert!(+ controller+ .capacity(+ PrincipalKind::SealedCollaboration,+ None,+ json!({"max_active_bytes":null}),+ true+ )+ .is_err()+ );+ let readonly = ProtectedClientGrantV2 {+ principal_kind: PrincipalKind::InteractiveUser,+ capability: "launcher-grant".into(),+ };+ assert!(+ controller+ .capacity(+ PrincipalKind::Skill,+ Some(&readonly),+ json!({"max_active_bytes":null}),+ true+ )+ .is_err()+ );+ let delegated = ProtectedClientGrantV2 {+ principal_kind: PrincipalKind::SealedCollaboration,+ capability: "launcher-grant".into(),+ };+ assert!(+ controller+ .capacity(+ PrincipalKind::SealedCollaboration,+ Some(&delegated),+ json!({"max_active_bytes":null}),+ true+ )+ .is_err()+ );+ assert!(+ controller+ .capacity(+ PrincipalKind::InteractiveUser,+ None,+ json!({"max_active_bytes":0}),+ true+ )+ .is_err()+ );+ assert_eq!(+ controller+ .capacity(PrincipalKind::InteractiveUser, None, json!({}), false)+ .unwrap()["max_active_bytes"],+ 20_000_000_000u64+ );+ }+#[test]fn receipt_and_session_content_require_distinct_operation_grants() {let ports = Arc::new(FixedPorts::new(10 * DAY_MILLIS, &[]));diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rsindex 7f832e9c..95542011 100644--- a/crates/cli/src/lib.rs+++ b/crates/cli/src/lib.rs@@ -386,6 +386,8 @@ fn command_payload(command_id: &str, options: &[String]) -> Result<Value, ErrorC"sessions.receipt" => observation::session_receipt(options),"value.show" => observation::value_show(options),"observation.plan-quality.samples" => observation::plan_quality_samples(options),+ "observation.storage.show" => observation::capacity_get(options),+ "observation.storage.set" => observation::capacity_set(options),_ => control_plane::command_payload(command_id, options).unwrap_or(Err(ErrorCode::UnknownCommand)),}@@ -886,7 +888,7 @@ mod tests {assert_eq!(execution.exit_code, 0);assert_eq!(execution.stdout.lines().count(), 1);let value: Value = serde_json::from_str(&execution.stdout).unwrap();- assert_eq!(value["data"]["commands"].as_array().unwrap().len(), 59);+ assert_eq!(value["data"]["commands"].as_array().unwrap().len(), 61);assert!(!execution.stdout.contains(HIDDEN_AGENT_GRANT_HELPER_VERB_V1));}diff --git a/crates/cli/src/observation.rs b/crates/cli/src/observation.rsindex 48642050..d4ff6fc8 100644--- a/crates/cli/src/observation.rs+++ b/crates/cli/src/observation.rs@@ -352,6 +352,40 @@ fn parse_score(options: &[String], index: &mut usize) -> Result<f64, ErrorCode>}}+pub(super) fn capacity_get(options: &[String]) -> Result<Value, ErrorCode> {+ if let Some(input) = RequestInput::optional(options)? {+ let request: hiroute_application_api::ClientEmptyRequestV1 =+ serde_json::from_value(input.read(MAX_REQUEST_BYTES)?)+ .map_err(|_| ErrorCode::InvalidArguments)?;+ return serde_json::to_value(request).map_err(|_| ErrorCode::Internal);+ }+ if options.is_empty() {+ Ok(serde_json::json!({}))+ } else {+ Err(ErrorCode::InvalidArguments)+ }+}+pub(super) fn capacity_set(options: &[String]) -> Result<Value, ErrorCode> {+ if let Some(input) = RequestInput::optional(options)? {+ let request: hiroute_application_api::ObservationCapacityUpdateV1 =+ serde_json::from_value(input.read(MAX_REQUEST_BYTES)?)+ .map_err(|_| ErrorCode::InvalidArguments)?;+ return serde_json::to_value(request).map_err(|_| ErrorCode::Internal);+ }+ let value = match options {+ [flag] if flag == "--disable" => None,+ [flag, value] if flag == "--max-active-bytes" => Some(+ value+ .parse::<u64>()+ .ok()+ .filter(|n| *n > 0)+ .ok_or(ErrorCode::InvalidArguments)?,+ ),+ _ => return Err(ErrorCode::InvalidArguments),+ };+ Ok(serde_json::json!({"max_active_bytes":value}))+}+#[cfg(test)]mod tests {use serde_json::json;diff --git a/crates/cli/tests/contracts.rs b/crates/cli/tests/contracts.rsindex 4aca24c9..9589d4b3 100644--- a/crates/cli/tests/contracts.rs+++ b/crates/cli/tests/contracts.rs@@ -16,7 +16,7 @@ fn checked_in_contracts_are_exact_generator_output() {#[test]fn each_p0_leaf_has_stable_positive_and_negative_plans() {let commands = planned_commands();- assert_eq!(commands.len(), 92); // Full staged surface including decision services.+ assert_eq!(commands.len(), 94); // Full staged surface including Observation storage policy.for command in commands {assert!(command.positive.scenario_id.ends_with(".positive"));assert!(command.negative.scenario_id.ends_with(".negative"));@@ -93,6 +93,8 @@ fn release_manifest_is_a_strict_subset_of_planned_registry() {"sessions.receipt","sessions.status","value.show",+ "observation.storage.show",+ "observation.storage.set","observation.plan-quality.samples",]);diff --git a/crates/client-core/src/observation_queries.rs b/crates/client-core/src/observation_queries.rsindex ac780d4e..d4554841 100644--- a/crates/client-core/src/observation_queries.rs+++ b/crates/client-core/src/observation_queries.rs@@ -56,3 +56,28 @@ impl Client {.await}}++impl Client {+ pub async fn observation_capacity(+ &self,+ request_id: &str,+ update: Option<ObservationCapacityUpdateV1>,+ ) -> Result<MachineEnvelopeV2<ObservationCapacityStatusV1>, ClientFailure> {+ let (operation, payload) = match update {+ Some(input) => (+ "SetObservationCapacity",+ serde_json::to_value(input)+ .map_err(|_| ClientFailure::before_send(FailureCode::FrameInvalid))?,+ ),+ None => ("GetObservationCapacity", serde_json::json!({})),+ };+ self.call_typed(LocalControlWireRequestV2 {+ schema_version: LOCAL_CONTROL_SCHEMA_V2,+ request_id: request_id.into(),+ operation_id: operation.into(),+ payload,+ protected_grant: None,+ })+ .await+ }+}diff --git a/crates/daemon/src/console/http.rs b/crates/daemon/src/console/http.rsindex b85e6e0d..bf9c17c0 100644--- a/crates/daemon/src/console/http.rs+++ b/crates/daemon/src/console/http.rs@@ -350,6 +350,7 @@ fn audited(name: &str) -> bool {| "model_reference_query"| "effective_price_query"| "plan_editor_options"+ | "observation_capacity_get"| "worker_settings_get"| "worker_executor_availability"| "worker_plans"diff --git a/crates/daemon/src/control/capacity_tests.rs b/crates/daemon/src/control/capacity_tests.rsnew file mode 100644index 00000000..3875a050--- /dev/null+++ b/crates/daemon/src/control/capacity_tests.rs@@ -0,0 +1,68 @@+//! Exercise the ambient principal actually reconstructed by Local Control.+use super::*;+use hiroute_application_api::{LOCAL_CONTROL_SCHEMA_V2, LocalControlWireRequestV2};+use serde_json::json;++#[test]+fn local_owner_capacity_roundtrip_reopens_and_does_not_accept_a_delegated_grant() {+ #[cfg(unix)]+ if crate::test_support::isolated_agent_home(+ "control::capacity_tests::local_owner_capacity_roundtrip_reopens_and_does_not_accept_a_delegated_grant",+ ) {+ return;+ }+ let dir = crate::test_support::private_tempdir();+ let open = || {+ runtime::ProductionControlRuntime::open_with_release_catalog(+ dir.path().join("storage"),+ crate::release_catalog::fixture_catalog(),+ )+ .unwrap()+ };+ let runtime = open();+ let call = |runtime: &runtime::ProductionControlRuntime, operation: &str, payload| {+ LocalControlDaemon::new(ApplicationService::new(runtime.application_ports())).dispatch_wire(+ LocalControlWireRequestV2 {+ schema_version: LOCAL_CONTROL_SCHEMA_V2,+ request_id: "capacity-owner".into(),+ operation_id: operation.into(),+ payload,+ protected_grant: None,+ },+ )+ };+ let saved = call(+ &runtime,+ "SetObservationCapacity",+ json!({"max_active_bytes":20_000_000_000u64}),+ );+ assert!(saved.error.is_none(), "{saved:?}");+ assert_eq!(saved.data.unwrap()["max_active_bytes"], 20_000_000_000u64);+ let denied = LocalControlDaemon::new(ApplicationService::new(runtime.application_ports()))+ .dispatch_wire(LocalControlWireRequestV2 {+ schema_version: LOCAL_CONTROL_SCHEMA_V2,+ request_id: "capacity-delegated".into(),+ operation_id: "SetObservationCapacity".into(),+ payload: json!({"max_active_bytes":null}),+ protected_grant: Some(hiroute_application_api::ProtectedClientGrantV2 {+ principal_kind: hiroute_application_api::PrincipalKind::Skill,+ capability: "read-only-grant".into(),+ }),+ });+ assert_eq!(+ denied.error.unwrap().code,+ hiroute_application_api::ErrorCode::CapabilityDenied+ );+ drop(runtime);+ let runtime = open();+ let read = call(&runtime, "GetObservationCapacity", json!({}));+ assert!(read.error.is_none(), "{read:?}");+ assert_eq!(read.data.unwrap()["max_active_bytes"], 20_000_000_000u64);+ let disabled = call(+ &runtime,+ "SetObservationCapacity",+ json!({"max_active_bytes":null}),+ );+ assert!(disabled.error.is_none(), "{disabled:?}");+ assert!(disabled.data.unwrap()["max_active_bytes"].is_null());+}diff --git a/crates/daemon/src/control/mod.rs b/crates/daemon/src/control/mod.rsindex 76005c67..71bc59a1 100644--- a/crates/daemon/src/control/mod.rs+++ b/crates/daemon/src/control/mod.rs@@ -26,6 +26,8 @@ use hiroute_platform::windows::pipe::{Listener as LocalListener, Stream as Local#[cfg(unix)]use std::os::unix::net::{UnixListener as LocalListener, UnixStream as LocalStream};+#[cfg(test)]+mod capacity_tests;#[cfg(any(unix, windows))]mod managed;pub(crate) mod runtime;@@ -338,6 +340,8 @@ impl LocalControlDaemon {| "GetObservationStatus"| "GetValue"| "GetPlanQualitySamples"+ | "GetObservationCapacity"+ | "SetObservationCapacity");if request.operation_id.starts_with("Worker") && request.protected_grant.is_some() {return MachineEnvelopeV2::failed(diff --git a/crates/daemon/src/control/runtime.rs b/crates/daemon/src/control/runtime.rsindex b9b8d636..e432d9d8 100644--- a/crates/daemon/src/control/runtime.rs+++ b/crates/daemon/src/control/runtime.rs@@ -335,11 +335,19 @@ impl ProductionControlRuntime {recover: bool,overrides: RuntimeOpenOverrides,) -> Result<Self, String> {- let stores = LocalStorageSet::open_for_daemon_startup_with_options(- storage_root,- &overrides.storage_startup,- )- .map_err(|error| error.to_string())?;+ observation_key::validate_existing_authority(storage_root)?;+ let mut startup = overrides.storage_startup.clone();+ startup.preserve_observation_source =+ LocalObservationStore::legacy_history_requires_seal(storage_root.join("observation"))+ .map_err(|error| format!("{error}; preserve all storage generations and use the original whole-group recovery guide"))?+ || LocalStorageSet::observation_cutover_pending(storage_root);+ if startup.preserve_observation_source+ && let Some(progress) = &startup.upgrade_progress+ {+ let _ = progress.send(hiroute_host_runtime::StorageUpgradePhase::SourceCheck);+ }+ let stores = LocalStorageSet::open_for_daemon_startup_with_options(storage_root, &startup)+ .map_err(|error| error.to_string())?;stores.control().begin_plan_version_recovery()@@ -525,12 +533,30 @@ impl ProductionControlRuntime {.chain(scoped_targets),).map_err(|error| error.to_string())?;+ if startup.preserve_observation_source+ && let Some(progress) = &startup.upgrade_progress+ {+ let _ = progress.send(hiroute_host_runtime::StorageUpgradePhase::Conversion);+ }let observation_workspace_key = observation_workspace_key(storage_root)?;let digest_authority = DigestAuthority::new(*observation_workspace_key);+ let observation_root = storage_root.join("observation");let observation = Arc::new(- LocalObservationStore::open(storage_root.join("observation"), digest_authority.clone())- .map_err(|error| error.to_string())?,+ if startup.preserve_observation_source {+ LocalObservationStore::open_fresh_after_seal(+ observation_root,+ digest_authority.clone(),+ )+ } else {+ LocalObservationStore::open(observation_root, digest_authority.clone())+ }+ .map_err(|error| error.to_string())?,);+ if startup.preserve_observation_source+ && let Some(progress) = &startup.upgrade_progress+ {+ let _ = progress.send(hiroute_host_runtime::StorageUpgradePhase::Validation);+ }let delegation_epoch = delegation_epoch()?;let plan_admission =Arc::new(hiroute_application::publication::admission::SharedAdmissionGate::new());@@ -1670,6 +1696,13 @@ fn map_port(error: PortError) -> ControlReadError {#[path = "runtime_observation_tests.rs"]mod tests;+#[cfg(test)]+mod history_cutover_tests;++#[cfg(test)]+#[path = "runtime_startup_progress_tests.rs"]+mod startup_progress_tests;+#[cfg(test)]#[path = "runtime_worker_dependencies_tests.rs"]mod worker_dependencies_tests;diff --git a/crates/daemon/src/control/runtime/history_cutover_tests.rs b/crates/daemon/src/control/runtime/history_cutover_tests.rsnew file mode 100644index 00000000..10277783--- /dev/null+++ b/crates/daemon/src/control/runtime/history_cutover_tests.rs@@ -0,0 +1,149 @@+use super::*;+use rusqlite::Connection;+use std::fs;++#[test]+fn production_startup_seals_old_history_and_preserves_the_new_generation_on_restart() {+ if crate::test_support::isolated_agent_home(+ "control::runtime::history_cutover_tests::production_startup_seals_old_history_and_preserves_the_new_generation_on_restart",+ ) {+ return;+ }+ let temporary = tempfile::tempdir().unwrap();+ let root = temporary.path().join("storage");+ let open = |options| {+ ProductionControlRuntime::prepare_for_role_all_with_codex_desktop_engine(+ &root,+ crate::release_catalog::fixture_catalog(),+ None,+ None,+ options,+ )+ };+ drop(open(Default::default()).unwrap());+ let key = fs::read(root.join("observation-hmac-key")).unwrap();+ let old_db = root.join("observation/activity.db");+ let db = Connection::open(&old_db).unwrap();+ // Synthetic admission coverage only; real released old bytes/packages are+ // verified separately by installed field acceptance.+ db.execute(+ "UPDATE observation_meta SET value='4' WHERE key='schema_version'",+ [],+ )+ .unwrap();+ db.execute(+ "INSERT INTO observation_meta VALUES('generation_probe','original-history')",+ [],+ )+ .unwrap();+ drop(db);+ fs::write(+ root.join("observation/original-object"),+ b"unmodified old contents",+ )+ .unwrap();+ let package = temporary.path().join("old-package");+ fs::create_dir(&package).unwrap();+ fs::write(package.join("old-program"), b"synthetic matching package").unwrap();+ let runtime = open(hiroute_local_storage::StorageStartupOptions {+ recovery_package: Some(package),+ ..Default::default()+ })+ .unwrap();+ let link: serde_json::Value =+ serde_json::from_slice(&fs::read(root.join("observation-source-backup.json")).unwrap())+ .unwrap();+ let source = hiroute_local_storage::upgrade_backup_root(&root)+ .unwrap()+ .parent()+ .unwrap()+ .join(link["backup_name"].as_str().unwrap())+ .join("source/storage");+ assert_eq!(+ fs::read(source.join("observation/original-object")).unwrap(),+ b"unmodified old contents"+ );+ let db = Connection::open(source.join("observation/activity.db")).unwrap();+ assert_eq!(+ db.query_row(+ "SELECT value FROM observation_meta WHERE key='generation_probe'",+ [],+ |row| row.get::<_, String>(0)+ )+ .unwrap(),+ "original-history"+ );+ drop(db);+ assert_eq!(fs::read(root.join("observation-hmac-key")).unwrap(), key);+ assert!(!root.join("observation/original-object").exists());+ assert_eq!(+ runtime+ .observation+ .capacity_status()+ .unwrap()+ .max_active_bytes,+ Some(20_000_000_000)+ );+ runtime+ .observation+ .set_capacity_limit(Some(20_000_000_000))+ .unwrap();+ drop(runtime);+ let db = Connection::open(&old_db).unwrap();+ assert_eq!(+ db.query_row(+ "SELECT value FROM observation_meta WHERE key='schema_version'",+ [],+ |row| row.get::<_, String>(0)+ )+ .unwrap(),+ "5"+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_meta WHERE key='generation_probe'",+ [],+ |row| row.get::<_, i64>(0)+ )+ .unwrap(),+ 0+ );+ db.execute(+ "INSERT INTO observation_meta VALUES('generation_probe','new-current-history')",+ [],+ )+ .unwrap();+ drop(db);+ let runtime = open(Default::default()).unwrap();+ assert_eq!(+ runtime+ .observation+ .capacity_status()+ .unwrap()+ .max_active_bytes,+ Some(20_000_000_000)+ );+ let db = Connection::open(&old_db).unwrap();+ assert_eq!(+ db.query_row(+ "SELECT value FROM observation_meta WHERE key='generation_probe'",+ [],+ |row| row.get::<_, String>(0)+ )+ .unwrap(),+ "new-current-history"+ );+ assert_eq!(fs::read(root.join("observation-hmac-key")).unwrap(), key);+ drop(db);+ runtime.observation.set_capacity_limit(None).unwrap();+ drop(runtime);+ let runtime = open(Default::default()).unwrap();+ assert_eq!(+ runtime+ .observation+ .capacity_status()+ .unwrap()+ .max_active_bytes,+ None+ );+}diff --git a/crates/daemon/src/control/runtime/observation_key.rs b/crates/daemon/src/control/runtime/observation_key.rsindex 7f31b62f..04cc6720 100644--- a/crates/daemon/src/control/runtime/observation_key.rs+++ b/crates/daemon/src/control/runtime/observation_key.rs@@ -56,6 +56,21 @@ pub(super) fn observation_workspace_key(Err(INVALID.into())}+pub(super) fn validate_existing_authority(storage_root: &Path) -> Result<(), String> {+ if storage_root.join("observation/activity.db").exists()+ || storage_root.join("observation.upgrade.json").exists()+ || storage_root.join("observation-source-backup.json").exists()+ {+ let mut file = open_key(&storage_root.join("observation-hmac-key")).map_err(|_| INVALID)?;+ if file.metadata().map_err(|_| INVALID)?.len() != 32 {+ return Err(INVALID.into());+ }+ let mut key = Zeroizing::new([0u8; 32]);+ file.read_exact(&mut *key).map_err(|_| INVALID)?;+ }+ Ok(())+}+fn open_key(path: &Path) -> std::io::Result<File> {hiroute_platform::private_fs::open_private_file(path)}@@ -78,4 +93,18 @@ mod tests {assert_eq!(std::fs::read(&path).unwrap(), malformed);}}++ #[test]+ fn a_sealed_history_link_requires_the_original_key_even_without_a_live_database() {+ let fixture = tempfile::tempdir().unwrap();+ std::fs::write(+ fixture.path().join("observation-source-backup.json"),+ b"pending source",+ )+ .unwrap();+ assert!(validate_existing_authority(fixture.path()).is_err());+ assert!(!fixture.path().join("observation-hmac-key").exists());+ std::fs::write(fixture.path().join("observation-hmac-key"), [7; 32]).unwrap();+ validate_existing_authority(fixture.path()).unwrap();+ }}diff --git a/crates/daemon/src/control/runtime_startup_progress_tests.rs b/crates/daemon/src/control/runtime_startup_progress_tests.rsnew file mode 100644index 00000000..397ec077--- /dev/null+++ b/crates/daemon/src/control/runtime_startup_progress_tests.rs@@ -0,0 +1,35 @@+use super::*;++#[test]+fn ordinary_startup_and_reopen_do_not_claim_a_history_upgrade() {+ #[cfg(unix)]+ if crate::test_support::isolated_agent_home(+ "control::runtime::startup_progress_tests::ordinary_startup_and_reopen_do_not_claim_a_history_upgrade",+ ) {+ return;+ }+ let directory = crate::test_support::private_tempdir();+ for _ in 0..2 {+ let (sender, receiver) = std::sync::mpsc::channel();+ let runtime = ProductionControlRuntime::open_inner_with_cpa_and_overrides(+ directory.path(),+ crate::release_catalog::fixture_catalog(),+ None,+ None,+ true,+ RuntimeOpenOverrides {+ storage_startup: hiroute_local_storage::StorageStartupOptions {+ upgrade_progress: Some(sender),+ ..Default::default()+ },+ ..Default::default()+ },+ )+ .unwrap();+ assert!(+ receiver.try_recv().is_err(),+ "ordinary startup must retain its normal readiness deadline"+ );+ drop(runtime);+ }+}diff --git a/crates/daemon/src/delegation/executor/tests.rs b/crates/daemon/src/delegation/executor/tests.rsindex 3f2bf60e..db13c11c 100644--- a/crates/daemon/src/delegation/executor/tests.rs+++ b/crates/daemon/src/delegation/executor/tests.rs@@ -319,7 +319,11 @@ fn fixture(let report = diagnostic_report(temporary.path());let runtime = Arc::new(MemoryRuntime::accepted());let observation = Arc::new(- LocalObservationStore::open(temporary.path(), DigestAuthority::new([seed; 32])).unwrap(),+ LocalObservationStore::open(+ temporary.path().join("observation"),+ DigestAuthority::new([seed; 32]),+ )+ .unwrap(),);let safety = Arc::new(RunSafetyProjection::new(Arc::new(SharedAdmissionGate::new()), "epoch".into()).unwrap(),diff --git a/crates/daemon/src/delegation/persistent_journal_tests.rs b/crates/daemon/src/delegation/persistent_journal_tests.rsindex 35f81cf0..d0b6f533 100644--- a/crates/daemon/src/delegation/persistent_journal_tests.rs+++ b/crates/daemon/src/delegation/persistent_journal_tests.rs@@ -626,8 +626,11 @@ fn interleaved_runs_are_reconstructable_from_their_own_tokens() {.unwrap(),);let observation = Arc::new(- LocalObservationStore::open(temporary.path(), DigestAuthority::new([marker as u8; 32]))- .unwrap(),+ LocalObservationStore::open(+ temporary.path().join("observation"),+ DigestAuthority::new([marker as u8; 32]),+ )+ .unwrap(),);PersistentWorkerRunJournal::new(Arc::new(MemoryRuntime::new(record.clone())),diff --git a/crates/daemon/src/role_all.rs b/crates/daemon/src/role_all.rsindex ebe5052b..821c813f 100644--- a/crates/daemon/src/role_all.rs+++ b/crates/daemon/src/role_all.rs@@ -434,6 +434,9 @@ pub fn start_role_all(config: RoleAllConfig) -> Result<RoleAllHandle, RoleAllErrhiroute_local_storage::StorageStartupOptions {gateway_lkg: Some(config.gateway_lkg.clone()),upgrade_progress: config.upgrade_progress.clone(),+ recovery_package: std::env::var_os("HIROUTE_RECOVERY_PACKAGE")+ .map(std::path::PathBuf::from),+ ..Default::default()},).map_err(|error| RoleAllError::Component("Storage", error));diff --git a/crates/domain/src/observation/query.rs b/crates/domain/src/observation/query.rsindex 60ba83a1..92df5fb0 100644--- a/crates/domain/src/observation/query.rs+++ b/crates/domain/src/observation/query.rs@@ -235,7 +235,27 @@ pub struct ObservationStatusV1 {pub production_exporter: String,}+#[derive(schemars::JsonSchema, Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]+#[serde(deny_unknown_fields)]+pub struct ObservationCapacityStatusV1 {+ pub max_active_bytes: Option<u64>,+ pub active_bytes: Option<u64>,+ pub measured_at_ms: Option<i64>,+ pub state: String,+ pub reason: String,+}+pub trait ObservationQueryPort: Send + Sync {+ fn observation_capacity(&self) -> Result<ObservationCapacityStatusV1, ObservationQueryError> {+ Err(ObservationQueryError::Unavailable)+ }+ fn set_observation_capacity(+ &self,+ _max_active_bytes: Option<u64>,+ ) -> Result<ObservationCapacityStatusV1, ObservationQueryError> {+ Err(ObservationQueryError::Unavailable)+ }+fn observation_analysis(&self,_reader: &super::ObservationReaderContext,diff --git a/crates/local-storage/src/backup/digest_tests.rs b/crates/local-storage/src/backup/digest_tests.rsnew file mode 100644index 00000000..a9ef4bad--- /dev/null+++ b/crates/local-storage/src/backup/digest_tests.rs@@ -0,0 +1,69 @@+use super::*;+use std::io::{self, Cursor, Read};++struct ShortReads<'a> {+ inner: Cursor<&'a [u8]>,+ max_requested: usize,+ interrupt_next: bool,+}++impl Read for ShortReads<'_> {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ self.max_requested = self.max_requested.max(buffer.len());+ if self.interrupt_next {+ self.interrupt_next = false;+ return Err(io::ErrorKind::Interrupted.into());+ }+ self.interrupt_next = true;+ let length = buffer.len().min(997);+ self.inner.read(&mut buffer[..length])+ }+}++#[test]+fn recovery_digest_streams_large_files_with_the_original_digest() {+ let bytes: Vec<u8> = (0..(17 * 64 * 1024 + 37))+ .map(|index| (index % 251) as u8)+ .collect();+ let expected = CanonicalDigest::of_bytes(&bytes);+ let mut reader = ShortReads {+ inner: Cursor::new(bytes.as_slice()),+ max_requested: 0,+ interrupt_next: true,+ };+ assert_eq!(+ stream_digest(&mut reader, bytes.len() as u64).unwrap(),+ expected+ );+ assert!(reader.max_requested <= 64 * 1024);+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("source.sqlite");+ fs::write(&path, &bytes).unwrap();+ assert_eq!(file_digest(&path).unwrap(), expected);+ fs::write(&path, b"changed source").unwrap();+ assert_ne!(file_digest(&path).unwrap(), expected);+}++#[test]+fn recovery_digest_rejects_truncated_or_growing_source() {+ assert!(stream_digest(&mut Cursor::new(b"short"), 6).is_err());+ assert!(stream_digest(&mut Cursor::new(b"growing"), 6).is_err());+ assert_eq!(+ stream_digest(&mut Cursor::new([]), 0).unwrap(),+ CanonicalDigest::of_bytes(b"")+ );+}++struct ReadFailure;++impl Read for ReadFailure {+ fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {+ Err(io::ErrorKind::PermissionDenied.into())+ }+}++#[test]+fn recovery_digest_preserves_read_failures() {+ assert!(stream_digest(&mut ReadFailure, 1).is_err());+ assert!(stream_digest(&mut ReadFailure, 0).is_err());+}diff --git a/crates/local-storage/src/backup/mod.rs b/crates/local-storage/src/backup/mod.rsindex 9586d123..493d7892 100644--- a/crates/local-storage/src/backup/mod.rs+++ b/crates/local-storage/src/backup/mod.rs@@ -1,3 +1,5 @@+#[cfg(test)]+mod digest_tests;pub(crate) mod source_snapshot;use std::fs;use std::path::{Path, PathBuf};@@ -303,7 +305,7 @@ impl BackupSet {/// The caller must retain the startup barrier and directory ownership until publication ends.pub fn publish_stopped_source(&self,- _barrier: &SingleWriterBackupBarrier,+ barrier: &SingleWriterBackupBarrier,storage_root: &Path,options: &crate::StorageStartupOptions,) -> Result<(), LocalStorageError> {@@ -328,10 +330,27 @@ impl BackupSet {return Err(LocalStorageError::InvalidData);}}- let metadata = serde_json::json!({- "batch_id": self.set_id(), "target_schema_version": self.target_schema_version(),- "source_stores": {"control":self.control.manifest(),"runtime":self.runtime.manifest(),"secrets":self.secrets.manifest()}- });+ self.publish_coordinated_source(barrier, storage_root, options)+ }++ /// The SQL coordinator admits only its verified source/target phases. Those+ /// phases have never opened business or Observation writers, so unchanged+ /// non-core state can be paired with the original three BackupSet sources+ /// even when an older binary committed a SQL step before it stopped.+ pub(crate) fn publish_coordinated_source(+ &self,+ _barrier: &SingleWriterBackupBarrier,+ storage_root: &Path,+ options: &crate::StorageStartupOptions,+ ) -> Result<(), LocalStorageError> {+ if self+ .root+ .canonicalize()?+ .starts_with(storage_root.canonicalize()?)+ {+ return Err(LocalStorageError::InvalidData);+ }+ let metadata = serde_json::json!({"batch_id":self.set_id(),"target_schema_version":self.target_schema_version(),"source_stores":{"control":self.control.manifest(),"runtime":self.runtime.manifest(),"secrets":self.secrets.manifest()},"source_group":"original_core_batch_and_stopped_non_core_state"});source_snapshot::publish(storage_root, &self.root, &metadata, options)}@@ -1144,7 +1163,44 @@ fn write_owner_file(path: &Path, bytes: &[u8]) -> Result<(), LocalStorageError>}fn file_digest(path: &Path) -> Result<CanonicalDigest, LocalStorageError> {- Ok(CanonicalDigest::of_bytes(&fs::read(path)?))+ let mut file = fs::File::open(path)?;+ let before = file.metadata()?;+ if !before.is_file() {+ return Err(LocalStorageError::InvalidData);+ }+ let digest = stream_digest(&mut file, before.len())?;+ let after = file.metadata()?;+ if before.len() != after.len() || before.modified()? != after.modified()? {+ return Err(LocalStorageError::InvalidData);+ }+ Ok(digest)+}++// Preserve the released raw-byte SHA-256 while bounding memory and the admitted read length.+// A stopped source that shrinks or grows cannot become a verified recovery source.+fn stream_digest(+ reader: &mut impl std::io::Read,+ byte_count: u64,+) -> Result<CanonicalDigest, LocalStorageError> {+ let mut remaining = byte_count;+ let mut buffer = zeroize::Zeroizing::new([0_u8; 64 * 1024]);+ let mut hasher = Sha256::new();+ while remaining != 0 {+ let count = remaining.min(buffer.len() as u64) as usize;+ reader.read_exact(&mut buffer[..count])?;+ hasher.update(&buffer[..count]);+ remaining -= count as u64;+ }+ loop {+ match reader.read(&mut buffer[..1]) {+ Ok(0) => break,+ Ok(_) => return Err(LocalStorageError::InvalidData),+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,+ Err(error) => return Err(error.into()),+ }+ }+ CanonicalDigest::parse(format!("sha256:{}", encode_hex(&hasher.finalize())))+ .map_err(|_| LocalStorageError::InvalidData)}fn random_attempt_id() -> Result<String, LocalStorageError> {diff --git a/crates/local-storage/src/backup/source_snapshot.rs b/crates/local-storage/src/backup/source_snapshot.rsindex 74034d1f..9e7002a7 100644--- a/crates/local-storage/src/backup/source_snapshot.rs+++ b/crates/local-storage/src/backup/source_snapshot.rs@@ -3,6 +3,7 @@ use super::*;use crate::migrations::{prepare_storage_root as prepare_owner_directory, validate_owner_file};use serde_json::{Value, json};use std::collections::BTreeMap;+mod seal;#[cfg(windows)]use std::io::{Read, Write};@@ -103,7 +104,14 @@ fn copy_tree(source: &Path, target: &Path) -> Result<(), LocalStorageError> {Ok(())}fn excluded(name: &str) -> bool {- name == "live"+ matches!(+ name,+ "observation.pre-v5-source"+ | "observation.v5-target"+ | "observation.v5-legacy-input"+ | "observation.upgrade.json"+ | "observation-source-backup.json"+ ) || name == "live"|| name.starts_with("migration-set")|| name == "startup.lock"|| name.ends_with(".pid")@@ -121,10 +129,24 @@ pub(super) fn publish() -> Result<(), LocalStorageError> {let source = backup.join("source");if source.exists() {- return verify(backup);+ verify(backup)?;+ return publish_locator(+ backup,+ metadata["batch_id"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?,+ );}let stage = backup.join("source-staging");+ if options.preserve_observation_source && stage.join(MANIFEST).exists() {+ return seal::finish(root, backup, &stage, metadata);+ }+ preflight_space(root, backup, options)?;if stage.exists() {+ // Never put a moved historical tree into a generic incomplete-copy path.+ if stage.join("storage/observation").exists() {+ return Err(LocalStorageError::InvalidData);+ }// Preserve an interrupted attempt; it is never promoted or reused as complete.let mut id = [0u8; 8];getrandom::fill(&mut id).map_err(|_| LocalStorageError::Crypto)?;@@ -140,7 +162,9 @@ pub(super) fn publish(for entry in fs::read_dir(root)? {let entry = entry?;let name = entry.file_name();- if excluded(&name.to_string_lossy()) {+ if excluded(&name.to_string_lossy())+ || (options.preserve_observation_source && name == "observation")+ {continue;}copy_tree(&entry.path(), &storage.join(name))?;@@ -180,45 +204,153 @@ pub(super) fn publish(}}}+ let package_identity = if let Some(package) = &options.recovery_package {+ let meta = fs::symlink_metadata(package)?;+ if !meta.is_dir()+ || meta.file_type().is_symlink()+ || package.canonicalize()?.starts_with(backup.canonicalize()?)+ || storage_root_conflict(package, root)?+ {+ return Err(LocalStorageError::InvalidData);+ }+ copy_tree(package, &stage.join("recovery-package"))?;+ format!(+ "配套旧包原路径:{};完整副本位于 recovery-package/,身份以清单中的逐文件 SHA-256 为准。",+ package.display()+ )+ } else {+ "使用创建备份时的配套完整安装包;本说明不承诺支持其它存储格式。保留旧包后再开始安装验证。"+ .to_owned()+ };let extra_paths = extra.iter().map(|(name, target)| format!("- {name} → {target}\n")).collect::<String>();+ let history = if options.preserve_observation_source {+ "原 Observation 目录已在同一文件系统原地封存;包含原始数据库、WAL/SHM、对象和暂存文件,未逐文件扫描、哈希、转换或压缩。清单仅校验其余恢复材料并绑定封存目录身份,不宣称已逐字节验证历史内容。新版历史和统计从切换后开始。封存仍占用磁盘空间,不受当前观测容量回收影响;它是本机恢复点,不是独立灾备。需要独立备份时,在启动之外复制并校验本目录。恢复前离线检查原始数据库和内容。"+ } else {+ "本次可能转换核心数据库和 Observation 历史;该目录同时保存原始 Observation、密钥、原生恢复材料及其余自有持久状态。"+ };let text = format!(- "# HiRoute 手工恢复\n\n备份批次 ID:{}\n\n备份源以本说明所在目录为准,目录改名或归档后仍从这里读取;不要使用其它批次。以下源文件路径均相对于本说明所在目录,目标路径为绝对位置。\n\n1. 完全退出 HiRoute,确认服务、CPA、任务与后台自动重启均已停止。\n2. 将 storage 目录 {} 改名保留;不要改名它的上级目录,备份仍保留在上述位置。恢复将舍弃备份之后的新数据。\n3. 将 storage/ 整个复制回 {},保留权限。不要逐库覆盖,不要遗留不配套的 WAL/SHM。再按以下源 → 目标路径复制额外文件:\n\n{}\n4. 按下面的完整 App 路径和身份覆盖安装旧包,启动后检查配置、Agent 接入及一次真实请求。\n\n{}\n\n本次只修改三个数据库;该目录同时保存密钥、原生恢复材料及其余自有持久状态。不要将此备份加入诊断导出。\n",+ "# HiRoute 手工恢复\n\n备份批次 ID:{}\n\n备份源以本说明所在目录为准,目录改名或归档后仍从这里读取;不要使用其它批次。以下源文件路径均相对于本说明所在目录,目标路径为绝对位置。\n\n1. 完全退出 HiRoute,确认服务、CPA、任务与后台自动重启均已停止。\n2. 将 storage 目录 {} 改名保留;不要改名它的上级目录,备份仍保留在上述位置。备份之后的新数据必须与配套新版本单独保留,不能混入旧格式恢复。\n3. 将 storage/ 整个复制回 {},保留权限。不要逐库覆盖,不要遗留不配套的 WAL/SHM。再按以下源 → 目标路径复制额外文件:\n\n{}\n4. 按下面的完整 App 路径和身份覆盖安装旧包,启动后检查配置、Agent 接入及一次真实请求。\n\n{}\n\n{history}\n\n不要将此备份加入诊断导出。\n",metadata["batch_id"].as_str().ok_or(LocalStorageError::InvalidData)?,root.display(),root.display(),extra_paths,- "使用创建备份时的配套完整安装包;本说明不承诺支持其它存储格式。"+ package_identity);write_private(&stage.join("恢复说明.md"), text.as_bytes())?;let mut files = BTreeMap::new();- collect(&stage, &stage, &mut files)?;+ collect(&stage, &stage, &mut files, false)?;let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|_| LocalStorageError::InvalidData)?.as_secs();let mut info = metadata.clone();info["created_unix_seconds"] = json!(created);- let manifest = json!({"schema":"hiroute.upgrade-source-backup/v1","complete":true,+ let mut manifest = json!({"schema":"hiroute.upgrade-source-backup/v1","complete":true,"source_storage":root.to_string_lossy(),"extra_locations":extra,"files":files,"source_info":info});+ if options.preserve_observation_source {+ manifest["schema"] = json!("hiroute.upgrade-source-backup/v2");+ manifest["sealed_observation"] = seal::capture(root, &storage)?;+ }write_private(&stage.join(MANIFEST),&serde_json::to_vec_pretty(&manifest).map_err(|_| LocalStorageError::InvalidData)?,)?;hiroute_platform::durable::directory_barrier(&stage)?;+ if options.preserve_observation_source {+ return seal::finish(root, backup, &stage, metadata);+ }hiroute_platform::durable::rename(stage, &source)?;hiroute_platform::durable::directory_barrier(backup)?;- verify(backup)+ verify(backup)?;+ publish_locator(+ backup,+ metadata["batch_id"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?,+ )+}++/// A bounded directory hint after full verification. It is outside the immutable+/// source and never replaces the authoritative file manifest during recovery.+pub(crate) fn publish_locator(backup: &Path, batch_id: &str) -> Result<(), LocalStorageError> {+ if manifest(backup)?["source_info"]["batch_id"] != batch_id {+ return Err(LocalStorageError::InvalidData);+ }+ let bytes = serde_json::to_vec(&json!({"schema":1,"complete":true,"batch_id":batch_id}))+ .map_err(|_| LocalStorageError::InvalidData)?;+ hiroute_platform::durable::write_private_atomic(&backup.join("source-location.json"), &bytes)?;+ hiroute_platform::durable::directory_barrier(backup)?;+ Ok(())+}++fn preflight_space(+ root: &Path,+ backup: &Path,+ options: &crate::StorageStartupOptions,+) -> Result<(), LocalStorageError> {+ fn bytes(path: &Path) -> Result<u64, LocalStorageError> {+ let meta = fs::symlink_metadata(path)?;+ if meta.file_type().is_symlink() {+ return Err(LocalStorageError::Permission);+ }+ if meta.is_file() {+ return Ok(meta+ .len()+ .max(fs2::FileExt::allocated_size(&fs::File::open(path)?)?));+ }+ if !meta.is_dir() {+ return Err(LocalStorageError::InvalidData);+ }+ let mut total = 0u64;+ for entry in fs::read_dir(path)? {+ total = total+ .checked_add(bytes(&entry?.path())?)+ .ok_or(LocalStorageError::InvalidData)?;+ }+ Ok(total)+ }+ let mut required = 256_u64 * 1024 * 1024;+ for entry in fs::read_dir(root)? {+ let entry = entry?;+ if !excluded(&entry.file_name().to_string_lossy())+ && !(options.preserve_observation_source && entry.file_name() == "observation")+ {+ required = required.saturating_add(bytes(&entry.path())?);+ }+ }+ for name in ["control.db", "runtime.db", "secrets.db"] {+ required = required.saturating_add(bytes(&backup.join(name))?);+ }+ if let Some(package) = &options.recovery_package {+ let meta = fs::symlink_metadata(package)?;+ if !meta.is_dir()+ || meta.file_type().is_symlink()+ || package.canonicalize()?.starts_with(backup.canonicalize()?)+ || storage_root_conflict(package, root)?+ {+ return Err(LocalStorageError::InvalidData);+ }+ required = required.saturating_add(bytes(package)?);+ }+ if fs2::available_space(root)? < required {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage("Insufficient space for recovery material and startup headroom; preserve the source and free unrelated space before retrying"));+ }+ Ok(())}fn collect(root: &Path,path: &Path,files: &mut BTreeMap<String, String>,+ sealed: bool,) -> Result<(), LocalStorageError> {+ if sealed && path == root.join("storage/observation") {+ return Ok(());+ }let metadata = fs::symlink_metadata(path)?;if metadata.file_type().is_symlink() {return Err(LocalStorageError::Permission);@@ -229,7 +361,7 @@ fn collect(if entry.path() == root.join(MANIFEST) {continue;}- collect(root, &entry.path(), files)?;+ collect(root, &entry.path(), files, sealed)?;}} else if metadata.is_file() {files.insert(@@ -241,7 +373,7 @@ fn collect(.map(|component| component.as_os_str().to_string_lossy().into_owned()).collect::<Vec<_>>().join("/"),- CanonicalDigest::of_bytes(&fs::read(path)?).to_string(),+ file_digest(path)?.to_string(),);} else {return Err(LocalStorageError::InvalidData);@@ -249,21 +381,46 @@ fn collect(Ok(())}fn manifest(backup: &Path) -> Result<Value, LocalStorageError> {- validate_owner_file(&backup.join("source").join(MANIFEST))?;- let value: Value = serde_json::from_slice(&fs::read(backup.join("source").join(MANIFEST))?)+ read_manifest(&backup.join("source"))+}+fn read_manifest(source: &Path) -> Result<Value, LocalStorageError> {+ validate_owner_file(&source.join(MANIFEST))?;+ let value: Value = serde_json::from_slice(&fs::read(source.join(MANIFEST))?).map_err(|_| LocalStorageError::InvalidData)?;- if value["schema"] != "hiroute.upgrade-source-backup/v1"- || value["complete"] != true- || value.as_object().is_none_or(|o| o.len() != 6)- {+ let fields = match value["schema"].as_str() {+ Some("hiroute.upgrade-source-backup/v1") => 6,+ Some("hiroute.upgrade-source-backup/v2") if value.get("sealed_observation").is_some() => 7,+ _ => return Err(LocalStorageError::InvalidData),+ };+ if value["complete"] != true || value.as_object().is_none_or(|o| o.len() != fields) {return Err(LocalStorageError::InvalidData);}Ok(value)}-pub(super) fn verify(backup: &Path) -> Result<(), LocalStorageError> {+pub(crate) fn verify(backup: &Path) -> Result<(), LocalStorageError> {let value = manifest(backup)?;+ verify_files(&backup.join("source"), &value)?;+ if value.get("sealed_observation").is_some() {+ seal::verify_directory(&backup.join("source/storage/observation"), &value)?;+ }+ Ok(())+}+pub(crate) fn verify_sealed(backup: &Path) -> Result<(), LocalStorageError> {+ if manifest(backup)?["schema"] != "hiroute.upgrade-source-backup/v2" {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage(+ "An earlier full-history conversion source exists; preserve it and use its whole-group recovery guide before retrying this upgrade",+ ));+ }+ verify(backup)+}+fn verify_files(source: &Path, value: &Value) -> Result<(), LocalStorageError> {let mut actual = BTreeMap::new();- collect(&backup.join("source"), &backup.join("source"), &mut actual)?;+ collect(+ source,+ source,+ &mut actual,+ value.get("sealed_observation").is_some(),+ )?;if json!(actual) != value["files"] {return Err(LocalStorageError::InvalidData);}@@ -281,7 +438,12 @@ pub(crate) fn verify_unchanged(root: &Path, backup: &Path) -> Result<(), LocalStif excluded(&entry.file_name().to_string_lossy()) {continue;}- collect(root, &entry.path(), &mut actual)?;+ if value.get("sealed_observation").is_some() && entry.file_name() == "observation" {+ // The old generation is bound at its sealed location, not compared+ // with the intentionally empty/new live Observation generation.+ continue;+ }+ collect(root, &entry.path(), &mut actual, false)?;}let expected = value["files"].as_object()@@ -301,13 +463,19 @@ pub(crate) fn verify_unchanged(root: &Path, backup: &Path) -> Result<(), LocalSt.ok_or(LocalStorageError::InvalidData)?{let path = path.as_str().ok_or(LocalStorageError::InvalidData)?;- if json!(CanonicalDigest::of_bytes(&fs::read(path)?).to_string()) != value["files"][name] {+ if json!(file_digest(Path::new(path))?.to_string()) != value["files"][name] {return Err(LocalStorageError::InvalidData);}}Ok(())}+fn storage_root_conflict(package: &Path, root: &Path) -> Result<bool, LocalStorageError> {+ let package = package.canonicalize()?;+ let root = root.canonicalize()?;+ Ok(package.starts_with(&root) || root.starts_with(&package))+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/local-storage/src/backup/source_snapshot/seal.rs b/crates/local-storage/src/backup/source_snapshot/seal.rsnew file mode 100644index 00000000..09290ff9--- /dev/null+++ b/crates/local-storage/src/backup/source_snapshot/seal.rs@@ -0,0 +1,118 @@+//! A stopped historical directory changes namespace, never representation.+use super::*;++fn identity(path: &Path) -> Result<[u64; 2], LocalStorageError> {+ // Keep ancestor/type admission with the existing held-directory adapter.+ let _directory = hiroute_diagnostics::files::PrivateDir::open_existing(path)+ .map_err(|_| LocalStorageError::Permission)?;+ #[cfg(unix)]+ {+ use std::os::unix::fs::MetadataExt;+ let file = rustix::fs::open(+ path,+ rustix::fs::OFlags::RDONLY+ | rustix::fs::OFlags::DIRECTORY+ | rustix::fs::OFlags::NOFOLLOW+ | rustix::fs::OFlags::CLOEXEC,+ rustix::fs::Mode::empty(),+ )+ .map_err(std::io::Error::from)?;+ let file = fs::File::from(file);+ let meta = file.metadata()?;+ Ok([meta.dev(), meta.ino()])+ }+ #[cfg(windows)]+ {+ let file = hiroute_platform::windows::security::open_identity(path, true)?;+ let (volume, index) = hiroute_platform::windows::security::file_identity(&file)?;+ Ok([u64::from(volume), index])+ }+}++pub(super) fn capture(root: &Path, destination: &Path) -> Result<Value, LocalStorageError> {+ let source = identity(&root.join("observation"))?;+ if source[0] != identity(destination)?[0] {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage(+ "Old Observation must be sealed on the same filesystem; no full-copy fallback is allowed",+ ));+ }+ Ok(json!({"identity":source,"contents":"preserved_without_recursive_verification"}))+}++pub(super) fn verify_directory(path: &Path, value: &Value) -> Result<(), LocalStorageError> {+ let seal = &value["sealed_observation"];+ if seal.as_object().is_none_or(|o| o.len() != 2)+ || seal["contents"] != "preserved_without_recursive_verification"+ || json!(identity(path)?) != seal["identity"]+ {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage(+ "Sealed Observation is missing or its directory identity changed; preserve all generations and use the original recovery guide",+ ));+ }+ Ok(())+}++pub(super) fn finish(+ root: &Path,+ backup: &Path,+ stage: &Path,+ metadata: &Value,+) -> Result<(), LocalStorageError> {+ let _stage = identity(stage)?;+ let value = read_manifest(stage)?;+ if value["schema"] != "hiroute.upgrade-source-backup/v2"+ || value["source_storage"] != root.to_string_lossy().as_ref()+ || value["source_info"]["batch_id"] != metadata["batch_id"]+ {+ return Err(LocalStorageError::InvalidData);+ }+ // This scans/checks only the copied recovery materials. The old history is+ // not yet present here, or is deliberately excluded after an interrupted move.+ verify_files(stage, &value)?;+ let live = root.join("observation");+ let sealed = stage.join("storage/observation");+ if sealed.exists() {+ if live.exists() {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage(+ "Both live and sealed Observation exist during cutover; preserve both and use the original recovery guide",+ ));+ }+ verify_directory(&sealed, &value)?;+ } else {+ verify_directory(&live, &value)?;+ if identity(&stage.join("storage"))?[0] != identity(&live)?[0] {+ return Err(LocalStorageError::InvalidData);+ }+ checkpoint("prepared")?;+ hiroute_platform::durable::rename(&live, &sealed)?;+ hiroute_platform::durable::directory_barrier(root)?;+ checkpoint("renamed")?;+ verify_directory(&sealed, &value)?;+ }+ hiroute_platform::durable::directory_barrier(root)?;+ hiroute_platform::durable::rename(stage, backup.join("source"))?;+ hiroute_platform::durable::directory_barrier(backup)?;+ checkpoint("published")?;+ publish_locator(+ backup,+ value["source_info"]["batch_id"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?,+ )+}++fn checkpoint(_phase: &str) -> Result<(), LocalStorageError> {+ #[cfg(test)]+ if FAULT.with(|fault| fault.get() == Some(_phase)) {+ return Err(std::io::Error::other("interrupted sealed-history publication").into());+ }+ Ok(())+}++#[cfg(test)]+thread_local! {+ pub(crate) static FAULT: std::cell::Cell<Option<&'static str>> = const { std::cell::Cell::new(None) };+}++#[cfg(test)]+mod tests;diff --git a/crates/local-storage/src/backup/source_snapshot/seal/tests.rs b/crates/local-storage/src/backup/source_snapshot/seal/tests.rsnew file mode 100644index 00000000..a74e1a77--- /dev/null+++ b/crates/local-storage/src/backup/source_snapshot/seal/tests.rs@@ -0,0 +1,160 @@+use super::*;+use crate::{LocalStorageSet, StorageStartupOptions};++fn fixture() -> (crate::TestTempDir, PathBuf, StorageStartupOptions) {+ let temporary = crate::test_tempdir().unwrap();+ let root = temporary.path().join("storage");+ drop(LocalStorageSet::open_for_daemon_startup(&root).unwrap());+ let observation = root.join("observation");+ crate::prepare_owner_directory(&observation).unwrap();+ let db = Connection::open(observation.join("activity.db")).unwrap();+ db.execute_batch("CREATE TABLE observation_meta(key TEXT PRIMARY KEY,value TEXT); INSERT INTO observation_meta VALUES('schema_version','4'); CREATE TABLE retained(id INTEGER PRIMARY KEY,body BLOB); INSERT INTO retained VALUES(17,X'00ff616263');").unwrap();+ drop(db);+ for (name, body) in [+ ("activity.db-wal", b"retained sidecar".as_slice()),+ ("activity.db-shm", b"retained shared memory".as_slice()),+ (+ "conversation-content/blobs/body",+ b"original body\0\xff".as_slice(),+ ),+ (+ "conversation-content/staging/pending",+ b"pending capture".as_slice(),+ ),+ ] {+ let path = observation.join(name);+ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(path, body).unwrap();+ }+ let package = temporary.path().join("old-package");+ crate::prepare_owner_directory(&package).unwrap();+ fs::write(package.join("old-daemon"), b"matching program fixture").unwrap();+ (+ temporary,+ root,+ StorageStartupOptions {+ preserve_observation_source: true,+ recovery_package: Some(package),+ ..Default::default()+ },+ )+}++fn backup(root: &Path) -> PathBuf {+ let link: Value =+ serde_json::from_slice(&fs::read(root.join("observation-source-backup.json")).unwrap())+ .unwrap();+ crate::upgrade_backup_root(root)+ .unwrap()+ .parent()+ .unwrap()+ .join(link["backup_name"].as_str().unwrap())+}++#[test]+fn sealing_keeps_the_original_directory_and_never_opens_historical_files() {+ let (_temporary, root, options) = fixture();+ let before = identity(&root.join("observation")).unwrap();+ #[cfg(unix)]+ {+ use std::os::unix::fs::{PermissionsExt, symlink};+ // A recursive reader/copy/hash must fail here; sealing preserves the+ // stopped opaque tree without following this old unknown entry.+ fs::set_permissions(+ root.join("observation/conversation-content/blobs/body"),+ fs::Permissions::from_mode(0o0),+ )+ .unwrap();+ symlink("missing-target", root.join("observation/unopened-link")).unwrap();+ }+ drop(LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).unwrap());+ let backup = backup(&root);+ let sealed = backup.join("source/storage/observation");+ assert!(!root.join("observation").exists());+ assert!(LocalStorageSet::observation_cutover_pending(&root));+ assert_eq!(identity(&sealed).unwrap(), before);+ assert_eq!(+ fs::read(sealed.join("activity.db-wal")).unwrap(),+ b"retained sidecar"+ );+ assert_eq!(+ fs::read(sealed.join("activity.db-shm")).unwrap(),+ b"retained shared memory"+ );+ assert_eq!(+ fs::read(sealed.join("conversation-content/staging/pending")).unwrap(),+ b"pending capture"+ );+ super::super::verify_sealed(&backup).unwrap();+ let manifest = super::super::manifest(&backup).unwrap();+ assert!(+ !manifest["files"]+ .as_object()+ .unwrap()+ .keys()+ .any(|key| key.starts_with("storage/observation/"))+ );+ assert_eq!(+ manifest["sealed_observation"]["contents"],+ "preserved_without_recursive_verification"+ );+ assert!(backup.join("source/storage/live/control.db").is_file());+ assert!(backup.join("source/storage/master-key").is_file());+ assert!(backup.join("source/recovery-package/old-daemon").is_file());+}++#[test]+fn restart_finishes_the_same_cutover_at_each_namespace_checkpoint() {+ for phase in ["prepared", "renamed", "published"] {+ let (_temporary, root, options) = fixture();+ let before = identity(&root.join("observation")).unwrap();+ FAULT.with(|fault| fault.set(Some(phase)));+ let result = LocalStorageSet::open_for_daemon_startup_with_options(&root, &options);+ FAULT.with(|fault| fault.set(None));+ assert!(result.is_err(), "{phase}");+ let link = fs::read(root.join("observation-source-backup.json")).unwrap();+ drop(LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).unwrap());+ let backup = backup(&root);+ assert_eq!(+ identity(&backup.join("source/storage/observation")).unwrap(),+ before+ );+ assert_eq!(+ fs::read(root.join("observation-source-backup.json")).unwrap(),+ link+ );+ assert!(!backup.join("source-staging").exists());+ super::super::verify_sealed(&backup).unwrap();+ }+}++#[test]+fn conflicting_live_root_or_replaced_sealed_identity_never_resets_either() {+ let (_temporary, root, options) = fixture();+ FAULT.with(|fault| fault.set(Some("renamed")));+ assert!(LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).is_err());+ FAULT.with(|fault| fault.set(None));+ let backup = backup(&root);+ fs::create_dir(root.join("observation")).unwrap();+ fs::write(+ root.join("observation/foreign"),+ b"preserve conflicting generation",+ )+ .unwrap();+ assert!(LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).is_err());+ assert_eq!(+ fs::read(root.join("observation/foreign")).unwrap(),+ b"preserve conflicting generation"+ );+ fs::rename(root.join("observation"), root.join("conflicting-preserved")).unwrap();+ let sealed = backup.join("source-staging/storage/observation");+ fs::rename(&sealed, sealed.with_file_name("retained-original")).unwrap();+ fs::create_dir(&sealed).unwrap();+ assert!(LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).is_err());+ assert!(+ sealed+ .with_file_name("retained-original")+ .join("activity.db")+ .exists()+ );+}diff --git a/crates/local-storage/src/lib.rs b/crates/local-storage/src/lib.rsindex 8cf8071d..e025bf2e 100644--- a/crates/local-storage/src/lib.rs+++ b/crates/local-storage/src/lib.rs@@ -40,11 +40,22 @@ pub struct LocalStorageSet {#[derive(Clone, Debug, Default)]pub struct StorageStartupOptions {pub gateway_lkg: Option<std::path::PathBuf>,+ /// Seal old Observation unchanged with the complete matching-version source.+ pub preserve_observation_source: bool,+ /// An admitted old package retained by the installer/operator for recovery.+ pub recovery_package: Option<std::path::PathBuf>,pub upgrade_progress:Option<std::sync::mpsc::Sender<hiroute_host_runtime::StorageUpgradePhase>>,}impl LocalStorageSet {+ /// A published source link also protects the intentionally absent live root+ /// between the old-directory rename and fresh Observation initialization.+ pub fn observation_cutover_pending(storage_root: &Path) -> bool {+ storage_root.join("observation-source-backup.json").exists()+ && !storage_root.join("observation/activity.db").exists()+ }+/// Opens and migrates the coordinated three-store set before daemon listeners or/// transaction admission become available. The authority and writer barrier are minted only/// inside this crate so daemon composition cannot open an individual store or assert an@@ -68,14 +79,20 @@ impl LocalStorageSet {authority: &DaemonStorageAuthority,barrier: &SingleWriterBackupBarrier,storage_root: impl AsRef<Path>,- _options: &StorageStartupOptions,+ options: &StorageStartupOptions,) -> Result<Self, LocalStorageError> {let storage_root = storage_root.as_ref();migrations::prepare_storage_root(storage_root)?;migrations::validate_startup_format(storage_root)?;let startup_lock = std::sync::Arc::new(migrations::acquire_startup_lock(storage_root)?);migrations::validate_startup_format(storage_root)?;- Self::open_locked_stores(authority, barrier, storage_root, startup_lock)+ Self::open_locked_stores(+ authority,+ barrier,+ storage_root,+ startup_lock,+ Some(options),+ )}// SQL coordinator component harness only: unsupported formats never pass through the@@ -88,7 +105,7 @@ impl LocalStorageSet {) -> Result<Self, LocalStorageError> {migrations::prepare_storage_root(storage_root)?;let lock = std::sync::Arc::new(migrations::acquire_startup_lock(storage_root)?);- Self::open_locked_stores(authority, barrier, storage_root, lock)+ Self::open_locked_stores(authority, barrier, storage_root, lock, None)}fn open_locked_stores(@@ -96,6 +113,7 @@ impl LocalStorageSet {barrier: &SingleWriterBackupBarrier,storage_root: &Path,startup_lock: std::sync::Arc<std::fs::File>,+ options: Option<&StorageStartupOptions>,) -> Result<Self, LocalStorageError> {let live_root = storage_root.join("live");let backup_root = upgrade_backup_root(storage_root)?;@@ -125,6 +143,16 @@ impl LocalStorageSet {&backup_root,secret_binding.as_ref(),)?;+ if let Some(options) = options {+ migration.preserve_complete_source(+ authority,+ barrier,+ storage_root,+ &backup_root,+ options,+ secret_binding.as_ref(),+ )?;+ }let target_schema_version = migration.target_schema_version();let control_result = if migration.durable_set_prepared() {ControlStore::open_from_migration_set(@@ -233,7 +261,13 @@ impl LocalStorageSet {drop(runtime);drop(control);drop(migration);- return Self::open_locked_stores(authority, barrier, storage_root, startup_lock);+ return Self::open_locked_stores(+ authority,+ barrier,+ storage_root,+ startup_lock,+ options,+ );}control.startup_lock = Some(startup_lock.clone());runtime.startup_lock = Some(startup_lock.clone());diff --git a/crates/local-storage/src/migrations/mod.rs b/crates/local-storage/src/migrations/mod.rsindex 61ba7456..892df701 100644--- a/crates/local-storage/src/migrations/mod.rs+++ b/crates/local-storage/src/migrations/mod.rs@@ -15,6 +15,8 @@ mod agent_surface_checks_v25;#[cfg(test)]mod agent_surface_checks_v25_tests;mod current_storage;+mod observation_source;+mod recovery_package;mod startup_format;pub(crate) use startup_format::validate_startup_format;#[cfg(test)]diff --git a/crates/local-storage/src/migrations/observation_source.rs b/crates/local-storage/src/migrations/observation_source.rsnew file mode 100644index 00000000..cf55cfa9--- /dev/null+++ b/crates/local-storage/src/migrations/observation_source.rs@@ -0,0 +1,280 @@+//! Complete stopped-source publication before either core or Observation changes.+use super::*;+use crate::StorageStartupOptions;+use hiroute_platform::durable;++impl MigrationSetCoordinator {+ pub(crate) fn preserve_complete_source(+ &self,+ authority: &DaemonStorageAuthority,+ barrier: &SingleWriterBackupBarrier,+ root: &Path,+ backup_root: &Path,+ options: &StorageStartupOptions,+ binding: Option<&MigrationSecretBinding>,+ ) -> Result<(), LocalStorageError> {+ let link = root.join("observation-source-backup.json");+ if options.preserve_observation_source && link.exists() {+ validate_owner_file(&link)?;+ if fs::metadata(&link)?.len() > 16 * 1024 {+ return Err(LocalStorageError::InvalidData);+ }+ let record: serde_json::Value = serde_json::from_slice(&fs::read(&link)?)+ .map_err(|_| LocalStorageError::InvalidData)?;+ let id = record["set_id"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?;+ let name = record["backup_name"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?;+ let portable = |s: &str| {+ !s.is_empty()+ && s.len() <= 128+ && s.bytes()+ .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.'))+ && !s.contains("..")+ };+ if record["schema"] != 1 || !portable(id) || !portable(name) {+ return Err(LocalStorageError::InvalidData);+ }+ let parent = backup_root.parent().ok_or(LocalStorageError::InvalidData)?;+ let direct = parent.join(name);+ let archived = parent.join(format!("migration-set.{id}"));+ let path = if direct.exists() {+ let set = BackupSet::open(authority, &direct)?;+ if set.set_id() == id { direct } else { archived }+ } else {+ archived+ };+ let set = BackupSet::open(authority, &path)?;+ if set.set_id() != id {+ return Err(LocalStorageError::InvalidData);+ }+ if path.join("source").exists() {+ crate::backup::source_snapshot::verify_sealed(&path)?;+ crate::backup::source_snapshot::publish_locator(&path, set.set_id())?;+ } else {+ let mut resumed = options.clone();+ resumed.recovery_package = record["recovery_package"].as_str().map(PathBuf::from);+ if resumed.recovery_package.is_none() {+ return Err(LocalStorageError::InvalidData);+ }+ // The SQL coordinator already admitted this original batch,+ // including any committed core steps. Never re-snapshot a mixed+ // live group or require it to equal the original core sources.+ set.publish_coordinated_source(barrier, root, &resumed)?;+ }+ return Ok(());+ }+ let mut options = options.clone();+ if options.preserve_observation_source && options.recovery_package.is_none() {+ options.recovery_package = super::recovery_package::previous_desktop(root)?;+ if options.recovery_package.is_none() {+ return Err(LocalStorageError::InvalidData.at_upgrade_stage("Observation history sealing requires a matching old package: set HIROUTE_RECOVERY_PACKAGE"));+ }+ }+ let source = self.set.as_ref().filter(|set| {+ !matches!(+ set.phase(),+ BackupSetPhase::Completed | BackupSetPhase::Restoring | BackupSetPhase::Restored+ )+ });+ let owned;+ let set = if let Some(set) = source {+ set+ } else if options.preserve_observation_source {+ let path = backup_root+ .parent()+ .ok_or(LocalStorageError::InvalidData)?+ .join("observation-format-5");+ if BackupSet::durable_manifest_present(&path)? && path.join("source").exists() {+ let previous = BackupSet::open(authority, &path)?;+ let archived =+ path.with_file_name(format!("observation-format-5.{}", previous.set_id()));+ if archived.exists() {+ return Err(LocalStorageError::InvalidData);+ }+ crate::backup::source_snapshot::archive_guide(&path, &archived, previous.set_id())?;+ durable::rename(&path, &archived)?;+ }+ owned = if BackupSet::durable_manifest_present(&path)? {+ BackupSet::open(authority, &path)?+ } else {+ let binding = binding.ok_or(LocalStorageError::Locked)?;+ let live = root.join("live");+ let control = open_existing_for_snapshot(&live.join("control.db"))?;+ let runtime = open_existing_for_snapshot(&live.join("runtime.db"))?;+ let secrets = open_existing_for_snapshot(&live.join("secrets.db"))?;+ BackupSet::create_migration(+ barrier,+ &path,+ &control,+ &runtime,+ &secrets,+ &binding.key_id,+ LATEST_SCHEMA_VERSION,+ )?+ };+ &owned+ } else {+ return Ok(());+ };+ if options.preserve_observation_source {+ let name = set+ .directory()+ .file_name()+ .and_then(|s| s.to_str())+ .ok_or(LocalStorageError::InvalidData)?;+ let body=serde_json::to_vec(&serde_json::json!({"schema":1,"backup_name":name,"set_id":set.set_id(),"recovery_package":options.recovery_package})).map_err(|_|LocalStorageError::InvalidData)?;+ durable::write_private_new(&link, &body)?;+ durable::directory_barrier(root)?;+ }+ if let Some(progress) = &options.upgrade_progress {+ let _ = progress.send(hiroute_host_runtime::StorageUpgradePhase::Backup);+ }+ set.publish_coordinated_source(barrier, root, &options)?;+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn observation_only_startup_preserves_complete_group_options_and_matching_package_once() {+ let directory = crate::test_tempdir().unwrap();+ let root = directory.path().join("storage");+ drop(crate::LocalStorageSet::open_for_daemon_startup(&root).unwrap());+ let observation = root.join("observation");+ crate::prepare_owner_directory(&observation).unwrap();+ let db = Connection::open(observation.join("activity.db")).unwrap();+ db.execute_batch("CREATE TABLE observation_meta(key TEXT PRIMARY KEY,value TEXT); INSERT INTO observation_meta VALUES('schema_version','4'); CREATE TABLE retained(id INTEGER PRIMARY KEY,body BLOB); INSERT INTO retained VALUES(17,X'00ff616263');").unwrap();+ drop(db);+ fs::write(observation.join("body"), b"original content").unwrap();+ let lkg = directory.path().join("custom-gateway.lkg");+ fs::write(&lkg, b"custom recovery input").unwrap();+ let package = directory.path().join("old-package");+ fs::create_dir(&package).unwrap();+ fs::write(package.join("old-daemon"), b"exact original binary fixture").unwrap();+ let options = StorageStartupOptions {+ preserve_observation_source: true,+ gateway_lkg: Some(lkg),+ recovery_package: Some(package),+ ..Default::default()+ };+ drop(+ crate::LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).unwrap(),+ );+ // Fast upgrade moves the original tree; a recursive backup plus a live+ // old directory would still impose history-sized startup work.+ assert!(!observation.exists());+ let link: serde_json::Value =+ serde_json::from_slice(&fs::read(root.join("observation-source-backup.json")).unwrap())+ .unwrap();+ let backup = crate::upgrade_backup_root(&root)+ .unwrap()+ .parent()+ .unwrap()+ .join(link["backup_name"].as_str().unwrap());+ crate::backup::source_snapshot::verify(&backup).unwrap();+ assert_eq!(+ fs::read(backup.join("source/storage/observation/body")).unwrap(),+ b"original content"+ );+ assert_eq!(+ fs::read(backup.join("source/gateway.lkg")).unwrap(),+ b"custom recovery input"+ );+ assert_eq!(+ fs::read(backup.join("source/recovery-package/old-daemon")).unwrap(),+ b"exact original binary fixture"+ );+ let before = fs::read(backup.join("source/source-backup.json")).unwrap();+ drop(+ crate::LocalStorageSet::open_for_daemon_startup_with_options(&root, &options).unwrap(),+ );+ assert_eq!(+ before,+ fs::read(backup.join("source/source-backup.json")).unwrap()+ );+ // Recovery is the complete old group, not a replacement of activity.db+ // alone. Core stores, key, old Observation and old package stay paired.+ for file in [+ "live/control.db",+ "live/runtime.db",+ "live/secrets.db",+ "master-key",+ "observation/activity.db",+ ] {+ assert!(backup.join("source/storage").join(file).exists());+ }+ }+}++#[cfg(test)]+#[test]+fn missing_old_package_refuses_observation_sealing_before_mutation() {+ let directory = crate::test_tempdir().unwrap();+ let root = directory.path().join("storage");+ drop(crate::LocalStorageSet::open_for_daemon_startup(&root).unwrap());+ let path = root.join("live/control.db");+ let before = fs::read(&path).unwrap();+ let error = crate::LocalStorageSet::open_for_daemon_startup_with_options(+ &root,+ &StorageStartupOptions {+ preserve_observation_source: true,+ ..Default::default()+ },+ )+ .err()+ .unwrap();+ assert!(error.to_string().contains("HIROUTE_RECOVERY_PACKAGE"));+ assert_eq!(before, fs::read(&path).unwrap());+ assert!(!root.join("observation-source-backup.json").exists());+}++#[cfg(test)]+#[cfg(unix)]+#[test]+fn recovery_space_reservation_stops_before_sealing_observation() {+ let directory = crate::test_tempdir().unwrap();+ let root = directory.path().join("storage");+ drop(crate::LocalStorageSet::open_for_daemon_startup(&root).unwrap());+ let original = fs::read(root.join("live/control.db")).unwrap();+ let package = directory.path().join("old-package");+ fs::create_dir(&package).unwrap();+ fs::write(package.join("daemon"), b"retained old package").unwrap();+ let pad = fs::File::create(package.join("sparse-recovery-input")).unwrap();+ pad.set_len(+ fs2::available_space(&root)+ .unwrap()+ .saturating_add(512 * 1024 * 1024),+ )+ .unwrap();+ let observation = root.join("observation");+ fs::create_dir(&observation).unwrap();+ fs::write(observation.join("body"), b"do not move on failure").unwrap();+ let error = crate::LocalStorageSet::open_for_daemon_startup_with_options(+ &root,+ &StorageStartupOptions {+ preserve_observation_source: true,+ recovery_package: Some(package),+ ..Default::default()+ },+ )+ .err()+ .unwrap();+ assert!(error.to_string().contains("Insufficient space"));+ assert_eq!(fs::read(root.join("live/control.db")).unwrap(), original);+ assert_eq!(+ fs::read(observation.join("body")).unwrap(),+ b"do not move on failure"+ );+ let backup = crate::upgrade_backup_root(&root)+ .unwrap()+ .parent()+ .unwrap()+ .join("observation-format-5");+ assert!(!backup.join("source-staging").exists());+ assert!(!backup.join("source").exists());+}diff --git a/crates/local-storage/src/migrations/recovery_package.rs b/crates/local-storage/src/migrations/recovery_package.rsnew file mode 100644index 00000000..b7de9d90--- /dev/null+++ b/crates/local-storage/src/migrations/recovery_package.rs@@ -0,0 +1,77 @@+//! Reuse the desktop installer's existing retained previous-App handoff. A+//! headless owner can explicitly supply the old package; missing proof stops an+//! incompatible Observation upgrade before either store family is changed.+use super::*;+use hiroute_host_runtime::{DesktopPackageIdentity, read_desktop_package_identity};+fn identity(+ value: &serde_json::Value,+ path: &Path,+) -> Result<DesktopPackageIdentity, LocalStorageError> {+ let mut value: DesktopPackageIdentity =+ serde_json::from_value(value.clone()).map_err(|_| LocalStorageError::InvalidData)?;+ value.app_path = path.to_path_buf();+ Ok(value)+}+pub(super) fn previous_desktop(storage: &Path) -> Result<Option<PathBuf>, LocalStorageError> {+ let executable = std::env::current_exe()?;+ let Some(current) = executable+ .ancestors()+ .find(|p| p.extension().is_some_and(|ext| ext == "app"))+ else {+ return Ok(None);+ };+ let data = storage.parent().ok_or(LocalStorageError::InvalidData)?;+ let directory = data.join("updates");+ if !directory.exists() {+ return Ok(None);+ }+ let actual =+ read_desktop_package_identity(current).map_err(|_| LocalStorageError::InvalidData)?;+ for (i, entry) in fs::read_dir(directory)?.enumerate() {+ if i >= 64 {+ return Err(LocalStorageError::InvalidData);+ }+ let entry = entry?;+ if !entry.file_name().to_string_lossy().starts_with("install-") {+ continue;+ }+ let path = entry.path().join("handoff.json");+ if !path.exists() || !entry.path().join("installed.json").exists() {+ continue;+ }+ validate_owner_file(&path)?;+ if fs::metadata(&path)?.len() > 1024 * 1024 {+ return Err(LocalStorageError::InvalidData);+ }+ let record: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)+ .map_err(|_| LocalStorageError::InvalidData)?;+ if record["schema"] != "hiroute.desktop-upgrade-handoff/v1"+ || record["data_root"].as_str() != data.to_str()+ || record["current"].as_str() != current.to_str()+ {+ continue;+ }+ if identity(&record["new_identity"], current)? != actual {+ continue;+ }+ let previous = PathBuf::from(+ record["previous"]+ .as_str()+ .ok_or(LocalStorageError::InvalidData)?,+ );+ if previous.parent() != current.parent()+ || !previous+ .file_name()+ .is_some_and(|n| n.to_string_lossy().starts_with(".HiRoute-previous-"))+ {+ return Err(LocalStorageError::InvalidData);+ }+ let old =+ read_desktop_package_identity(&previous).map_err(|_| LocalStorageError::InvalidData)?;+ if identity(&record["old_identity"], &previous)? != old {+ return Err(LocalStorageError::InvalidData);+ }+ return Ok(Some(previous));+ }+ Ok(None)+}diff --git a/crates/observation/Cargo.toml b/crates/observation/Cargo.tomlindex 57cbfee5..24ec892c 100644--- a/crates/observation/Cargo.toml+++ b/crates/observation/Cargo.toml@@ -6,6 +6,7 @@ license.workspace = truerust-version.workspace = true[dependencies]+getrandom.workspace = truehmac.workspace = truehiroute-domain = { path = "../domain" }parking_lot.workspace = true@@ -15,6 +16,9 @@ serde_json.workspace = truesha2.workspace = truethiserror.workspace = truezeroize.workspace = true+zstd.workspace = true+hiroute-platform = { path = "../platform" }+fs2.workspace = true[dev-dependencies]rusqlite = { workspace = true, features = ["trace"] }diff --git a/crates/observation/README.md b/crates/observation/README.mdindex 953bea7e..02037efe 100644--- a/crates/observation/README.md+++ b/crates/observation/README.md@@ -18,3 +18,52 @@ deletion and retention expectations are covered by this crate's tests and the prscenarios under `e2e/product`.Focused package checks use `cargo test -p hiroute-observation`.++## Compact storage and capacity retention++The current physical layout is format 5. Only the three repeated protocol+carriers share a stream descriptor; content metadata and frozen feedback use+independent checksummed level-1 zstd records with an identity fallback. Content+objects keep their existing digest and file representation. No segment-file+format or small-object database is introduced.++`store/upgrade` admits only fresh/current writers and classifies supported old+formats for sealing. Production startup preserves the complete original group,+keys and matching old package, seals the old Observation directory by a+same-filesystem rename, then initializes a fresh format-5 generation privately+before publishing it. It does not scan, copy, hash or convert old history.+Original DB/WAL/SHM, object and staging files remain together and unchanged;+current history/statistics begin at cutover. Current-format data and policy are+never reset on ordinary restart. Interrupted namespace publication uses the same+original recovery batch; unknown partial conversion generations remain preserved+with actionable whole-group recovery. See+[release recovery](../../docs/release-contracts.md#upgrade-and-manual-recovery).++`store/capacity` uses the existing maintenance owner. Its optional positive byte+limit is installation-wide and defaults to 20 GB (20,000,000,000 bytes).+An explicitly saved disable is retained across restart; seven-day age retention+continues independently. Accounting counts allocated active+Observation DB/WAL/SHM and content files, excluding protected source generations+and recovery packages. Initial file inventory is incremental and then maintained+by owned file mutations. Unix accounting reads path metadata without opening live+SQLite files, preserving their process locks. Checks are at least 10 seconds apart from completion,+with a cooperative 100 ms SQL/filesystem budget and backoff after failed or+ineffective work. SQLite checkpoint/fsync can overrun that cooperative budget;+the next wait is also at least 100 times the actual elapsed pass cost, so a slow+unit cannot turn into continuous CPU work. Oldest eligible completed requests+retire toward 90% of the limit. Durable request/blob progress, shared-reference+rechecks and incremental page release keep logical expiry separate from actual+physical release. Busy,+unreclaimable or protected data may keep usage above this soft threshold.+Page release consumes each bounded incremental-vacuum result and yields to pinned+readers before selecting more requests. Request cleanup starts from existing+request indices so unrelated workspace history does not consume its SQL budget.+Turn cleanup has a workspace/turn request lookup index, including on current-store+reopen. A large WAL still receives a nonwaiting checkpoint attempt: its physical+high-water mark is not evidence of remaining frames or a reason to block forever.+An initial session-list read retries one visibility change within its original+query deadline. Existing pagination cursors still report stale visibility.++The ignored `store::field_tests` acceptance entry accepts only an explicitly+marked private stopped copy. It exercises current-format open/reopen and real+maintenance cadence; it never authorizes mutation of a live source.diff --git a/crates/observation/src/content/lifecycle.rs b/crates/observation/src/content/lifecycle.rsindex 16450faa..7ec89681 100644--- a/crates/observation/src/content/lifecycle.rs+++ b/crates/observation/src/content/lifecycle.rs@@ -234,6 +234,7 @@ fn apply_append(}Err(_) => return Err(ContentProjectionError::ContentStorage),}+ store.note_object_path(&path);transaction.execute("INSERT INTO content_chunks_v2@@ -244,7 +245,9 @@ fn apply_append(envelope.correlation.workspace_id.as_str(),content_id.as_str(),i64::from(rejected),- path.to_string_lossy(),+ store+ .object_locator(&path)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,prior_byte_count,i64_from_u64(bytes.len() as u64)?,envelope.event_id.as_str(),diff --git a/crates/observation/src/content/storage.rs b/crates/observation/src/content/storage.rsindex 5d890e10..1d52a0c0 100644--- a/crates/observation/src/content/storage.rs+++ b/crates/observation/src/content/storage.rs@@ -16,7 +16,7 @@ use super::lifecycle::ContentProjectionError;type StoredContentEventReplay = (String,- String,+ Vec<u8>,Option<String>,Option<i64>,Option<i64>,@@ -35,7 +35,7 @@ pub(crate) fn verify_content_replay("SELECT event_id, metadata_json, canonical_chunk_object_ref,canonical_chunk_byte_offset, canonical_chunk_byte_count, envelope_digestFROM conversation_content_events_v2- WHERE producer_id=?1 AND producer_epoch=?2 AND stream_id=?3 AND sequence=?4",+ WHERE stream_ref=(SELECT stream_ref FROM observation_streams_v5 WHERE channel='conversation_content' AND producer_id=?1 AND producer_epoch=?2 AND stream_id=?3) AND sequence=?4",params![envelope.producer.stream.producer_id.as_str(),envelope.producer.stream.producer_epoch.as_str(),@@ -65,8 +65,11 @@ pub(crate) fn verify_content_replay(.as_object_mut().ok_or(ContentProjectionError::ContentCorrupt)?.remove("canonical_bytes_base64");- let stored_metadata: serde_json::Value =- serde_json::from_str(&metadata).map_err(|_| ContentProjectionError::ContentCorrupt)?;+ let stored_metadata: serde_json::Value = serde_json::from_slice(+ &crate::store::records::decode(&metadata)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,+ )+ .map_err(|_| ContentProjectionError::ContentCorrupt)?;if event_id != envelope.event_id.as_str()|| stored_digest != digest.as_str()|| stored_metadata != expected_metadata@@ -127,7 +130,12 @@ pub(crate) fn verify_content_replay(|row| row.get(0),).map_err(|_| ContentProjectionError::ContentCorrupt)?;- let mut file = File::open(path).map_err(|_| ContentProjectionError::ContentStorage)?;+ let mut file = File::open(+ store+ .object_path(&path)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,+ )+ .map_err(|_| ContentProjectionError::ContentStorage)?;let object_offset = if state == "complete" {u64::try_from(offset).map_err(|_| ContentProjectionError::ContentCorrupt)?} else if state == "installing" {@@ -278,7 +286,12 @@ fn verify_acknowledged_blob(if digest != acknowledgement.digest || byte_count < 0 {return Err(ContentProjectionError::ContentCorrupt);}- let mut input = File::open(path).map_err(|_| ContentProjectionError::ContentStorage)?;+ let mut input = File::open(+ store+ .object_path(&path)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,+ )+ .map_err(|_| ContentProjectionError::ContentStorage)?;let mut accumulator = store.authority.content_accumulator(&media);let mut observed_count = 0_u64;let mut buffer = [0_u8; 16 * 1024];@@ -362,7 +375,12 @@ pub(crate) fn finalize_content(let mut accumulator = store.authority.content_accumulator(&media);let mut buffer = [0_u8; 16 * 1024];for path in paths {- let mut input = File::open(path).map_err(|_| ContentProjectionError::ContentStorage)?;+ let mut input = File::open(+ store+ .object_path(&path)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,+ )+ .map_err(|_| ContentProjectionError::ContentStorage)?;loop {let read = input.read(&mut buffer)@@ -380,6 +398,7 @@ pub(crate) fn finalize_content(.sync_all().map_err(|_| ContentProjectionError::ContentStorage)?;drop(output);+ store.note_object_path(&install_path);let (actual_digest, byte_count) = accumulator.finish();if actual_digest.as_str() != expected_digest {return Err(ContentProjectionError::DigestMismatch {@@ -413,12 +432,14 @@ pub(crate) fn finalize_content(fs::rename(&install_path, &blob_path).map_err(|_| ContentProjectionError::ContentStorage)?;}+ store.note_object_path(&install_path);+ store.note_object_path(&blob_path);transaction.execute("INSERT INTO content_blobs_v2(workspace_id, blob_digest, object_path, byte_count, media_type, state, created_at_unix_nanos)VALUES (?1, ?2, ?3, ?4, ?5, 'complete', ?6)ON CONFLICT(workspace_id, blob_digest) DO NOTHING",- params![envelope.correlation.workspace_id.as_str(), actual_digest.as_str(), blob_path.to_string_lossy(),+ params![envelope.correlation.workspace_id.as_str(), actual_digest.as_str(), store.object_locator(&blob_path).map_err(|_|ContentProjectionError::ContentCorrupt)?,i64_from_u64(byte_count)?, media, i64_from_u64(envelope.occurred_at_unix_nanos)?],).map_err(|_| ContentProjectionError::ActivityStorage)?;transaction@@ -439,7 +460,9 @@ pub(crate) fn finalize_content(params![envelope.correlation.workspace_id.as_str(),content_id,- blob_path.to_string_lossy(),+ store+ .object_locator(&blob_path)+ .map_err(|_| ContentProjectionError::ContentCorrupt)?,],).map_err(|_| ContentProjectionError::ActivityStorage)?;diff --git a/crates/observation/src/maintenance.rs b/crates/observation/src/maintenance.rsindex 97653122..f7a8fb99 100644--- a/crates/observation/src/maintenance.rs+++ b/crates/observation/src/maintenance.rs@@ -87,6 +87,15 @@ fn run(let Some(store) = store.upgrade() else {break;};+ let capacity_failed = store+ .capacity_cycle(+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .ok()+ .and_then(|t| i64::try_from(t.as_millis()).ok())+ .unwrap_or(0),+ )+ .is_err();let backfill_failed = store.backfill_inline_payloads().is_err();let settlement_failed = store.settle_pending_valuations(16).is_err();let now_ms = std::time::SystemTime::now()@@ -146,7 +155,8 @@ fn run(index.is_none()}};- if backfill_failed+ if capacity_failed+ || backfill_failed|| settlement_failed|| retention_failed|| gc_faileddiff --git a/crates/observation/src/query/content.rs b/crates/observation/src/query/content.rsindex 34cf1ac1..c61a1a72 100644--- a/crates/observation/src/query/content.rs+++ b/crates/observation/src/query/content.rs@@ -5,7 +5,7 @@ use std::io::Read;pub(super) fn messages(connection: &rusqlite::Connection,- authority: &crate::DigestAuthority,+ store: &LocalObservationStore,workspace_id: &WorkspaceId,session_id: &SessionId,turn_id: &TurnId,@@ -79,15 +79,19 @@ pub(super) fn messages(.checked_sub(4096).ok_or(ObservationQueryError::InvalidQuery)?;let mut bytes = Vec::new();- fs::File::open(&row.10)- .map_err(|_| ObservationQueryError::Unavailable)?- .take((*remaining_bytes as u64) + 1)- .read_to_end(&mut bytes)- .map_err(|_| ObservationQueryError::Unavailable)?;+ fs::File::open(+ store+ .object_path(&row.10)+ .map_err(|_| ObservationQueryError::Corrupt)?,+ )+ .map_err(|_| ObservationQueryError::Unavailable)?+ .take((*remaining_bytes as u64) + 1)+ .read_to_end(&mut bytes)+ .map_err(|_| ObservationQueryError::Unavailable)?;*remaining_bytes = remaining_bytes.checked_sub(bytes.len()).ok_or(ObservationQueryError::InvalidQuery)?;- let mut accumulator = authority.content_accumulator(&row.8);+ let mut accumulator = store.authority.content_accumulator(&row.8);accumulator.update(&bytes);if accumulator.finish().0 != blob_digest {return Err(ObservationQueryError::Corrupt);diff --git a/crates/observation/src/query/mod.rs b/crates/observation/src/query/mod.rsindex b43bb125..31393b49 100644--- a/crates/observation/src/query/mod.rs+++ b/crates/observation/src/query/mod.rs@@ -25,6 +25,18 @@ use crate::value::aggregate_entries;use filters::{matches_switch, validate_list_query};impl ObservationQueryPort for LocalObservationStore {+ fn observation_capacity(+ &self,+ ) -> Result<hiroute_domain::ObservationCapacityStatusV1, ObservationQueryError> {+ self.capacity_status()+ }+ fn set_observation_capacity(+ &self,+ limit: Option<u64>,+ ) -> Result<hiroute_domain::ObservationCapacityStatusV1, ObservationQueryError> {+ self.set_capacity_limit(limit)+ }+fn observation_analysis(&self,reader: &hiroute_domain::ObservationReaderContext,@@ -373,13 +385,7 @@ impl ObservationQueryPort for LocalObservationStore {model_switch,self.retention.detail_retention_ms,)?;- let turns = read_turns(- &connection,- &self.authority,- workspace_id,- session_id,- content_mode,- )?;+ let turns = read_turns(&connection, self, workspace_id, session_id, content_mode)?;if query_visibility(&connection)? != visibility {return Err(ObservationQueryError::StalePreview);}@@ -586,7 +592,7 @@ fn session_summary(fn read_turns(connection: &rusqlite::Connection,- authority: &crate::DigestAuthority,+ store: &LocalObservationStore,workspace_id: &WorkspaceId,session_id: &SessionId,content_mode: ContentMode,@@ -624,7 +630,7 @@ fn read_turns(} else {content::messages(connection,- authority,+ store,workspace_id,session_id,&turn_id,diff --git a/crates/observation/src/query_v2/search_tests.rs b/crates/observation/src/query_v2/search_tests.rsindex 5e798568..6109b2b2 100644--- a/crates/observation/src/query_v2/search_tests.rs+++ b/crates/observation/src/query_v2/search_tests.rs@@ -8,7 +8,7 @@ pub(crate) fn source(store: &LocalObservationStore, text: &[u8]) -> String {.authority.content_blob_digest("text/plain", text).to_string();- let path = store.activity_path.with_file_name("indexed-source");+ let path = store.blob_path(&digest);std::fs::write(&path, text).unwrap();let connection = store.connection.lock();connection.execute("INSERT INTO sessions(workspace_id,session_id,started_at_ms,updated_at_ms) VALUES(?1,'session',100,100)",[WorkspaceId::DEFAULT]).unwrap();@@ -162,12 +162,8 @@ fn deletion_erases_derived_text_and_invalidates_pending_search_pages() {fn digest_corruption_never_publishes_rebuilt_text() {let root = tempfile::tempdir().unwrap();let store = LocalObservationStore::open(root.path(), DigestAuthority::new([2; 32])).unwrap();- source(&store, b"original");- std::fs::write(- store.activity_path.with_file_name("indexed-source"),- b"needle!!",- )- .unwrap();+ let digest = source(&store, b"original");+ std::fs::write(store.blob_path(&digest), b"needle!!").unwrap();index(&store);let page = store.search_observed_text(&reader(true), &query("needle", 10), 500)diff --git a/crates/observation/src/query_v2/session_search_tests.rs b/crates/observation/src/query_v2/session_search_tests.rsindex 90db4e18..2758330e 100644--- a/crates/observation/src/query_v2/session_search_tests.rs+++ b/crates/observation/src/query_v2/session_search_tests.rs@@ -277,9 +277,7 @@ fn add(store: &LocalObservationStore, session: &str, request: &str, at: i64, tex.authority.content_blob_digest("text/plain", text.as_bytes()).to_string();- let path = store- .activity_path- .with_file_name(format!("source-{request}"));+ let path = store.blob_path(&digest);std::fs::write(&path, text).unwrap();let db = store.connection.lock();db.execute("INSERT OR IGNORE INTO sessions(workspace_id,session_id,started_at_ms,updated_at_ms,agent_id) VALUES(?1,?2,?3,?3,'test-agent')",params![WorkspaceId::DEFAULT,session,at]).unwrap();diff --git a/crates/observation/src/query_v2/sessions.rs b/crates/observation/src/query_v2/sessions.rsindex fc097ed9..7728f861 100644--- a/crates/observation/src/query_v2/sessions.rs+++ b/crates/observation/src/query_v2/sessions.rs@@ -26,6 +26,28 @@ impl crate::LocalObservationStore {reader: &ObservationReaderContext,query: &ObservationRequestQuery,now_ms: i64,+ ) -> Result<ObservationSessionPageV2, ObservationV2Error> {+ if query.cursor.is_some() {+ return self.observed_sessions_once(reader, query, now_ms);+ }+ let deadline = reader+ .query_deadline()+ .unwrap_or_else(|| std::time::Instant::now() + std::time::Duration::from_millis(500));+ let reader = reader.clone().with_query_deadline(deadline);+ match self.observed_sessions_once(&reader, query, now_ms) {+ // Automatic retention can invalidate a snapshot while this first+ // page is being read. Retry once under the SAME original deadline;+ // never return removed membership or silently reset a paged cursor.+ Err(ObservationV2Error::Stale) => self.observed_sessions_once(&reader, query, now_ms),+ result => result,+ }+ }++ fn observed_sessions_once(+ &self,+ reader: &ObservationReaderContext,+ query: &ObservationRequestQuery,+ now_ms: i64,) -> Result<ObservationSessionPageV2, ObservationV2Error> {reader.check(now_ms, false, false)?;if query.from_ms < 0@@ -190,6 +212,10 @@ impl crate::LocalObservationStore {None};tx.commit()?;+ #[cfg(test)]+ if let Some(hook) = self.session_read_test_hook.lock().take() {+ hook(self);+ }check_visibility(&connection, visibility)?;Ok(ObservationSessionPageV2 {sessions,diff --git a/crates/observation/src/query_v2/sessions_tests.rs b/crates/observation/src/query_v2/sessions_tests.rsindex 0080ad10..e5dc5982 100644--- a/crates/observation/src/query_v2/sessions_tests.rs+++ b/crates/observation/src/query_v2/sessions_tests.rs@@ -3,6 +3,72 @@ use super::*;use hiroute_domain::{ContentCompleteness, FactsCompleteness, ObservationQueryPort};use rusqlite::params;+fn retire_during_session_read(store: &crate::LocalObservationStore) {+ let mut db = store.connection.lock();+ let transaction = db.transaction().unwrap();+ transaction+ .execute("DELETE FROM logical_requests WHERE request_id='first'", [])+ .unwrap();+ invalidate_visibility(&transaction).unwrap();+ transaction.commit().unwrap();+}++#[test]+fn a_fresh_session_page_restarts_after_retirement_without_returning_expired_membership() {+ let directory = tempfile::tempdir().unwrap();+ let store = open(directory.path());+ seed(&store, "first", 100);+ seed(&store, "second", 200);+ *store.session_read_test_hook.lock() = Some(Box::new(retire_during_session_read));+ let page = store+ .observed_sessions(&reader(None), &query(50), 500)+ .unwrap();+ assert_eq!(page.sessions.len(), 1);+ assert_eq!(page.sessions[0].request_count, 1);+}++#[test]+fn a_paged_session_read_stays_stale_after_retirement_and_keeps_its_cursor_contract() {+ let directory = tempfile::tempdir().unwrap();+ let store = open(directory.path());+ seed(&store, "first", 100);+ seed(&store, "second", 200);+ {+ let db = store.connection.lock();+ db.execute("INSERT INTO sessions(workspace_id,session_id,started_at_ms,updated_at_ms) VALUES(?1,'other-session',200,200)", [WorkspaceId::DEFAULT]).unwrap();+ db.execute(+ "UPDATE logical_requests SET session_id='other-session' WHERE request_id='second'",+ [],+ )+ .unwrap();+ }+ let first = store+ .observed_sessions(&reader(None), &query(1), 500)+ .unwrap();+ assert!(first.next_cursor.is_some());+ *store.session_read_test_hook.lock() = Some(Box::new(retire_during_session_read));+ let next = ObservationRequestQuery {+ cursor: first.next_cursor,+ ..query(1)+ };+ assert!(matches!(+ store.observed_sessions(&reader(None), &next, 500),+ Err(ObservationV2Error::Stale)+ ));+}++#[test]+fn a_fresh_session_page_never_extends_an_existing_query_deadline() {+ let directory = tempfile::tempdir().unwrap();+ let store = open(directory.path());+ seed(&store, "first", 100);+ let scope = reader(None).with_query_deadline(std::time::Instant::now());+ assert!(matches!(+ store.observed_sessions(&scope, &query(50), 500),+ Err(ObservationV2Error::Timeout)+ ));+}+#[test]fn session_content_completeness_covers_all_visible_requests_and_recovers() {let root = tempfile::tempdir().unwrap();diff --git a/crates/observation/src/store/capacity.rs b/crates/observation/src/store/capacity.rsnew file mode 100644index 00000000..030c7f35--- /dev/null+++ b/crates/observation/src/store/capacity.rs@@ -0,0 +1,670 @@+//! Optional installation-wide early retention. One completion-based schedule owns+//! selection, resumable row retirement, orphan objects and SQLite page release.+use super::LocalObservationStore;+#[cfg(test)]+pub(super) mod probe;+#[cfg(test)]+mod tests;+use hiroute_domain::{ObservationCapacityStatusV1 as Status, ObservationQueryError as Error};+use rusqlite::{Connection, OptionalExtension, params};+use std::{+ collections::{BTreeMap, VecDeque},+ fs,+ path::PathBuf,+ time::{Duration, Instant},+};++pub(crate) const INTERVAL: Duration = Duration::from_secs(10);+const DEFAULT_LIMIT: u64 = 20_000_000_000;+const BUDGET: Duration = Duration::from_millis(100);+const ROWS: usize = 128;++pub(crate) const SCHEMA: &str = r#"+CREATE INDEX IF NOT EXISTS observation_capacity_oldest_v5 ON logical_requests(started_at_ms,workspace_id,request_id) WHERE finished_at_ms IS NOT NULL;+CREATE INDEX IF NOT EXISTS observation_capacity_turn_v5 ON logical_requests(workspace_id,turn_id);+CREATE TABLE IF NOT EXISTS observation_capacity_pending_v5(workspace TEXT NOT NULL,request TEXT NOT NULL,session TEXT NOT NULL,step INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(workspace,request));+CREATE TABLE IF NOT EXISTS observation_capacity_blobs_v5(workspace TEXT NOT NULL,digest TEXT NOT NULL,PRIMARY KEY(workspace,digest));+CREATE INDEX IF NOT EXISTS observation_capacity_quality_assessment_v5 ON plan_quality_segments(workspace_id,assessment_trigger_request_id);+CREATE INDEX IF NOT EXISTS observation_capacity_quality_last_v5 ON plan_quality_segments(workspace_id,last_request_id);+CREATE INDEX IF NOT EXISTS observation_capacity_quality_first_v5 ON plan_quality_segments(workspace_id,first_request_id);+CREATE INDEX IF NOT EXISTS content_instances_v2_live_blob_global ON content_instances_v2(content_blob_digest) WHERE state='complete';+"#;++pub(crate) struct State {+ pub next_due: Instant,+ failures: u32,+ inventory: Option<Inventory>,+ pub status: Status,+ draining: bool,+ committed_progress: bool,+}+impl Default for State {+ fn default() -> Self {+ Self {+ next_due: Instant::now(),+ failures: 0,+ inventory: None,+ status: Status {+ state: "idle".into(),+ reason: "disabled".into(),+ ..Default::default()+ },+ draining: false,+ committed_progress: false,+ }+ }+}++struct Inventory {+ pending: VecDeque<PathBuf>,+ entries: Option<fs::ReadDir>,+ bytes: u64,+ files: BTreeMap<PathBuf, u64>,+ complete: bool,+}+impl Inventory {+ fn new(store: &LocalObservationStore) -> Self {+ Self {+ pending: VecDeque::from([store.blob_root.clone(), store.staging_root.clone()]),+ entries: None,+ bytes: 0,+ files: BTreeMap::new(),+ complete: false,+ }+ }+ fn advance(&mut self, deadline: Instant) -> Result<bool, Error> {+ if self.complete {+ return Ok(true);+ }+ let mut visited = 0;+ while visited < 16384 && Instant::now() < deadline {+ if let Some(entries) = &mut self.entries {+ match entries.next() {+ Some(Ok(entry)) => {+ let path = entry.path();+ let meta = fs::symlink_metadata(&path).map_err(|_| Error::Unavailable)?;+ if meta.file_type().is_symlink() {+ return Err(Error::Corrupt);+ }+ if meta.is_dir() {+ self.pending.push_back(path);+ } else if meta.is_file() {+ #[cfg(unix)]+ let bytes = {+ use std::os::unix::fs::MetadataExt;+ meta.blocks().checked_mul(512).ok_or(Error::Corrupt)?+ };+ #[cfg(not(unix))]+ let bytes = allocated(&path)?;+ self.record(path, bytes)?;+ }+ visited += 1;+ }+ Some(Err(_)) => return Err(Error::Unavailable),+ None => self.entries = None,+ }+ } else if let Some(path) = self.pending.pop_front() {+ self.entries = Some(fs::read_dir(path).map_err(|_| Error::Unavailable)?);+ } else {+ self.complete = true;+ return Ok(true);+ }+ }+ Ok(false)+ }+ fn record(&mut self, path: PathBuf, bytes: u64) -> Result<(), Error> {+ let prior = if bytes == 0 {+ self.files.remove(&path).unwrap_or(0)+ } else {+ self.files.insert(path, bytes).unwrap_or(0)+ };+ self.bytes = self+ .bytes+ .checked_sub(prior)+ .and_then(|n| n.checked_add(bytes))+ .ok_or(Error::Corrupt)?;+ Ok(())+ }+}+#[cfg(unix)]+fn allocated(path: &std::path::Path) -> Result<u64, Error> {+ use std::os::unix::fs::MetadataExt;+ // Opening and closing a separate descriptor cancels this process's POSIX+ // locks on a live SQLite DB/SHM file. Path metadata needs no descriptor.+ match fs::symlink_metadata(path) {+ Ok(meta) if meta.is_file() => meta.blocks().checked_mul(512).ok_or(Error::Corrupt),+ Ok(_) => Err(Error::Corrupt),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(0),+ Err(_) => Err(Error::Unavailable),+ }+}+#[cfg(not(unix))]+fn allocated(path: &std::path::Path) -> Result<u64, Error> {+ match fs::File::open(path) {+ Ok(file) => fs2::FileExt::allocated_size(&file).map_err(|_| Error::Unavailable),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(0),+ Err(_) => Err(Error::Unavailable),+ }+}+fn db_bytes(store: &LocalObservationStore) -> Result<u64, Error> {+ let mut n = 0u64;+ for suffix in ["", "-wal", "-shm"] {+ n = n+ .checked_add(allocated(&PathBuf::from(format!(+ "{}{suffix}",+ store.activity_path.display()+ )))?)+ .ok_or(Error::Corrupt)?;+ }+ Ok(n)+}+fn get_limit(db: &Connection) -> Result<Option<u64>, Error> {+ let text: Option<String> = db+ .query_row(+ "SELECT value FROM observation_meta WHERE key='max_active_bytes'",+ [],+ |r| r.get(0),+ )+ .optional()+ .map_err(|_| Error::Unavailable)?;+ match text.as_deref() {+ None => Ok(Some(DEFAULT_LIMIT)),+ Some("null") => Ok(None),+ Some(text) => text+ .parse::<u64>()+ .ok()+ .filter(|n| *n > 0)+ .map(Some)+ .ok_or(Error::Corrupt),+ }+}+fn sql<T>(r: rusqlite::Result<T>) -> Result<T, Error> {+ r.map_err(|_| Error::Unavailable)+}++fn vacuum_pages(db: &Connection) -> Result<bool, Error> {+ let mut statement = sql(db.prepare("PRAGMA incremental_vacuum(16)"))?;+ let mut rows = sql(statement.query([]))?;+ let mut released = false;+ // This pragma returns one row per released page. execute_batch only steps+ // its first row, silently reducing a bounded batch to a single page.+ while sql(rows.next())?.is_some() {+ released = true;+ }+ Ok(released)+}++fn checkpoint_pages(db: &Connection, deadline: Instant) -> Result<bool, Error> {+ let checkpoint: (i64, i64, i64) =+ sql(db.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |r| {+ Ok((r.get(0)?, r.get(1)?, r.get(2)?))+ }))?;+ if checkpoint.0 != 0 || checkpoint.1 != checkpoint.2 || Instant::now() >= deadline {+ return Ok(false);+ }+ let busy: i64 = sql(db.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0)))?;+ Ok(busy == 0)+}++impl LocalObservationStore {+ pub(crate) fn note_object_path(&self, path: &std::path::Path) {+ let mut state = self.capacity.lock();+ if let Some(inventory) = &mut state.inventory {+ match allocated(path) {+ Ok(bytes) => {+ let _ = inventory.record(path.to_owned(), bytes);+ }+ Err(_) => state.inventory = None,+ }+ }+ }+ pub(crate) fn note_removed_directory(&self, path: &std::path::Path) {+ let mut state = self.capacity.lock();+ if let Some(inventory) = &mut state.inventory {+ while let Some(key) = inventory+ .files+ .range(path.to_path_buf()..)+ .next()+ .map(|(k, _)| k.clone())+ .filter(|k| k.starts_with(path))+ {+ let _ = inventory.record(key, 0);+ }+ }+ }++ pub fn capacity_status(&self) -> Result<Status, Error> {+ let limit = get_limit(&self.connection.lock())?;+ let mut value = self.capacity.lock().status.clone();+ value.max_active_bytes = limit;+ if limit.is_some() && value.reason == "disabled" {+ value.state = "pending".into();+ value.reason = "measuring".into();+ }+ Ok(value)+ }+ pub fn set_capacity_limit(&self, limit: Option<u64>) -> Result<Status, Error> {+ if limit == Some(0) {+ return Err(Error::InvalidQuery);+ }+ let mut db = self.connection.lock();+ let tx = sql(db.transaction())?;+ // Absence means the product default. A deliberate disable must remain+ // a stored choice, including when this database is reopened.+ let stored = limit+ .map(|n| n.to_string())+ .unwrap_or_else(|| "null".into());+ sql(tx.execute("INSERT INTO observation_meta(key,value) VALUES('max_active_bytes',?1) ON CONFLICT(key) DO UPDATE SET value=excluded.value",[stored]))?;+ sql(tx.commit())?;+ drop(db);+ let mut state = self.capacity.lock();+ state.next_due = Instant::now();+ state.status.max_active_bytes = limit;+ state.status.state = if limit.is_some() { "pending" } else { "idle" }.into();+ state.status.reason = if limit.is_some() {+ "measuring"+ } else {+ "disabled"+ }+ .into();+ state.draining = false;+ if limit.is_none() {+ state.inventory = None;+ }+ Ok(state.status.clone())+ }++ /// False means not due. The mutex is never waited on by maintenance.+ pub(crate) fn capacity_cycle(&self, now_ms: i64) -> Result<bool, Error> {+ let Some(mut state) = self.capacity.try_lock() else {+ return Ok(false);+ };+ if Instant::now() < state.next_due {+ return Ok(false);+ }+ let started = Instant::now();+ let deadline = started + BUDGET;+ let Some(mut db) = self.connection.try_lock() else {+ state.next_due = Instant::now() + INTERVAL;+ state.status.state = "pending".into();+ state.status.reason = "writer_busy".into();+ return Ok(true);+ };+ #[cfg(test)]+ let probe = probe::begin();+ let prior_timeout: i64 = sql(db.query_row("PRAGMA busy_timeout", [], |r| r.get(0)))?;+ sql(db.busy_timeout(Duration::ZERO))?;+ db.progress_handler(1000, Some(move || Instant::now() >= deadline));+ state.committed_progress = false;+ let mut result = self.capacity_work(&mut db, &mut state, now_ms, deadline);+ db.progress_handler(0, None::<fn() -> bool>);+ sql(db.busy_timeout(Duration::from_millis(prior_timeout.max(0) as u64)))?;+ if result.is_err() && Instant::now() >= deadline && state.committed_progress {+ // Earlier bounded transactions remain committed when the next unit+ // reaches the deadline. Yield normally rather than backing off as+ // though the whole pass had made no progress.+ result = Ok(true);+ state.status.state = "pending".into();+ state.status.reason = "work_budget".into();+ }+ match &result {+ Ok(progress) => {+ state.failures = if *progress {+ 0+ } else {+ state.failures.saturating_add(1).min(4)+ };+ }+ Err(_) => {+ state.failures = state.failures.saturating_add(1).min(4);+ state.status.state = "blocked".into();+ state.status.reason = if Instant::now() >= deadline {+ "work_budget"+ } else {+ "storage_unavailable"+ }+ .into();+ }+ }+ let completed = Instant::now();+ // SQLite checkpoint/fsync is not preempted by SQL progress callbacks.+ // Charge its whole elapsed cost to the existing cooldown too: even an+ // overrun gets at most a 1% duty cycle, without a new thread or timer.+ let cooldown = INTERVAL+ .saturating_mul(1 << state.failures)+ .max(completed.duration_since(started).saturating_mul(100));+ state.next_due = completed + cooldown;+ #[cfg(test)]+ probe::record(probe, now_ms, &state.status.reason);+ // Budget exhaustion is a yielding pending pass, not an unbounded retry.+ if result.is_err() && Instant::now() >= deadline {+ return Ok(true);+ }+ result.map(|_| true)+ }+ fn capacity_work(+ &self,+ db: &mut Connection,+ state: &mut State,+ now_ms: i64,+ deadline: Instant,+ ) -> Result<bool, Error> {+ let mut progress = false;+ while Instant::now() < deadline {+ let made = self.capacity_step(db, state, now_ms, deadline)?;+ progress |= made;+ // Selection, cleanup and page release share this one budget. Idle+ // and blocked states stop immediately; inventory retains its file+ // batch bound. No phase incurs another full scheduling interval.+ if !made || state.status.state != "pending" || state.status.reason == "measuring" {+ break;+ }+ }+ Ok(progress)+ }++ fn capacity_step(+ &self,+ db: &mut Connection,+ state: &mut State,+ now_ms: i64,+ deadline: Instant,+ ) -> Result<bool, Error> {+ let limit = get_limit(db)?;+ state.status.max_active_bytes = limit;+ let pending:bool=sql(db.query_row("SELECT EXISTS(SELECT 1 FROM observation_capacity_pending_v5) OR EXISTS(SELECT 1 FROM observation_capacity_blobs_v5)",[],|r|r.get(0)))?;+ // Disabling cancels future selection, but finishes already committed+ // visibility retirement so no partial erase becomes stranded forever.+ if pending {+ let progress = self.capacity_cleanup(db, state, deadline)?;+ state.status.state = "pending".into();+ state.status.reason = "physical_release".into();+ return Ok(progress);+ }+ let Some(limit) = limit else {+ state.status.state = "idle".into();+ state.status.reason = "disabled".into();+ return Ok(true);+ };+ let inventory = state.inventory.get_or_insert_with(|| Inventory::new(self));+ if !inventory.advance(deadline)? {+ state.status.state = "pending".into();+ state.status.reason = "measuring".into();+ return Ok(true);+ }+ let bytes = inventory+ .bytes+ .checked_add(db_bytes(self)?)+ .ok_or(Error::Corrupt)?;+ state.status.active_bytes = Some(bytes);+ state.status.measured_at_ms = Some(now_ms);+ let goal = if state.draining {+ limit / 10 * 9 + limit % 10 * 9 / 10+ } else {+ limit+ };+ if bytes <= goal {+ state.draining = false;+ state.status.state = "idle".into();+ state.status.reason = "within_limit".into();+ return Ok(true);+ }+ state.draining = true;+ let wal = PathBuf::from(format!("{}-wal", self.activity_path.display()));+ let wal_bytes = fs::metadata(&wal).map(|m| m.len()).unwrap_or(0);+ if wal_bytes > 0 {+ // Complete physical release before retiring another request. A+ // reader-pinned WAL yields without waiting or deleting more data.+ // Physical length retains the WAL high-water mark even after all+ // frames are checkpointed, so it cannot prohibit this attempt.+ if !checkpoint_pages(db, deadline)? {+ state.status.state = "blocked".into();+ state.status.reason = "page_release_pending".into();+ return Ok(false);+ }+ state.committed_progress = true;+ state.status.state = "pending".into();+ state.status.reason = "physical_release".into();+ return Ok(true);+ }+ let free: u64 = sql(db.query_row("PRAGMA freelist_count", [], |r| r.get(0)))?;+ if free > 0 {+ // Each step releases at most 16 pages. No live VACUUM or+ // full checkpoint waits on readers; busy yields to the next due pass.+ let released = vacuum_pages(db)?;+ state.committed_progress |= released;+ state.status.state = "pending".into();+ state.status.reason = "reusable_pages".into();+ return Ok(released);+ }+ let selected:Option<(String,String,String)>=sql(db.query_row("SELECT workspace_id,session_id,request_id FROM logical_requests r WHERE finished_at_ms IS NOT NULL AND NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=r.workspace_id AND c.request_id=r.request_id AND c.state='installing') AND NOT EXISTS(SELECT 1 FROM conversation_content_streams_v2 c WHERE c.workspace_id=r.workspace_id AND c.request_id=r.request_id AND (c.state='open' OR c.active_content_id IS NOT NULL)) ORDER BY started_at_ms,workspace_id,request_id LIMIT 1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional())?;+ let Some((workspace, session, request)) = selected else {+ state.status.state = "blocked".into();+ state.status.reason = "protected_floor".into();+ return Ok(false);+ };+ self.capacity_retire(db, &workspace, &session, &request, now_ms)?;+ state.committed_progress = true;+ state.status.state = "pending".into();+ state.status.reason = "retiring_request".into();+ Ok(true)+ }+ fn capacity_retire(+ &self,+ db: &mut Connection,+ w: &str,+ s: &str,+ r: &str,+ now: i64,+ ) -> Result<(), Error> {+ use hiroute_domain::{DeletionDataClass, SessionDeletionSpecV1, SessionId, WorkspaceId};+ let tx = sql(db.transaction())?;+ sql(tx.execute("INSERT INTO observation_capacity_pending_v5(workspace,request,session) VALUES(?1,?2,?3)",params![w,r,s]))?;+ let spec = SessionDeletionSpecV1 {+ workspace_id: WorkspaceId::parse(w).map_err(|_| Error::Corrupt)?,+ session_id: SessionId::parse(s).map_err(|_| Error::Corrupt)?,+ data_class: DeletionDataClass::FactsAndContent,+ delete_rollups: false,+ };+ let archive = self+ .authority+ .content_blob_digest("retention-archive/v2", format!("{w}\0{r}").as_bytes());+ super::value_rollup::archive_request_value(+ &tx,+ &spec,+ r,+ SessionId::parse(format!("archive-{archive}")).map_err(|_| Error::Corrupt)?,+ )?;+ crate::valuation::archive_request(&tx, w, s, r).map_err(|_| Error::Unavailable)?;+ sql(tx.execute("INSERT OR IGNORE INTO observation_request_tombstones_v2(workspace_id,request_id,deleted_ms) VALUES(?1,?2,?3)",params![w,r,now]))?;+ // Every content/analysis read joins current request membership. Remove+ // that membership atomically with the archive and signed-scope barrier.+ sql(tx.execute(+ "DELETE FROM logical_requests WHERE workspace_id=?1 AND request_id=?2",+ params![w, r],+ ))?;+ sql(tx.execute(+ "DELETE FROM observation_analysis_dirty_v1 WHERE workspace_id=?1 AND request_id=?2",+ params![w, r],+ ))?;+ sql(crate::query_v2::invalidate_visibility(&tx))?;+ // Do not call increment_store_revision: its 200-request analysis drain+ // would transfer capacity work to the unrelated accelerated catch-up loop.+ sql(tx.execute(+ "UPDATE observation_meta SET value=CAST(value AS INTEGER)+1 WHERE key='store_revision'",+ [],+ ))?;+ sql(tx.commit())+ }++ fn capacity_cleanup(+ &self,+ db: &mut Connection,+ state: &mut State,+ deadline: Instant,+ ) -> Result<bool, Error> {+ let mut progress = false;+ while Instant::now() < deadline {+ let next:Option<(String,String,String,i64)>=sql(db.query_row("SELECT workspace,request,session,step FROM observation_capacity_pending_v5 ORDER BY workspace,request LIMIT 1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?))).optional())?;+ if let Some((w, r, s, step)) = next {+ let tx = sql(db.transaction())?;+ if step == 0 {+ let chunks: Vec<(i64, String)> = {+ // Keep the small selected request as the outer loop;+ // SQLite otherwise scans the whole workspace's chunks.+ let mut stmt=sql(tx.prepare("SELECT k.rowid,k.object_path FROM content_instances_v2 c INDEXED BY content_instances_v2_request CROSS JOIN content_chunks_v2 k ON c.workspace_id=k.workspace_id AND c.content_id=k.content_id WHERE c.workspace_id=?1 AND c.request_id=?2 LIMIT 128"))?;+ sql(stmt+ .query_map(params![w, r], |r| Ok((r.get(0)?, r.get(1)?)))+ .map_err(|_| Error::Unavailable)?+ .collect())?+ };+ for (id, path) in &chunks {+ let path = self.object_path(path).map_err(|_| Error::Corrupt)?;+ if path.starts_with(&self.staging_root) {+ match fs::remove_file(&path) {+ Ok(()) => {}+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(_) => return Err(Error::Unavailable),+ }+ }+ if path.starts_with(&self.staging_root)+ && let Some(inventory) = &mut state.inventory+ {+ inventory.record(path, 0)?;+ }+ sql(tx.execute("DELETE FROM content_chunks_v2 WHERE rowid=?1", [id]))?;+ }+ if chunks.is_empty() {+ sql(tx.execute("UPDATE observation_capacity_pending_v5 SET step=1 WHERE workspace=?1 AND request=?2",params![w,r]))?;+ }+ } else if step == 1 {+ let objects: Vec<(i64, Option<String>)> = {+ let mut stmt=sql(tx.prepare("SELECT rowid,content_blob_digest FROM content_instances_v2 WHERE workspace_id=?1 AND request_id=?2 LIMIT 128"))?;+ sql(stmt+ .query_map(params![w, r], |r| Ok((r.get(0)?, r.get(1)?)))+ .map_err(|_| Error::Unavailable)?+ .collect())?+ };+ for (id, digest) in &objects {+ if let Some(digest) = digest {+ sql(tx.execute(+ "INSERT OR IGNORE INTO observation_capacity_blobs_v5 VALUES(?1,?2)",+ params![w, digest],+ ))?;+ }+ sql(tx.execute("DELETE FROM content_instances_v2 WHERE rowid=?1", [id]))?;+ }+ if objects.is_empty() {+ sql(tx.execute("UPDATE observation_capacity_pending_v5 SET step=2 WHERE workspace=?1 AND request=?2",params![w,r]))?;+ }+ } else if let Some(table) = CLEANUP.get((step - 2) as usize) {+ let feedback = sql(super::records::retire_request_feedback(+ &tx, &w, &r, table, true,+ ))?;+ if feedback == 0 {+ let order = super::records::retirement_order(table)+ .map(|order| format!(" ORDER BY {order}"))+ .unwrap_or_default();+ let n=sql(tx.execute(&format!("DELETE FROM {table} WHERE rowid IN(SELECT e.rowid FROM {table} e WHERE e.workspace_id=?1 AND e.request_id=?2{order} LIMIT {ROWS})"),params![w,r]))?;+ if n == 0 {+ sql(tx.execute("UPDATE observation_capacity_pending_v5 SET step=step+1 WHERE workspace=?1 AND request=?2",params![w,r]))?;+ }+ }+ } else {+ sql(tx.execute("DELETE FROM plan_quality_segments WHERE rowid IN(SELECT rowid FROM plan_quality_segments WHERE workspace_id=?1 AND last_request_id=?2 LIMIT 128)",params![w,r]))?;+ // Quality annotations are updated only for this selected+ // request, never through a broad age cutoff.+ sql(tx.execute("UPDATE plan_quality_segments SET assessment_event_id=NULL,assessment_sequence=NULL,assessment_trigger_request_id=NULL,assessed_at_ms=NULL,target_from_turn_id=NULL,target_through_turn_id=NULL,target_from_ordinal=NULL,target_through_ordinal=NULL,score=NULL,assessment_partial=NULL,reason_present=0 WHERE rowid IN(SELECT rowid FROM plan_quality_segments WHERE workspace_id=?1 AND assessment_trigger_request_id=?2 LIMIT 128)",params![w,r]))?;+ sql(tx.execute("UPDATE plan_quality_segments SET first_request_id=NULL,history_partial=1 WHERE rowid IN(SELECT rowid FROM plan_quality_segments WHERE workspace_id=?1 AND first_request_id=?2 LIMIT 128)",params![w,r]))?;+ let remaining:bool=sql(tx.query_row("SELECT EXISTS(SELECT 1 FROM plan_quality_segments WHERE workspace_id=?1 AND (first_request_id=?2 OR last_request_id=?2 OR assessment_trigger_request_id=?2))",params![w,r],|r|r.get(0)))?;+ if !remaining {+ sql(tx.execute("UPDATE observation_meta SET value=CAST(value AS INTEGER)+1 WHERE key='plan_quality_generation'",[]))?;+ let turns = sql(tx.execute("DELETE FROM turns WHERE workspace_id=?1 AND session_id=?2 AND rowid IN(SELECT rowid FROM turns WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests r WHERE r.workspace_id=?1 AND r.turn_id=turns.turn_id) LIMIT 128)",params![w,s]))?;+ sql(tx.execute("UPDATE sessions SET content_completeness='expired',facts_completeness='unknown',tombstone_reason='retention_expired',agent_id='',correlation='unproven' WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests WHERE workspace_id=?1 AND session_id=?2)",params![w,s]))?;+ sql(tx.execute("DELETE FROM observation_analysis_dirty_v1 WHERE workspace_id=?1 AND request_id=?2",params![w,r]))?;+ if turns < ROWS {+ sql(tx.execute("DELETE FROM observation_capacity_pending_v5 WHERE workspace=?1 AND request=?2",params![w,r]))?;+ }+ }+ }+ // Capacity-caused analysis notifications never escape this budget.+ sql(tx.execute("DELETE FROM observation_analysis_dirty_v1 WHERE workspace_id=?1 AND request_id=?2",params![w,r]))?;+ sql(tx.commit())?;+ state.committed_progress = true;+ progress = true;+ continue;+ }+ let blob:Option<(String,String)>=sql(db.query_row("SELECT workspace,digest FROM observation_capacity_blobs_v5 ORDER BY workspace,digest LIMIT 1",[],|r|Ok((r.get(0)?,r.get(1)?))).optional())?;+ let Some((w, d)) = blob else {+ break;+ };+ let tx = sql(db.transaction())?;+ let live:bool=sql(tx.query_row("SELECT EXISTS(SELECT 1 FROM content_instances_v2 WHERE workspace_id=?1 AND content_blob_digest=?2 AND state='complete')",params![w,d],|r|r.get(0)))?;+ if !live {+ let n=sql(tx.execute("DELETE FROM observation_text_blocks_v2 WHERE rowid IN(SELECT rowid FROM observation_text_blocks_v2 WHERE workspace=?1 AND digest=?2 LIMIT 128)",params![w,d]))?;+ if n > 0 {+ sql(tx.commit())?;+ state.committed_progress = true;+ progress = true;+ continue;+ }+ sql(tx.execute(+ "DELETE FROM observation_text_index_v2 WHERE workspace=?1 AND digest=?2",+ params![w, d],+ ))?;+ let path:Option<String>=sql(tx.query_row("SELECT object_path FROM content_blobs_v2 WHERE workspace_id=?1 AND blob_digest=?2",params![w,d],|r|r.get(0)).optional())?;+ if let Some(path) = path {+ let path = self.object_path(&path).map_err(|_| Error::Corrupt)?;+ // Blob files use an installation-wide digest. Workspace+ // catalog retirement cannot remove another workspace's bytes.+ let shared: bool = sql(tx.query_row("SELECT EXISTS(SELECT 1 FROM content_instances_v2 WHERE content_blob_digest=?1 AND state='complete')", [&d], |r| r.get(0)))?;+ if !shared {+ match fs::remove_file(&path) {+ Ok(()) => {}+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(_) => return Err(Error::Unavailable),+ }+ if let Some(inventory) = &mut state.inventory {+ inventory.record(path, 0)?;+ }+ }+ }+ sql(tx.execute(+ "DELETE FROM content_blobs_v2 WHERE workspace_id=?1 AND blob_digest=?2",+ params![w, d],+ ))?;+ }+ sql(tx.execute(+ "DELETE FROM observation_capacity_blobs_v5 WHERE workspace=?1 AND digest=?2",+ params![w, d],+ ))?;+ sql(tx.commit())?;+ state.committed_progress = true;+ progress = true;+ }+ Ok(progress)+ }+}+const CLEANUP: &[&str] = &[+ "observation_sensitive_payloads_v2",+ "content_message_instances_v2",+ "request_content_refs_v2",+ "conversation_content_streams_v2",+ "conversation_content_events_v2",+ "transcript_roots_v2",+ "observation_run_links",+ "observation_link_events",+ "observation_attempt_models_v2",+ "observation_safe_facts_v2",+ "execution_fact_events",+ "routing_receipts",+ "attempts",+ "value_ledger_entries",+ "observation_native_roots_v1",+ "observation_analysis_attempts_v1",+ "observation_analysis_recorded_v1",+ "observation_analysis_requests_v1",+ "observation_analysis_models_v1",+ "observation_analysis_reasoning_v1",+ "observation_analysis_turns_v1",+];diff --git a/crates/observation/src/store/capacity/probe.rs b/crates/observation/src/store/capacity/probe.rsnew file mode 100644index 00000000..3f24e085--- /dev/null+++ b/crates/observation/src/store/capacity/probe.rs@@ -0,0 +1,51 @@+//! Opt-in test-only attribution; absent from production builds and ordinary tests.+use std::{+ sync::{Mutex, OnceLock},+ time::Instant,+};++#[derive(Clone, serde::Serialize)]+pub(crate) struct Cost {+ at_ms: i64,+ wall_us: u128,+ cpu_ticks: Option<u64>,+ reason: String,+}+static COSTS: OnceLock<Mutex<Vec<Cost>>> = OnceLock::new();++pub(crate) fn enable() {+ COSTS.get_or_init(|| Mutex::new(Vec::new()));+}+pub(crate) fn samples() -> Vec<Cost> {+ COSTS+ .get()+ .map(|costs| costs.lock().unwrap().clone())+ .unwrap_or_default()+}+pub(crate) fn begin() -> Option<(Instant, Option<u64>)> {+ COSTS.get().map(|_| (Instant::now(), thread_ticks()))+}+pub(crate) fn record(start: Option<(Instant, Option<u64>)>, at_ms: i64, reason: &str) {+ if let Some((wall, cpu)) = start {+ let end = thread_ticks();+ COSTS.get().unwrap().lock().unwrap().push(Cost {+ at_ms,+ wall_us: wall.elapsed().as_micros(),+ cpu_ticks: cpu.zip(end).map(|(a, b)| b.saturating_sub(a)),+ reason: reason.into(),+ });+ }+}+fn thread_ticks() -> Option<u64> {+ #[cfg(target_os = "linux")]+ {+ let stat = std::fs::read_to_string("/proc/thread-self/stat").ok()?;+ let (_, fields) = stat.rsplit_once(") ")?;+ let mut fields = fields.split_whitespace();+ let user: u64 = fields.nth(11)?.parse().ok()?;+ let system: u64 = fields.next()?.parse().ok()?;+ user.checked_add(system)+ }+ #[cfg(not(target_os = "linux"))]+ None+}diff --git a/crates/observation/src/store/capacity/tests.rs b/crates/observation/src/store/capacity/tests.rsnew file mode 100644index 00000000..aa821178--- /dev/null+++ b/crates/observation/src/store/capacity/tests.rs@@ -0,0 +1,666 @@+use super::*;+use crate::store::tests::support::*;+use hiroute_domain::{ContentMode, ObservationQueryPort};+fn due(store: &LocalObservationStore) {+ store.capacity.lock().next_due = Instant::now();+}+#[test]+fn default_twenty_gb_persists_positive_u64_and_explicit_disable() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_store(dir.path());+ assert_eq!(+ store.capacity_status().unwrap().max_active_bytes,+ Some(20_000_000_000)+ );+ assert!(store.set_capacity_limit(Some(0)).is_err());+ assert_eq!(+ store+ .set_capacity_limit(Some(u64::MAX))+ .unwrap()+ .max_active_bytes,+ Some(u64::MAX)+ );+ drop(store);+ let store = open_store(dir.path());+ assert_eq!(+ store.capacity_status().unwrap().max_active_bytes,+ Some(u64::MAX)+ );+ store.set_capacity_limit(None).unwrap();+ drop(store);+ let store = open_store(dir.path());+ assert_eq!(store.capacity_status().unwrap().max_active_bytes, None);+ assert_eq!(store.capacity_status().unwrap().reason, "disabled");+ assert_eq!(+ store+ .connection+ .lock()+ .query_row(+ "SELECT value FROM observation_meta WHERE key='max_active_bytes'",+ [],+ |row| row.get::<_, String>(0)+ )+ .unwrap(),+ "null"+ );+}++#[test]+fn page_release_consumes_the_bounded_pragma_instead_of_only_its_first_row() {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let db = store.connection.lock();+ db.execute_batch("CREATE TABLE capacity_page_fixture(payload BLOB); WITH RECURSIVE n(x) AS(VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<512) INSERT INTO capacity_page_fixture SELECT zeroblob(4096) FROM n; DELETE FROM capacity_page_fixture;").unwrap();+ let free = || {+ db.query_row("PRAGMA freelist_count", [], |row| row.get::<_, u64>(0))+ .unwrap()+ };+ let before = free();+ assert!(before > 32);+ assert!(vacuum_pages(&db).unwrap());+ assert!(+ before - free() > 1,+ "execute_batch steps a row-producing pragma only once"+ );+}++#[test]+fn physical_release_yields_to_a_pinned_reader_then_truncates_after_it_finishes() {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let db = store.connection.lock();+ db.busy_timeout(Duration::ZERO).unwrap();+ db.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)").unwrap();+ let reader = Connection::open(&store.activity_path).unwrap();+ reader.execute_batch("BEGIN").unwrap();+ reader+ .query_row("SELECT COUNT(*) FROM observation_meta", [], |row| {+ row.get::<_, i64>(0)+ })+ .unwrap();+ db.execute(+ "INSERT INTO observation_meta(key,value) VALUES('capacity-reader-fixture','1')",+ [],+ )+ .unwrap();+ let start = Instant::now();+ assert!(!checkpoint_pages(&db, start + BUDGET).unwrap());+ assert!(start.elapsed() < BUDGET);+ drop(reader);+ assert!(checkpoint_pages(&db, Instant::now() + BUDGET).unwrap());+ assert_eq!(+ fs::metadata(format!("{}-wal", store.activity_path.display()))+ .unwrap()+ .len(),+ 0+ );+}++#[test]+fn large_wal_recovers_after_reader_release_without_reopening_or_over_retiring() {+ for already_checkpointed in [false, true] {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let fixture = Fixture::new("large-wal-retirement");+ let writer = writer(&store);+ let channel = fact_channel(&fixture, 512 * 1024);+ for event in request_facts(&fixture, "large-wal", finished("large-wal"), 100) {+ assert!(matches!(+ offer_fact(&writer, &channel, event),+ crate::WriterCycleOutcome::Ack(_)+ ));+ }+ install_content(&store, &fixture, b"eligible old detail", 200);+ store.set_capacity_limit(Some(1)).unwrap();+ store+ .connection+ .lock()+ .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")+ .unwrap();+ let reader = Connection::open(&store.activity_path).unwrap();+ reader.execute_batch("BEGIN").unwrap();+ reader+ .query_row("SELECT COUNT(*) FROM logical_requests", [], |r| {+ r.get::<_, u64>(0)+ })+ .unwrap();+ // A temporary old reader can let a single committed transaction grow+ // the WAL above the former 16 MiB admission guard. The unrelated table+ // remains allocated, so a lower threshold cannot be satisfied by it.+ store.connection.lock().execute_batch("CREATE TABLE capacity_wal_fixture(payload BLOB); INSERT INTO capacity_wal_fixture VALUES(zeroblob(20971520));").unwrap();+ let wal = format!("{}-wal", store.activity_path.display());+ assert!(fs::metadata(&wal).unwrap().len() > 16 * 1024 * 1024);+ due(&store);+ store.capacity_cycle(1000).unwrap();+ assert_eq!(+ store.capacity_status().unwrap().reason,+ "page_release_pending"+ );+ assert_eq!(+ store+ .connection+ .lock()+ .query_row("SELECT COUNT(*) FROM logical_requests", [], |r| r+ .get::<_, u64>(0))+ .unwrap(),+ 1,+ "reader-pinned space must not cause additional logical expiry"+ );+ drop(reader);+ if already_checkpointed {+ let (_, frames, backfilled): (i64, i64, i64) = store+ .connection+ .lock()+ .query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |r| {+ Ok((r.get(0)?, r.get(1)?, r.get(2)?))+ })+ .unwrap();+ assert_eq!(frames, backfilled);+ assert!(+ fs::metadata(&wal).unwrap().len() > 16 * 1024 * 1024,+ "checkpointed WAL allocation retains its physical high-water mark"+ );+ }+ for _ in 0..30 {+ due(&store);+ store.capacity_cycle(1000).unwrap();+ }+ let db = store.connection.lock();+ assert_eq!(+ db.query_row("SELECT COUNT(*) FROM logical_requests", [], |r| r+ .get::<_, u64>(0))+ .unwrap(),+ 0,+ "a released reader must allow capacity progress without reopening"+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_capacity_pending_v5",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_capacity_blobs_v5",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert!(fs::metadata(&wal).unwrap().len() < 16 * 1024 * 1024);+ }+}++#[test]+fn a_nonpreemptible_overrun_extends_the_completion_based_cooldown() {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ store.set_capacity_limit(Some(1)).unwrap();+ store.connection.lock().trace(Some(|sql| {+ if sql.starts_with("SELECT EXISTS(SELECT 1 FROM observation_capacity_pending_v5)") {+ // Model a slow SQLite/filesystem unit that progress callbacks+ // cannot preempt; the next pass must account for its full cost.+ std::thread::sleep(Duration::from_millis(250));+ }+ }));+ let start = Instant::now();+ store.capacity_cycle(1000).unwrap();+ store.connection.lock().trace(None);+ assert!(start.elapsed() >= Duration::from_millis(250));+ assert!(store.capacity.lock().next_due >= Instant::now() + Duration::from_secs(20));+ assert!(!store.capacity_cycle(2000).unwrap());+}++#[test]+fn turn_cleanup_progresses_with_unrelated_requests_and_surviving_session_turns() {+ use std::sync::{+ Arc,+ atomic::{AtomicU8, Ordering},+ };++ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let fixture = Fixture::new("turn-cleanup");+ let writer = writer(&store);+ let channel = fact_channel(&fixture, 512 * 1024);+ for event in request_facts(&fixture, "turn-cleanup", finished("turn-cleanup"), 100) {+ assert!(matches!(+ offer_fact(&writer, &channel, event),+ crate::WriterCycleOutcome::Ack(_)+ ));+ }+ let db = store.connection.lock();+ db.execute("WITH RECURSIVE n(x) AS(VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<50000) INSERT INTO logical_requests(workspace_id,request_id,session_id,turn_id,started_at_ms,finished_at_ms) SELECT ?1,'unrelated-request-'||x,'unrelated-session','unrelated-turn-'||x,200,300 FROM n", [fixture.workspace.as_str()]).unwrap();+ db.execute("WITH RECURSIVE n(x) AS(VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<500) INSERT INTO turns(workspace_id,turn_id,session_id,started_at_ms) SELECT ?1,'surviving-turn-'||x,?2,200 FROM n", params![fixture.workspace.as_str(),fixture.session.as_str()]).unwrap();+ db.execute("INSERT INTO logical_requests(workspace_id,request_id,session_id,turn_id,started_at_ms,finished_at_ms) SELECT workspace_id,'surviving-request-'||turn_id,session_id,turn_id,200,300 FROM turns WHERE workspace_id=?1 AND session_id=?2 AND turn_id!=?3", params![fixture.workspace.as_str(),fixture.session.as_str(),fixture.turn.as_str()]).unwrap();+ // Reopen an already-current store that predates the required lookup index.+ db.execute_batch("DROP INDEX IF EXISTS observation_capacity_turn_v5")+ .unwrap();+ drop(db);+ drop(writer);+ drop(store);+ let store = open_store(directory.path());+ let mut db = store.connection.lock();+ store+ .capacity_retire(+ &mut db,+ fixture.workspace.as_str(),+ fixture.session.as_str(),+ fixture.request.as_str(),+ 1000,+ )+ .unwrap();+ let mut state = State::default();+ // Maintenance may yield between durable phases on a busy host. Preserve+ // both the production time budget and the per-slice scan regression guard.+ for _ in 0..10 {+ let deadline = Instant::now() + BUDGET;+ let interrupted = Arc::new(AtomicU8::new(0));+ let reason = Arc::clone(&interrupted);+ let mut instructions = 0;+ db.progress_handler(+ 1000,+ Some(move || {+ instructions += 1000;+ let value = if instructions > 100_000 {+ 2+ } else if Instant::now() >= deadline {+ 1+ } else {+ return false;+ };+ reason.store(value, Ordering::Relaxed);+ true+ }),+ );+ let result = store.capacity_cleanup(&mut db, &mut state, deadline);+ db.progress_handler(0, None::<fn() -> bool>);+ let reason = interrupted.load(Ordering::Relaxed);+ assert_ne!(reason, 2, "turn cleanup must not scan unrelated history");+ assert!(result.is_ok() || reason == 1, "{result:?}");+ if !db+ .query_row(+ "SELECT EXISTS(SELECT 1 FROM observation_capacity_pending_v5)",+ [],+ |row| row.get::<_, bool>(0),+ )+ .unwrap()+ {+ break;+ }+ }+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_capacity_pending_v5",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM turns WHERE workspace_id=?1 AND session_id=?2",+ params![fixture.workspace.as_str(), fixture.session.as_str()],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 500+ );+ assert_eq!(+ db.query_row("SELECT COUNT(*) FROM logical_requests", [], |r| r+ .get::<_, u64>(0))+ .unwrap(),+ 50_500+ );+}++#[test]+fn request_chunk_cleanup_uses_request_membership_before_unrelated_workspace_history() {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let fixture = Fixture::new("bounded-chunk-selection");+ install_content(&store, &fixture, b"selected request body", 100);+ let mut db = store.connection.lock();+ // Match a large workspace while keeping the selected request tiny. Both+ // carriers have valid matching content membership, preceding it by ID.+ db.execute_batch("WITH RECURSIVE n(x) AS(VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<10000) INSERT INTO content_instances_v2 SELECT workspace_id,'aaa-unrelated-'||x,message_instance_id,'unrelated-request',direction,fork_id,part_ordinal,content_kind,canonical_media_type,content_blob_digest,expected_byte_count,has_content_ref,transport_frame_id,downstream_delivery,accumulated_byte_count,state,created_at_unix_nanos FROM content_instances_v2 CROSS JOIN n WHERE content_id LIKE 'content-%' LIMIT 10000;").unwrap();+ db.execute_batch("INSERT INTO content_chunks_v2 SELECT workspace_id,content_id,0,'not-a-selected-object',0,1,'unrelated-event' FROM content_instances_v2 WHERE request_id='unrelated-request';").unwrap();+ db.execute(+ "INSERT INTO observation_capacity_pending_v5(workspace,request,session) VALUES(?1,?2,?3)",+ params![+ fixture.workspace.as_str(),+ fixture.request.as_str(),+ fixture.session.as_str()+ ],+ )+ .unwrap();+ let mut steps = 0;+ db.progress_handler(+ 100,+ Some(move || {+ steps += 100;+ steps > 20000+ }),+ );+ let mut state = State::default();+ let _ = store.capacity_cleanup(&mut db, &mut state, Instant::now() + Duration::from_secs(5));+ db.progress_handler(0, None::<fn() -> bool>);+ let step: Option<i64> = db+ .query_row(+ "SELECT step FROM observation_capacity_pending_v5",+ [],+ |r| r.get(0),+ )+ .optional()+ .unwrap();+ assert!(+ step.is_none_or(|step| step > 0),+ "a tiny request cannot exhaust its SQL budget scanning other requests' chunks"+ );+}++#[cfg(unix)]+#[test]+fn capacity_accounting_preserves_sqlite_writer_locks_against_other_processes() {+ const CHILD_DB: &str = "HIROUTE_CAPACITY_LOCK_CHILD_DB";+ if let Some(path) = std::env::var_os(CHILD_DB) {+ let db = Connection::open(path).unwrap();+ db.busy_timeout(Duration::ZERO).unwrap();+ assert!(+ db.execute_batch("BEGIN IMMEDIATE").is_err(),+ "accounting must not release another SQLite connection's writer lock"+ );+ return;+ }+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let db = store.connection.lock();+ db.execute_batch("BEGIN IMMEDIATE").unwrap();+ assert!(db_bytes(&store).unwrap() > 0);+ let child = std::process::Command::new(std::env::current_exe().unwrap())+ .args(["--exact", "store::capacity::tests::capacity_accounting_preserves_sqlite_writer_locks_against_other_processes", "--nocapture"])+ .env(CHILD_DB, &store.activity_path)+ .status()+ .unwrap();+ assert!(+ child.success(),+ "a competing process acquired the active writer transaction after accounting"+ );+ db.execute_batch("ROLLBACK").unwrap();+}+#[test]+fn retires_completed_request_in_slices_preserves_active_shared_history_and_rollup() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_store(dir.path());+ let a = Fixture::new("capacity-old");+ let mut b = Fixture::new("capacity-live");+ b.workspace = hiroute_domain::WorkspaceId::parse("other-workspace").unwrap();+ let writer = writer(&store);+ let channel = fact_channel(&a, 512 * 1024);+ for event in request_facts(&a, "attempt-capacity", finished("attempt-capacity"), 100) {+ assert!(matches!(+ offer_fact(&writer, &channel, event),+ crate::WriterCycleOutcome::Ack(_)+ ));+ }+ install_content(&store, &a, b"shared retained body", 200);+ install_content(&store, &b, b"shared retained body", 300);+ let retired_fact =+ crate::store::fact_log::load_request(&store.connection.lock(), &a.workspace, &a.request)+ .unwrap()+ .remove(0);+ // A large request must make committed progress across multiple bounded units.+ {+ let mut db = store.connection.lock();+ let tx = db.transaction().unwrap();+ for i in 0..2048 {+ // Rowid order deliberately differs from the request index order.+ tx.execute("INSERT INTO conversation_content_events_v2(stream_ref,sequence,event_id,workspace_id,request_id,direction,fork_id,phase,metadata_json,envelope_digest) SELECT stream_ref,?1,?2,workspace_id,request_id,direction,fork_id,phase,metadata_json,envelope_digest FROM conversation_content_events_v2 WHERE request_id=?3 LIMIT 1",params![12047-i,format!("bulk-{i}"),a.request.as_str()]).unwrap();+ tx.execute("INSERT INTO observation_feedback_v2(stream_ref,rejected_or_acked_sequence,event_id,envelope_digest,feedback_kind,feedback_json,created_at_unix_nanos) SELECT stream_ref,sequence,event_id,envelope_digest,'ack',(SELECT feedback_json FROM observation_feedback_v2 WHERE feedback_kind='ack' LIMIT 1),100 FROM conversation_content_events_v2 WHERE event_id=?1",[format!("bulk-{i}")]).unwrap();+ }+ tx.commit().unwrap();+ }+ store.set_capacity_limit(Some(1)).unwrap();+ for _ in 0..30 {+ due(&store);+ store.capacity_cycle(1000).unwrap();+ }+ let db = store.connection.lock();+ assert!(+ !db.query_row(+ "SELECT EXISTS(SELECT 1 FROM logical_requests WHERE request_id=?1)",+ [a.request.as_str()],+ |r| r.get::<_, bool>(0)+ )+ .unwrap()+ );+ assert!(+ db.query_row(+ "SELECT EXISTS(SELECT 1 FROM logical_requests WHERE request_id=?1)",+ [b.request.as_str()],+ |r| r.get::<_, bool>(0)+ )+ .unwrap()+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM conversation_content_events_v2 WHERE request_id=?1",+ [a.request.as_str()],+ |r| r.get::<_, i64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_feedback_v2 WHERE event_id LIKE 'bulk-%'",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0,+ "retiring a carrier must not strand its accepted feedback"+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_analysis_dirty_v1 WHERE request_id=?1",+ [a.request.as_str()],+ |r| r.get::<_, i64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_feedback_v2 WHERE event_id=?1",+ [retired_fact.event_id.as_str()],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_events WHERE event_id=?1",+ [retired_fact.event_id.as_str()],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 1+ );+ let next: u64 = db+ .query_row(+ "SELECT MAX(highest_accounted_sequence)+1 FROM observation_checkpoints",+ [],+ |r| r.get(0),+ )+ .unwrap();+ drop(db);+ use crate::writer::{IngestOutcome, ObservationCommitPort};+ assert!(matches!(+ store.ingest_fact(&retired_fact, &[]).unwrap(),+ IngestOutcome::Nack(_)+ ));+ let mut reused_id = retired_fact;+ reused_id.sequence = next;+ reused_id.correlation.request_id =+ hiroute_domain::LogicalRequestId::parse("changed-request-cannot-reuse-retired-event")+ .unwrap();+ assert!(+ matches!(store.ingest_fact(&reused_id,&[]).unwrap(), IngestOutcome::Nack(nack) if matches!(nack.detail,hiroute_domain::ObservationNackDetailV1::ImmutableProjectionConflict { .. }))+ );+ let detail = store+ .get_session(&b.workspace, &b.session, ContentMode::Messages)+ .unwrap();+ assert_eq!(detail.summary.request_count, 1);+ let value = store+ .get_value(&a.workspace, &value_query("plan/codex-daily", "USD", None))+ .unwrap();+ assert_eq!(value.input_tokens, 100);+ assert_eq!(value.estimated_total_savings_micros, Some(900));+ assert!(+ store+ .blob_path(+ store+ .authority+ .content_blob_digest("text/plain", b"shared retained body")+ .as_str()+ )+ .exists()+ );+}+#[test]+fn busy_writer_yields_and_schedules_from_completion() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_store(dir.path());+ store.set_capacity_limit(Some(1)).unwrap();+ let lock = store.connection.lock();+ let start = Instant::now();+ store.capacity_cycle(1).unwrap();+ assert!(start.elapsed() < Duration::from_millis(20));+ drop(lock);+ let state = store.capacity.lock();+ assert!(state.next_due >= start + INTERVAL);+ assert_eq!(state.status.reason, "writer_busy");+ drop(state);+ assert!(!store.capacity_cycle(2).unwrap());+}++#[test]+fn disabling_after_committed_retirement_resumes_cleanup_after_restart_without_selecting_another_request()+ {+ let directory = tempfile::tempdir().unwrap();+ let store = open_store(directory.path());+ let a = Fixture::new("disable-retired");+ let b = Fixture::new("disable-survivor");+ let writer = writer(&store);+ for (fixture, at) in [(&a, 100), (&b, 200)] {+ let channel = fact_channel(fixture, 512 * 1024);+ let attempt = format!("attempt-{}", fixture.request);+ for event in request_facts(fixture, &attempt, finished(&attempt), at) {+ assert!(matches!(+ offer_fact(&writer, &channel, event),+ crate::WriterCycleOutcome::Ack(_)+ ));+ }+ }+ install_content(&store, &a, b"already committed retirement", 300);+ install_content(&store, &b, b"completed but policy disabled", 400);+ store.set_capacity_limit(Some(1)).unwrap();+ store+ .capacity_retire(+ &mut store.connection.lock(),+ a.workspace.as_str(),+ a.session.as_str(),+ a.request.as_str(),+ 1000,+ )+ .unwrap();+ store.set_capacity_limit(None).unwrap();+ drop(writer);+ drop(store);+ let store = open_store(directory.path());+ assert_eq!(store.capacity_status().unwrap().max_active_bytes, None);+ assert_eq!(+ store+ .connection+ .lock()+ .query_row(+ "SELECT COUNT(*) FROM observation_capacity_pending_v5",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 1+ );+ for _ in 0..30 {+ due(&store);+ store.capacity_cycle(1000).unwrap();+ }+ let db = store.connection.lock();+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_capacity_pending_v5",+ [],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM content_instances_v2 WHERE request_id=?1",+ [a.request.as_str()],+ |r| r.get::<_, u64>(0)+ )+ .unwrap(),+ 0+ );+ assert!(+ db.query_row(+ "SELECT EXISTS(SELECT 1 FROM logical_requests WHERE request_id=?1)",+ [b.request.as_str()],+ |r| r.get::<_, bool>(0)+ )+ .unwrap()+ );+ assert!(db.query_row("SELECT EXISTS(SELECT 1 FROM content_instances_v2 WHERE request_id=?1 AND state='complete')", [b.request.as_str()], |r| r.get::<_, bool>(0)).unwrap());+}+#[test]+fn exhausted_sql_budget_rolls_back_without_deleting_or_running_fast_follow_on_work() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_store(dir.path());+ let mut db = store.connection.lock();+ db.progress_handler(1, Some(|| true));+ let result = store.capacity_retire(&mut db, "default", "missing", "missing", 0);+ assert!(result.is_err());+ db.progress_handler(0, None::<fn() -> bool>);+ assert_eq!(+ db.query_row(+ "SELECT COUNT(*) FROM observation_capacity_pending_v5",+ [],+ |r| r.get::<_, i64>(0)+ )+ .unwrap(),+ 0+ );+}++#[test]+fn failed_policy_commit_keeps_the_previous_limit() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_store(dir.path());+ store.set_capacity_limit(Some(20_000_000_000)).unwrap();+ store.connection.lock().execute_batch("CREATE TRIGGER reject_capacity BEFORE UPDATE ON observation_meta WHEN OLD.key='max_active_bytes' BEGIN SELECT RAISE(ABORT,'injected write failure'); END;").unwrap();+ assert!(store.set_capacity_limit(Some(1)).is_err());+ assert!(store.set_capacity_limit(None).is_err());+ assert_eq!(+ store.capacity_status().unwrap().max_active_bytes,+ Some(20_000_000_000)+ );+}diff --git a/crates/observation/src/store/field_replay_tests.rs b/crates/observation/src/store/field_replay_tests.rsnew file mode 100644index 00000000..03d62552--- /dev/null+++ b/crates/observation/src/store/field_replay_tests.rs@@ -0,0 +1,81 @@+//! Paired writer acceptance accepts an actual captured suffix, never fabricated identities.+use crate::writer::{IngestOutcome, ObservationCommitPort};+use crate::{DigestAuthority, LocalObservationStore};+use hiroute_domain::{+ ConversationContentEnvelopeV1, ExecutionFactEnvelopeV1, ObservationAckV2,+ ObservationGapHeartbeatV1,+};+use std::{+ fs,+ io::{BufRead, BufReader},+ path::PathBuf,+ time::Instant,+};+use zeroize::Zeroizing;++#[test]+#[ignore = "requires a marked stopped history and genuine captured native envelopes"]+fn captured_native_suffix_uses_the_real_writer() {+ let root = PathBuf::from(std::env::var_os("HIROUTE_STORAGE_FIELD_ROOT").expect("field root"));+ assert_eq!(root.file_name().unwrap(), "observation");+ assert!(+ root.parent()+ .unwrap()+ .join(".hiroute-storage-validation")+ .is_file()+ );+ let key = Zeroizing::new(+ fs::read(std::env::var_os("HIROUTE_STORAGE_FIELD_KEY_FILE").expect("key file")).unwrap(),+ );+ let store = LocalObservationStore::open(+ &root,+ DigestAuthority::new(key.as_slice().try_into().unwrap()),+ )+ .unwrap();+ let input =+ fs::File::open(std::env::var_os("HIROUTE_STORAGE_FIELD_REPLAY").expect("captured suffix"))+ .unwrap();+ let start = Instant::now();+ let mut events = 0u64;+ for line in BufReader::new(input).lines() {+ let line = Zeroizing::new(line.unwrap());+ let record: serde_json::Value = serde_json::from_str(&line).unwrap();+ let expected: ObservationAckV2 = serde_json::from_value(record["ack"].clone()).unwrap();+ let apply = || {+ if record["kind"] == "gap_heartbeat" {+ let heartbeat: ObservationGapHeartbeatV1 =+ serde_json::from_value(record["envelope"].clone()).unwrap();+ return store.ingest_gap_heartbeat(&heartbeat).unwrap();+ }+ match record["channel"].as_str().unwrap() {+ "execution_fact" => {+ let event: ExecutionFactEnvelopeV1 =+ serde_json::from_value(record["envelope"].clone()).unwrap();+ store.ingest_fact(&event, &[]).unwrap()+ }+ "conversation_content" => {+ let event: ConversationContentEnvelopeV1 =+ serde_json::from_value(record["envelope"].clone()).unwrap();+ store.ingest_content(&event, &[]).unwrap()+ }+ _ => panic!("capture contains an unsupported carrier; do not rewrite or omit it"),+ }+ };+ assert_eq!(+ apply(),+ IngestOutcome::Ack(expected.clone()),+ "original first-ingest acknowledgement at event {events}"+ );+ assert_eq!(+ apply(),+ IngestOutcome::Ack(expected),+ "exact duplicate acknowledgement at event {events}"+ );+ events += 1;+ }+ assert!(events > 0, "zero captured events is not acceptance");+ println!(+ "{}",+ serde_json::json!({"phase":"native_suffix_replayed","events":events,"seconds":start.elapsed().as_secs_f64()})+ );+}diff --git a/crates/observation/src/store/field_tests.rs b/crates/observation/src/store/field_tests.rsnew file mode 100644index 00000000..6b80fb2c--- /dev/null+++ b/crates/observation/src/store/field_tests.rs@@ -0,0 +1,204 @@+//! Opt-in acceptance over an operator-owned stopped copy, never the live store.+use super::*;+use crate::maintenance::ObservationMaintenance;+use std::{+ sync::{+ Arc,+ atomic::{AtomicBool, AtomicU64, Ordering},+ },+ time::{Duration, Instant},+};+use zeroize::Zeroizing;++#[test]+#[ignore = "requires an explicitly prepared private stopped-source copy"]+fn current_history_and_real_maintenance() {+ let root = PathBuf::from(std::env::var_os("HIROUTE_STORAGE_FIELD_ROOT").expect("field root"));+ assert_eq!(root.file_name().unwrap(), "observation");+ assert!(+ root.parent()+ .unwrap()+ .join(".hiroute-storage-validation")+ .is_file(),+ "only an explicitly marked validation copy may be changed"+ );+ let key_path = std::env::var_os("HIROUTE_STORAGE_FIELD_KEY_FILE").expect("private key file");+ let key = Zeroizing::new(fs::read(key_path).unwrap());+ let authority = DigestAuthority::new(key.as_slice().try_into().expect("32-byte original key"));+ assert_eq!(+ super::upgrade::version(&root).unwrap().as_deref(),+ Some("5"),+ "archive old history through production startup before running current-store maintenance"+ );+ let start = Instant::now();+ let store = Arc::new(LocalObservationStore::open(&root, authority.clone()).unwrap());+ println!(+ "{}",+ serde_json::json!({"phase":"open", "seconds":start.elapsed().as_secs_f64(), "pid":std::process::id(), "counts":counts(&store, true)})+ );+ drop(store);+ let reopened = Instant::now();+ let store = Arc::new(LocalObservationStore::open(&root, authority).unwrap());+ println!(+ "{}",+ serde_json::json!({"phase":"reopen", "seconds":reopened.elapsed().as_secs_f64(), "counts":counts(&store, true)})+ );+ let seconds: u64 = std::env::var("HIROUTE_STORAGE_FIELD_SECONDS")+ .unwrap_or_else(|_| "0".into())+ .parse()+ .unwrap();+ if seconds == 0 {+ return;+ }+ let limit = std::env::var("HIROUTE_STORAGE_FIELD_LIMIT")+ .ok()+ .map(|s| s.parse::<u64>().unwrap());+ store.set_capacity_limit(limit).unwrap();+ capacity::probe::enable();+ let maintenance = ObservationMaintenance::start(&store).unwrap();+ let readers: usize = std::env::var("HIROUTE_STORAGE_FIELD_READERS")+ .unwrap_or_else(|_| "0".into())+ .parse()+ .unwrap();+ assert!(readers <= 4);+ let stop = Arc::new(AtomicBool::new(false));+ let stats = Arc::new(ReaderStats::default());+ let workspace = store.connection.lock().query_row("SELECT workspace_id FROM logical_requests GROUP BY workspace_id ORDER BY COUNT(*) DESC LIMIT 1",[],|r|r.get::<_,String>(0)).map(|s|WorkspaceId::parse(s).unwrap()).unwrap_or_default();+ let mut workers = Vec::new();+ for i in 0..readers {+ let (store, stop, stats, workspace) = (+ store.clone(),+ stop.clone(),+ stats.clone(),+ workspace.clone(),+ );+ workers.push(+ std::thread::Builder::new()+ .name(format!("field-read-{i}"))+ .spawn(move || {+ use hiroute_domain::{ObservationReaderContext, ObservationRequestQuery};+ let reader = ObservationReaderContext::local_user(+ workspace,+ format!("field-reader-{i}"),+ 1,+ now_ms()+ .saturating_add((seconds as i64).saturating_mul(1000))+ .saturating_add(60000),+ true,+ true,+ )+ .unwrap();+ while !stop.load(Ordering::Relaxed) {+ let query = ObservationRequestQuery {+ from_ms: 0,+ to_ms: now_ms().saturating_add(1),+ session_id: None,+ request_id: None,+ limit: 10,+ cursor: None,+ agent_id: None,+ plan_id: None,+ native_model: None,+ outcome: None,+ only_model_switch: false,+ };+ let start = Instant::now();+ match store.observed_sessions(&reader, &query, now_ms()) {+ Ok(page) => {+ stats.passed.fetch_add(1, Ordering::Relaxed);+ stats+ .rows+ .fetch_add(page.sessions.len() as u64, Ordering::Relaxed);+ }+ Err(error) => {+ eprintln!(+ "field reader failure: {error:?}, elapsed_us={}",+ start.elapsed().as_micros()+ );+ stats.failed.fetch_add(1, Ordering::Relaxed);+ }+ }+ stats.max_us.fetch_max(+ start.elapsed().as_micros().try_into().unwrap_or(u64::MAX),+ Ordering::Relaxed,+ );+ std::thread::sleep(Duration::from_millis(250));+ }+ })+ .unwrap(),+ );+ }+ let start = Instant::now();+ while start.elapsed() < Duration::from_secs(seconds) {+ let passes = capacity::probe::samples();+ println!(+ "{}",+ serde_json::json!({"phase":"maintenance", "seconds":start.elapsed().as_secs_f64(), "capacity":store.capacity_status().unwrap(), "capacity_pass_count":passes.len(),"capacity_last_pass":passes.last(),"counts":counts(&store, false), "maintenance":store.maintenance_status()})+ );+ std::thread::sleep(+ Duration::from_secs(10)+ .min(Duration::from_secs(seconds).saturating_sub(start.elapsed())),+ );+ }+ stop.store(true, Ordering::Relaxed);+ for worker in workers {+ worker.join().unwrap();+ }+ drop(maintenance);+ let capacity_passes = capacity::probe::samples();+ println!(+ "{}",+ serde_json::json!({"phase":"finished", "seconds":start.elapsed().as_secs_f64(), "capacity":store.capacity_status().unwrap(), "capacity_passes":capacity_passes,"counts":counts(&store, true), "readers":{"passed":stats.passed.load(Ordering::Relaxed),"failed":stats.failed.load(Ordering::Relaxed),"rows":stats.rows.load(Ordering::Relaxed),"max_us":stats.max_us.load(Ordering::Relaxed)}})+ );+ // A pressure run never leaves an enabled policy in its retained evidence.+ store.set_capacity_limit(None).unwrap();+ if readers > 0 {+ assert!(stats.passed.load(Ordering::Relaxed) > 0 && stats.rows.load(Ordering::Relaxed) > 0);+ assert_eq!(stats.failed.load(Ordering::Relaxed), 0);+ }+}++#[derive(Default)]+struct ReaderStats {+ passed: AtomicU64,+ failed: AtomicU64,+ rows: AtomicU64,+ max_us: AtomicU64,+}+fn now_ms() -> i64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_millis()+ .try_into()+ .unwrap()+}++fn counts(store: &LocalObservationStore, full: bool) -> serde_json::Value {+ let db = store.connection.lock();+ let mut result = serde_json::Map::new();+ let tables: &[&str] = if full {+ &[+ "logical_requests",+ "observation_events",+ "conversation_content_events_v2",+ "observation_feedback_v2",+ "content_blobs_v2",+ "observation_capacity_pending_v5",+ "observation_capacity_blobs_v5",+ ]+ } else {+ &[+ "logical_requests",+ "observation_capacity_pending_v5",+ "observation_capacity_blobs_v5",+ ]+ };+ for &table in tables {+ let count: u64 = db+ .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))+ .unwrap();+ result.insert(table.into(), count.into());+ }+ serde_json::Value::Object(result)+}diff --git a/crates/observation/src/store/ingest.rs b/crates/observation/src/store/ingest.rsindex 876c03b2..ff8c4331 100644--- a/crates/observation/src/store/ingest.rs+++ b/crates/observation/src/store/ingest.rs@@ -420,9 +420,10 @@ impl ObservationCommitPort for LocalObservationStore {.commit().map_err(|_| ObservationStoreError::ActivityUnavailable)?;for content_id in cleanup {- let _ = std::fs::remove_dir_all(- self.staging_directory(&envelope.correlation.workspace_id, &content_id),- );+ let directory = self.staging_directory(&envelope.correlation.workspace_id, &content_id);+ if std::fs::remove_dir_all(&directory).is_ok() {+ self.note_removed_directory(&directory);+ }}Ok(outcome)}@@ -501,7 +502,7 @@ impl ObservationCommitPort for LocalObservationStore {let existing: Option<String> = transaction.query_row("SELECT event_id FROM observation_events- WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4+ WHERE stream_ref=(SELECT stream_ref FROM observation_streams_v5 WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4)AND sequence=?5",params![channel.as_str(),diff --git a/crates/observation/src/store/mod.rs b/crates/observation/src/store/mod.rsindex 5c62d1f5..e264fd97 100644--- a/crates/observation/src/store/mod.rs+++ b/crates/observation/src/store/mod.rs@@ -1,12 +1,18 @@//! Physical `activity.db` plus an external content-object directory.mod backfill;+pub(crate) mod capacity;mod extensions;mod unified_retention;pub(crate) mod fact_log;+#[cfg(test)]+mod field_replay_tests;+#[cfg(test)]+mod field_tests;mod gap;mod ingest;mod protocol;+pub(crate) mod records;mod request_retention;mod retention;mod safe_facts;@@ -14,6 +20,7 @@ mod schema;pub(crate) mod sensitive;#[cfg(test)]pub(crate) mod tests;+mod upgrade;mod value_rollup;use std::fs;@@ -26,6 +33,9 @@ use rusqlite::Connection;use crate::DigestAuthority;use crate::writer::ObservationStoreError;+#[cfg(test)]+type SessionReadTestHook = Box<dyn FnOnce(&LocalObservationStore) + Send>;+pub struct LocalObservationStore {pub(crate) maintenance_running: std::sync::Arc<std::sync::atomic::AtomicBool>,pub(crate) index_running: std::sync::Arc<std::sync::atomic::AtomicBool>,@@ -37,27 +47,84 @@ pub struct LocalObservationStore {pub(crate) blob_root: PathBuf,pub(crate) authority: DigestAuthority,pub(crate) retention: RetentionPolicyV1,+ pub(crate) capacity: Mutex<capacity::State>,+ #[cfg(test)]+ pub(crate) session_read_test_hook: Mutex<Option<SessionReadTestHook>>,}impl LocalObservationStore {+ pub fn legacy_history_requires_seal(+ root: impl AsRef<Path>,+ ) -> Result<bool, ObservationStoreError> {+ upgrade::required(root.as_ref())+ }++ /// Called only after the startup coordinator has published the complete+ /// sealed source. Initialize privately, so a crash cannot leave a partly+ /// created format-4/5 database at the authoritative live name.+ pub fn open_fresh_after_seal(+ root: impl AsRef<Path>,+ authority: DigestAuthority,+ ) -> Result<Self, ObservationStoreError> {+ let root = root.as_ref();+ if root.exists() {+ return Self::open(root, authority);+ }+ let parent = root.parent().ok_or(ObservationStoreError::Io)?;+ let mut nonce = [0_u8; 8];+ getrandom::fill(&mut nonce).map_err(|_| ObservationStoreError::Io)?;+ let stage = parent.join(format!(+ ".observation-fresh-{:016x}",+ u64::from_le_bytes(nonce)+ ));+ hiroute_platform::private_fs::create_directory(&stage)+ .map_err(|_| ObservationStoreError::Io)?;+ let store = Self::open(&stage, authority.clone())?;+ store+ .connection+ .lock()+ .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ drop(store);+ hiroute_platform::durable::directory_barrier(&stage)+ .map_err(|_| ObservationStoreError::Io)?;+ if root.exists() {+ return Err(ObservationStoreError::Corrupt);+ }+ hiroute_platform::durable::rename(&stage, root).map_err(|_| ObservationStoreError::Io)?;+ hiroute_platform::durable::directory_barrier(parent)+ .map_err(|_| ObservationStoreError::Io)?;+ Self::open(root, authority)+ }+pub fn open(root: impl AsRef<Path>,authority: DigestAuthority,) -> Result<Self, ObservationStoreError> {let root = root.as_ref();+ upgrade::prepare(root)?;let content_root = root.join("conversation-content");let staging_root = content_root.join("staging");let blob_root = content_root.join("blobs");fs::create_dir_all(&staging_root).map_err(|_| ObservationStoreError::Io)?;fs::create_dir_all(&blob_root).map_err(|_| ObservationStoreError::Io)?;let activity_path = root.join("activity.db");+ let fresh = !activity_path.exists();let mut connection = Connection::open(&activity_path).map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ if fresh {+ connection+ .execute_batch("PRAGMA auto_vacuum=INCREMENTAL;")+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ }schema::migrate(&mut connection)?;extensions::migrate(&mut connection)?;// Optional, rebuildable analysis must not make otherwise supported core// storage unreadable when its own table-local format is unavailable.let _ = crate::analysis::migrate(&mut connection);+ connection+ .execute_batch(capacity::SCHEMA)+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;Ok(Self {maintenance_running: Default::default(),index_running: Default::default(),@@ -69,6 +136,9 @@ impl LocalObservationStore {blob_root,authority,retention: RetentionPolicyV1::default(),+ capacity: Mutex::default(),+ #[cfg(test)]+ session_read_test_hook: Mutex::default(),})}@@ -88,6 +158,43 @@ impl LocalObservationStore {)))}+ pub(crate) fn object_locator(&self, path: &Path) -> Result<String, ObservationStoreError> {+ let root = self+ .activity_path+ .parent()+ .ok_or(ObservationStoreError::Corrupt)?;+ let relative = path+ .strip_prefix(root)+ .map_err(|_| ObservationStoreError::Corrupt)?;+ if !relative.starts_with("conversation-content") {+ return Err(ObservationStoreError::Corrupt);+ }+ Ok(relative.to_string_lossy().replace('\\', "/"))+ }+ pub(crate) fn object_path(&self, stored: &str) -> Result<PathBuf, ObservationStoreError> {+ let path = Path::new(stored);+ let root = self+ .activity_path+ .parent()+ .ok_or(ObservationStoreError::Corrupt)?;+ // Legacy locators are admitted only inside this owned root. Conversion+ // rewrites them; no path outside the active generation is a dependency.+ let relative = if path.is_absolute() {+ path.strip_prefix(root)+ .map_err(|_| ObservationStoreError::Corrupt)?+ } else {+ path+ };+ if !relative.starts_with("conversation-content")+ || relative+ .components()+ .any(|c| !matches!(c, std::path::Component::Normal(_)))+ {+ return Err(ObservationStoreError::Corrupt);+ }+ Ok(root.join(relative))+ }+pub(crate) fn blob_path(&self, digest: &str) -> PathBuf {self.blob_root.join(portable_hash(digest))}diff --git a/crates/observation/src/store/protocol.rs b/crates/observation/src/store/protocol.rsindex 4c109833..46b19f49 100644--- a/crates/observation/src/store/protocol.rs+++ b/crates/observation/src/store/protocol.rs@@ -12,7 +12,7 @@ use crate::content::lifecycle::{ContentProjectionAck, ContentProjectionError};use crate::receipt::FactProjectionError;use crate::writer::{IngestOutcome, ObservationStoreError};-use super::{gap, increment_store_revision};+use super::{gap, increment_store_revision, records};#[derive(Clone, Copy, Debug)]pub(super) struct Checkpoint {@@ -51,7 +51,7 @@ pub(super) fn preflight(if sequence <= checkpoint.accounted {let existing: Option<(String, String)> = transaction.query_row("SELECT event_id, payload_digest FROM observation_events- WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4 AND sequence=?5",+ WHERE stream_ref=(SELECT stream_ref FROM observation_streams_v5 WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4) AND sequence=?5",params![channel.as_str(), stream.producer_id.as_str(), stream.producer_epoch.as_str(),stream.stream_id.as_str(), i64_from_u64(sequence)?],|row| Ok((row.get(0)?, row.get(1)?)),@@ -83,7 +83,7 @@ pub(super) fn preflight(}let existing_event: Option<(i64, String)> = transaction.query_row("SELECT sequence, payload_digest FROM observation_events- WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4 AND event_id=?5",+ WHERE stream_ref=(SELECT stream_ref FROM observation_streams_v5 WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4) AND event_id=?5",params![channel.as_str(), stream.producer_id.as_str(), stream.producer_epoch.as_str(),stream.stream_id.as_str(), event_id], |row| Ok((row.get(0)?, row.get(1)?)),).optional().map_err(|_| ObservationStoreError::ActivityUnavailable)?;@@ -253,23 +253,12 @@ pub(super) fn insert_event(event_id: &str,payload_digest: &str,) -> Result<(), ObservationStoreError> {- transaction- .execute(- "INSERT INTO observation_events- (channel, producer_id, producer_epoch, stream_id, sequence, event_id, payload_digest)- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",- params![- channel.as_str(),- stream.producer_id.as_str(),- stream.producer_epoch.as_str(),- stream.stream_id.as_str(),- i64_from_u64(sequence)?,- event_id,- payload_digest- ],- )+ let stream_ref = records::stream_ref(transaction, channel.as_str(), stream)?;+ transaction.execute("INSERT INTO observation_events(stream_ref,sequence,event_id,payload_digest) VALUES(?1,?2,?3,?4)",+ params![stream_ref,i64_from_u64(sequence)?,event_id,payload_digest]).map_err(|_| ObservationStoreError::ActivityUnavailable)?;gap::resolve_sequence(transaction, channel, stream, sequence)?;+Ok(())}@@ -281,22 +270,11 @@ pub(super) fn insert_gap_event(event_id: &str,payload_digest: &str,) -> Result<(), ObservationStoreError> {- transaction- .execute(- "INSERT INTO observation_events- (channel, producer_id, producer_epoch, stream_id, sequence, event_id, payload_digest)- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",- params![- channel.as_str(),- stream.producer_id.as_str(),- stream.producer_epoch.as_str(),- stream.stream_id.as_str(),- i64_from_u64(sequence)?,- event_id,- payload_digest,- ],- )+ let stream_ref = records::stream_ref(transaction, channel.as_str(), stream)?;+ transaction.execute("INSERT INTO observation_events(stream_ref,sequence,event_id,payload_digest) VALUES(?1,?2,?3,?4)",+ params![stream_ref,i64_from_u64(sequence)?,event_id,payload_digest]).map_err(|_| ObservationStoreError::ActivityUnavailable)?;+Ok(())}@@ -312,17 +290,20 @@ pub(super) fn insert_content_event(.as_object_mut().ok_or(ObservationStoreError::Corrupt)?.remove("canonical_bytes_base64");+ let stream_ref = records::stream_ref(+ transaction,+ "conversation_content",+ &envelope.producer.stream,+ )?;transaction.execute("INSERT INTO conversation_content_events_v2- (producer_id, producer_epoch, stream_id, sequence, event_id, workspace_id, request_id,+ (stream_ref, sequence, event_id, workspace_id, request_id,direction, fork_id, phase, metadata_json, canonical_chunk_object_ref,canonical_chunk_byte_offset, canonical_chunk_byte_count, envelope_digest)- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",params![- envelope.producer.stream.producer_id.as_str(),- envelope.producer.stream.producer_epoch.as_str(),- envelope.producer.stream.stream_id.as_str(),+ stream_ref,i64_from_u64(envelope.sequence)?,envelope.event_id.as_str(),envelope.correlation.workspace_id.as_str(),@@ -330,7 +311,9 @@ pub(super) fn insert_content_event(envelope.direction.as_str(),envelope.fork_id,envelope.phase.as_str(),- serde_json::to_string(&metadata).map_err(|_| ObservationStoreError::Corrupt)?,+ records::encode(+ &serde_json::to_vec(&metadata).map_err(|_| ObservationStoreError::Corrupt)?+ )?,projection.chunk_object_ref,projection.chunk_byte_offset.map(i64_from_u64).transpose()?,projection.chunk_byte_count.map(i64_from_u64).transpose()?,@@ -358,22 +341,20 @@ pub(super) fn persist_feedback(serde_json::to_string(value),),};+ let stream_ref = records::stream_ref(transaction, channel, stream)?;transaction.execute("INSERT INTO observation_feedback_v2- (channel, producer_id, producer_epoch, stream_id, rejected_or_acked_sequence,+ (stream_ref, rejected_or_acked_sequence,event_id, envelope_digest, feedback_kind, feedback_json, created_at_unix_nanos)- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",params![- channel,- stream.producer_id.as_str(),- stream.producer_epoch.as_str(),- stream.stream_id.as_str(),+ stream_ref,i64_from_u64(sequence)?,event_id,envelope_digest,kind,- json.map_err(|_| ObservationStoreError::Corrupt)?,+ records::encode(json.map_err(|_| ObservationStoreError::Corrupt)?.as_bytes())?,i64_from_u64(occurred_at_unix_nanos)?],)@@ -389,10 +370,10 @@ pub(super) fn load_feedback(event_id: &str,digest: &str,) -> Result<Option<IngestOutcome>, ObservationStoreError> {- let row: Option<(String, String)> = transaction+ let row: Option<(String, Vec<u8>)> = transaction.query_row("SELECT feedback_kind, feedback_json FROM observation_feedback_v2- WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4+ WHERE stream_ref=(SELECT stream_ref FROM observation_streams_v5 WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4)AND rejected_or_acked_sequence=?5 AND event_id=?6 AND envelope_digest=?7ORDER BY feedback_id DESC LIMIT 1",params![@@ -409,14 +390,18 @@ pub(super) fn load_feedback(.optional().map_err(|_| ObservationStoreError::ActivityUnavailable)?;match row {- Some((kind, json)) if kind == "ack" => serde_json::from_str::<ObservationAckV2>(&json)- .map(IngestOutcome::Ack)- .map(Some)- .map_err(|_| ObservationStoreError::Corrupt),- Some((kind, json)) if kind == "nack" => serde_json::from_str::<ObservationNackV1>(&json)- .map(IngestOutcome::Nack)- .map(Some)- .map_err(|_| ObservationStoreError::Corrupt),+ Some((kind, json)) if kind == "ack" => {+ serde_json::from_slice::<ObservationAckV2>(&records::decode(&json)?)+ .map(IngestOutcome::Ack)+ .map(Some)+ .map_err(|_| ObservationStoreError::Corrupt)+ }+ Some((kind, json)) if kind == "nack" => {+ serde_json::from_slice::<ObservationNackV1>(&records::decode(&json)?)+ .map(IngestOutcome::Nack)+ .map(Some)+ .map_err(|_| ObservationStoreError::Corrupt)+ }Some(_) => Err(ObservationStoreError::Corrupt),None => Ok(None),}diff --git a/crates/observation/src/store/records.rs b/crates/observation/src/store/records.rsnew file mode 100644index 00000000..0499c421--- /dev/null+++ b/crates/observation/src/store/records.rs@@ -0,0 +1,214 @@+//! Stream identity is shared only inside protocol evidence. Frozen outcomes and+//! content metadata remain independently committed, bounded records.+use crate::writer::ObservationStoreError as Error;+use hiroute_domain::ObservationStreamV1;+use rusqlite::{Connection, OptionalExtension, params};+use sha2::{Digest, Sha256};+use std::{borrow::Cow, io::Read};++pub(crate) const MAX_RECORD: usize = 4 * 1024 * 1024;++pub(crate) fn encode(bytes: &[u8]) -> Result<Vec<u8>, Error> {+ let length = u32::try_from(bytes.len()).map_err(|_| Error::Corrupt)?;+ // Large pre-existing replies retain their original identity representation.+ // Only the bounded compressed form may expand during decoding.+ let compressed = if bytes.len() <= MAX_RECORD {+ zstd::bulk::compress(bytes, 1).map_err(|_| Error::Corrupt)?+ } else {+ Vec::new()+ };+ let (tag, payload) = if !compressed.is_empty() && compressed.len() < bytes.len() {+ (1, compressed.as_slice())+ } else {+ (0, bytes)+ };+ let mut result = Vec::with_capacity(payload.len() + 37);+ result.push(tag);+ result.extend_from_slice(&length.to_le_bytes());+ result.extend_from_slice(&Sha256::digest(bytes));+ result.extend_from_slice(payload);+ Ok(result)+}++pub(crate) fn decode(bytes: &[u8]) -> Result<Cow<'_, [u8]>, Error> {+ let header = bytes.get(..37).ok_or(Error::Corrupt)?;+ let length = u32::from_le_bytes(header[1..5].try_into().map_err(|_| Error::Corrupt)?) as usize;+ if header[0] == 1 && (length > MAX_RECORD || bytes.len() > MAX_RECORD + 37) {+ return Err(Error::Corrupt);+ }+ let body = &bytes[37..];+ let result = match header[0] {+ 0 => Cow::Borrowed(body),+ 1 => {+ let mut decoder = zstd::stream::read::Decoder::new(body).map_err(|_| Error::Corrupt)?;+ decoder.window_log_max(22).map_err(|_| Error::Corrupt)?;+ let mut output = Vec::with_capacity(length.min(64 * 1024));+ decoder+ .take(length as u64 + 1)+ .read_to_end(&mut output)+ .map_err(|_| Error::Corrupt)?;+ Cow::Owned(output)+ }+ _ => return Err(Error::Corrupt),+ };+ if result.len() != length || Sha256::digest(&result)[..] != header[5..37] {+ return Err(Error::Corrupt);+ }+ Ok(result)+}++pub(crate) fn stream_ref(+ db: &Connection,+ channel: &str,+ stream: &ObservationStreamV1,+) -> Result<i64, Error> {+ intern_stream(+ db,+ channel,+ stream.producer_id.as_str(),+ stream.producer_epoch.as_str(),+ stream.stream_id.as_str(),+ )+}++pub(crate) fn intern_stream(+ db: &Connection,+ channel: &str,+ producer: &str,+ epoch: &str,+ stream: &str,+) -> Result<i64, Error> {+ let args = params![channel, producer, epoch, stream];+ let query = "SELECT stream_ref FROM observation_streams_v5 WHERE channel=?1 AND producer_id=?2 AND producer_epoch=?3 AND stream_id=?4";+ if let Some(id) = db+ .query_row(query, args, |r| r.get(0))+ .optional()+ .map_err(|_| Error::ActivityUnavailable)?+ {+ return Ok(id);+ }+ db.execute("INSERT INTO observation_streams_v5(channel,producer_id,producer_epoch,stream_id) VALUES(?1,?2,?3,?4)", args).map_err(|_| Error::ActivityUnavailable)?;+ Ok(db.last_insert_rowid())+}++pub(crate) const SCHEMA: &str = r#"+CREATE TABLE IF NOT EXISTS observation_streams_v5(+ stream_ref INTEGER PRIMARY KEY, channel TEXT NOT NULL, producer_id TEXT NOT NULL,+ producer_epoch TEXT NOT NULL, stream_id TEXT NOT NULL, references_count INTEGER NOT NULL DEFAULT 0,+ UNIQUE(channel,producer_id,producer_epoch,stream_id));+CREATE TABLE IF NOT EXISTS observation_events(+ stream_ref INTEGER NOT NULL REFERENCES observation_streams_v5, sequence INTEGER NOT NULL,+ event_id TEXT NOT NULL,payload_digest TEXT NOT NULL,+ PRIMARY KEY(stream_ref,sequence),UNIQUE(stream_ref,event_id));+CREATE TABLE IF NOT EXISTS conversation_content_events_v2(+ stream_ref INTEGER NOT NULL REFERENCES observation_streams_v5,sequence INTEGER NOT NULL,event_id TEXT NOT NULL,+ workspace_id TEXT NOT NULL,request_id TEXT NOT NULL,direction TEXT NOT NULL,fork_id TEXT NOT NULL,+ phase TEXT NOT NULL,metadata_json BLOB NOT NULL,canonical_chunk_object_ref TEXT,+ canonical_chunk_byte_offset INTEGER,canonical_chunk_byte_count INTEGER,envelope_digest TEXT NOT NULL,+ PRIMARY KEY(stream_ref,sequence),UNIQUE(stream_ref,event_id));+CREATE INDEX IF NOT EXISTS content_events_v2_request ON conversation_content_events_v2(workspace_id,request_id,direction,fork_id,sequence);+CREATE TABLE IF NOT EXISTS observation_feedback_v2(+ feedback_id INTEGER PRIMARY KEY AUTOINCREMENT,stream_ref INTEGER NOT NULL REFERENCES observation_streams_v5,+ rejected_or_acked_sequence INTEGER NOT NULL,event_id TEXT NOT NULL,envelope_digest TEXT NOT NULL,+ feedback_kind TEXT NOT NULL,feedback_json BLOB NOT NULL,created_at_unix_nanos INTEGER NOT NULL);+CREATE INDEX IF NOT EXISTS observation_feedback_v2_lookup ON observation_feedback_v2(stream_ref,rejected_or_acked_sequence,event_id,envelope_digest,feedback_id DESC);+"#;++/// Select the same request-owned batch using its existing index order.+pub(super) fn retirement_order(table: &str) -> Option<&'static str> {+ // Match the existing request indices, including their rowid tie-breaker.+ // Feedback and carrier deletion must select exactly the same batch.+ match table {+ "conversation_content_events_v2" => Some("e.direction,e.fork_id,e.sequence,e.rowid"),+ "execution_fact_events" => Some("e.sequence,e.rowid"),+ _ => None,+ }+}++/// Once the request barrier is committed, its accepted feedback is obsolete.+/// Keep event-ID/digest conflicts and stream checkpoints as compact replay proof.+/// Feedback without an accepted request carrier has no proven retirement scope.+pub(super) fn retire_request_feedback(+ db: &Connection,+ workspace: &str,+ request: &str,+ table: &str,+ bounded: bool,+) -> rusqlite::Result<usize> {+ let source = match table {+ "conversation_content_events_v2" => {+ "SELECT e.stream_ref,e.sequence,e.event_id,e.envelope_digest FROM conversation_content_events_v2 e WHERE e.workspace_id=?1 AND e.request_id=?2"+ }+ "execution_fact_events" => {+ "SELECT s.stream_ref,e.sequence,e.event_id,e.envelope_digest FROM execution_fact_events e JOIN observation_streams_v5 s ON s.channel='execution_fact' AND s.producer_id=e.producer_id AND s.producer_epoch=e.producer_epoch AND s.stream_id=e.stream_id WHERE e.workspace_id=?1 AND e.request_id=?2"+ }+ _ => return Ok(0),+ };+ let order = retirement_order(table).ok_or(rusqlite::Error::InvalidQuery)?;+ let retired: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM observation_request_tombstones_v2 WHERE workspace_id=?1 AND request_id=?2)", params![workspace,request], |r|r.get(0))?;+ if !retired {+ return Err(rusqlite::Error::InvalidQuery);+ }+ let limit = if bounded { " LIMIT 128" } else { "" };+ db.execute(&format!("WITH batch AS MATERIALIZED ({source} ORDER BY {order}{limit}) DELETE FROM observation_feedback_v2 WHERE feedback_id IN (SELECT f.feedback_id FROM batch b JOIN observation_feedback_v2 f ON f.stream_ref=b.stream_ref AND f.rejected_or_acked_sequence=b.sequence AND f.event_id=b.event_id AND f.envelope_digest=b.envelope_digest{limit})"), params![workspace,request])+}++pub(super) fn retire_session_feedback(+ db: &Connection,+ workspace: &str,+ session: &str,+) -> rusqlite::Result<()> {+ let requests: Vec<String> = db+ .prepare("SELECT request_id FROM logical_requests WHERE workspace_id=?1 AND session_id=?2")?+ .query_map(params![workspace, session], |r| r.get(0))?+ .collect::<rusqlite::Result<_>>()?;+ for request in requests {+ for table in ["conversation_content_events_v2", "execution_fact_events"] {+ retire_request_feedback(db, workspace, &request, table, false)?;+ }+ }+ Ok(())+}++pub(crate) fn install_reference_triggers(db: &Connection) -> rusqlite::Result<()> {+ for table in [+ "observation_events",+ "conversation_content_events_v2",+ "observation_feedback_v2",+ ] {+ db.execute_batch(&format!("CREATE TRIGGER IF NOT EXISTS {table}_stream_insert AFTER INSERT ON {table} BEGIN+ UPDATE observation_streams_v5 SET references_count=references_count+1 WHERE stream_ref=NEW.stream_ref; END;+ CREATE TRIGGER IF NOT EXISTS {table}_stream_delete AFTER DELETE ON {table} BEGIN+ UPDATE observation_streams_v5 SET references_count=references_count-1 WHERE stream_ref=OLD.stream_ref;+ DELETE FROM observation_streams_v5 WHERE stream_ref=OLD.stream_ref AND references_count=0; END;"))?;+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn bounded_identity_compression_and_corruption() {+ for bytes in [+ b"".as_slice(),+ b"tiny",+ b"abcdefghabcdefghabcdefghabcdefghabcdefghabcdefgh",+ ] {+ assert_eq!(decode(&encode(bytes).unwrap()).unwrap().as_ref(), bytes);+ }+ let mut frame = encode(&vec![b'x'; 100000]).unwrap();+ assert_eq!(frame[0], 1);+ frame[1..5].copy_from_slice(&((MAX_RECORD + 1) as u32).to_le_bytes());+ assert!(decode(&frame).is_err());+ assert!(decode(&[99, 0, 0, 0, 0]).is_err());+ assert!(decode(&[0, 5, 0, 0, 0, 1]).is_err());+ let mut frame = encode(&vec![b'x'; 100000]).unwrap();+ frame.truncate(frame.len() / 2);+ assert!(decode(&frame).is_err());+ let large = vec![0; MAX_RECORD + 1];+ let frame = encode(&large).unwrap();+ assert_eq!(frame[0], 0);+ assert_eq!(decode(&frame).unwrap().as_ref(), large.as_slice());+ }+}diff --git a/crates/observation/src/store/request_retention.rs b/crates/observation/src/store/request_retention.rsindex 067b7dba..4b0ab79e 100644--- a/crates/observation/src/store/request_retention.rs+++ b/crates/observation/src/store/request_retention.rs@@ -48,6 +48,16 @@ impl LocalObservationStore {crate::valuation::archive_request(&transaction, workspace, session, request).map_err(|_| Error::Unavailable)?;transaction.execute("INSERT OR IGNORE INTO observation_request_tombstones_v2(workspace_id,request_id,deleted_ms) VALUES(?1,?2,?3)",params![workspace,request,now_ms]).map_err(|_|Error::Unavailable)?;+ for table in ["conversation_content_events_v2", "execution_fact_events"] {+ super::records::retire_request_feedback(+ &transaction,+ workspace,+ request,+ table,+ false,+ )+ .map_err(|_| Error::Unavailable)?;+ }transaction.execute("DELETE FROM content_chunks_v2 WHERE workspace_id=?1 AND content_id IN(SELECT content_id FROM content_instances_v2 WHERE workspace_id=?1 AND request_id=?2)",params![workspace,request]).map_err(|_|Error::Unavailable)?;for table in ["observation_sensitive_payloads_v2",@@ -101,7 +111,7 @@ impl LocalObservationStore {if !requests.is_empty() || quality_changed {// Index cleanup is part of the same visibility barrier. Physical// blob GC is resumable and can follow this committed logical erase.- transaction.execute_batch("DELETE FROM observation_text_blocks_v2 WHERE NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=observation_text_blocks_v2.workspace AND c.content_blob_digest=observation_text_blocks_v2.digest AND c.state='complete'); DELETE FROM observation_text_index_v2 WHERE NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=observation_text_index_v2.workspace AND c.content_blob_digest=observation_text_index_v2.digest AND c.state='complete');").map_err(|_|Error::Unavailable)?;+ transaction.execute_batch("DELETE FROM observation_text_blocks_v2 WHERE NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=observation_text_blocks_v2.workspace AND c.content_blob_digest=observation_text_blocks_v2.digest AND c.state='complete') AND NOT EXISTS(SELECT 1 FROM observation_capacity_blobs_v5 b WHERE b.workspace=observation_text_blocks_v2.workspace AND b.digest=observation_text_blocks_v2.digest); DELETE FROM observation_text_index_v2 WHERE NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=observation_text_index_v2.workspace AND c.content_blob_digest=observation_text_index_v2.digest AND c.state='complete') AND NOT EXISTS(SELECT 1 FROM observation_capacity_blobs_v5 b WHERE b.workspace=observation_text_index_v2.workspace AND b.digest=observation_text_index_v2.digest);").map_err(|_|Error::Unavailable)?;crate::query_v2::invalidate_visibility(&transaction).map_err(|_| Error::Unavailable)?;increment_store_revision(&transaction).map_err(|_| Error::Unavailable)?;}diff --git a/crates/observation/src/store/retention.rs b/crates/observation/src/store/retention.rsindex 58dd64f6..6dd11384 100644--- a/crates/observation/src/store/retention.rs+++ b/crates/observation/src/store/retention.rs@@ -17,6 +17,7 @@ use super::{LocalObservationStore, increment_store_revision, store_revision};pub(super) const GARBAGE_CANDIDATES_SQL: &str ="SELECT workspace_id,blob_digest FROM content_blobs_v2EXCEPT SELECT workspace_id,content_blob_digest FROM content_instances_v2 WHERE state='complete'+ EXCEPT SELECT workspace,digest FROM observation_capacity_blobs_v5ORDER BY 1,2 LIMIT ?1";#[cfg(test)]@@ -363,16 +364,23 @@ impl LocalObservationStore {// under the writer reservation before touching any physical file.let path: Option<String> = transaction.query_row("SELECT b.object_path FROM content_blobs_v2 b WHERE b.workspace_id=?1 AND b.blob_digest=?2- AND NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=b.workspace_id AND c.content_blob_digest=b.blob_digest AND c.state='complete')",+ AND NOT EXISTS(SELECT 1 FROM content_instances_v2 c WHERE c.workspace_id=b.workspace_id AND c.content_blob_digest=b.blob_digest AND c.state='complete') AND NOT EXISTS(SELECT 1 FROM observation_capacity_blobs_v5 protected WHERE protected.workspace=b.workspace_id AND protected.digest=b.blob_digest)",params![workspace,digest], |row|row.get(0),).optional().map_err(|_|ObservationQueryError::Unavailable)?;let Some(path) = path else {continue;};- match fs::remove_file(path) {- Ok(()) => {}- Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}- Err(_) => return Err(ObservationQueryError::Unavailable),+ let path = self+ .object_path(&path)+ .map_err(|_| ObservationQueryError::Corrupt)?;+ let shared:bool=transaction.query_row("SELECT EXISTS(SELECT 1 FROM content_instances_v2 WHERE content_blob_digest=?1 AND state='complete')",[digest],|row|row.get(0)).map_err(|_|ObservationQueryError::Unavailable)?;+ if !shared {+ match fs::remove_file(&path) {+ Ok(()) => {}+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}+ Err(_) => return Err(ObservationQueryError::Unavailable),+ }+ self.note_object_path(&path);}transaction.execute(@@ -410,8 +418,16 @@ impl LocalObservationStore {let path = entry.map_err(|_| ObservationQueryError::Unavailable)?.path();- if path.is_file() && !referenced.contains(&path.to_string_lossy().into_owned()) {- fs::remove_file(path).map_err(|_| ObservationQueryError::Unavailable)?;+ if path.is_file()+ && !referenced.contains(+ &self+ .object_locator(&path)+ .map_err(|_| ObservationQueryError::Corrupt)?,+ )+ && !referenced.contains(&path.to_string_lossy().into_owned())+ {+ fs::remove_file(&path).map_err(|_| ObservationQueryError::Unavailable)?;+ self.note_object_path(&path);}}Ok(())@@ -451,9 +467,11 @@ impl LocalObservationStore {.ok_or(ObservationQueryError::Corrupt)?;if !active.contains(name) {if path.is_dir() {- fs::remove_dir_all(path).map_err(|_| ObservationQueryError::Unavailable)?;+ fs::remove_dir_all(&path).map_err(|_| ObservationQueryError::Unavailable)?;+ self.note_removed_directory(&path);} else {- fs::remove_file(path).map_err(|_| ObservationQueryError::Unavailable)?;+ fs::remove_file(&path).map_err(|_| ObservationQueryError::Unavailable)?;+ self.note_object_path(&path);}}}@@ -531,6 +549,8 @@ fn delete_content_rows() -> Result<(), ObservationQueryError> {let workspace = spec.workspace_id.as_str();let session = spec.session_id.as_str();+ super::records::retire_session_feedback(transaction, workspace, session)+ .map_err(|_| ObservationQueryError::Unavailable)?;transaction.execute("DELETE FROM observation_sensitive_payloads_v2 WHERE workspace_id=?1 AND session_id=?2",diff --git a/crates/observation/src/store/retention_gc_tests.rs b/crates/observation/src/store/retention_gc_tests.rsindex ef579d8d..3fb2c614 100644--- a/crates/observation/src/store/retention_gc_tests.rs+++ b/crates/observation/src/store/retention_gc_tests.rs@@ -1,5 +1,35 @@use super::*;+#[test]+fn ordinary_gc_retains_a_blob_still_referenced_from_another_workspace() {+ use crate::store::tests::support::{Fixture, install_content, open_store};+ let root = tempfile::tempdir().unwrap();+ let store = open_store(root.path());+ let expired = Fixture::new("gc-expired-workspace");+ let mut live = Fixture::new("gc-live-workspace");+ live.workspace = hiroute_domain::WorkspaceId::parse("other-workspace").unwrap();+ let body = b"shared across independent workspace catalogs";+ install_content(&store, &expired, body, 100);+ install_content(&store, &live, body, 200);+ store+ .connection+ .lock()+ .execute(+ "UPDATE content_instances_v2 SET state='installing' WHERE workspace_id=?1",+ [expired.workspace.as_str()],+ )+ .unwrap();+ let candidates = store.garbage_candidates(32).unwrap();+ assert_eq!(candidates.len(), 1);+ assert_eq!(+ store.collect_unreferenced_candidates(&candidates).unwrap(),+ 1+ );+ let digest = store.authority.content_blob_digest("text/plain", body);+ assert_eq!(fs::read(store.blob_path(digest.as_str())).unwrap(), body);+ assert!(store.connection.lock().query_row("SELECT EXISTS(SELECT 1 FROM content_blobs_v2 WHERE workspace_id=?1 AND blob_digest=?2)",rusqlite::params![live.workspace.as_str(),digest.as_str()],|r|r.get::<_,bool>(0)).unwrap());+}+#[test]fn blob_made_live_after_gc_candidate_discovery_keeps_its_file_and_metadata() {let root = tempfile::tempdir().unwrap();@@ -24,7 +54,7 @@ fn blob_made_live_after_gc_candidate_discovery_keeps_its_file_and_metadata() {0);assert_eq!(- std::fs::read(root.path().join("indexed-source")).unwrap(),+ std::fs::read(store.blob_path(&digest)).unwrap(),b"captured content");assert!(diff --git a/crates/observation/src/store/schema.rs b/crates/observation/src/store/schema.rsindex df6a0b7d..ae4c8449 100644--- a/crates/observation/src/store/schema.rs+++ b/crates/observation/src/store/schema.rs@@ -3,10 +3,38 @@ use rusqlite::{Connection, Transaction, params};use crate::writer::ObservationStoreError;-pub(super) const STORE_SCHEMA_VERSION: &str = "4";+pub(super) const STORE_SCHEMA_VERSION: &str = "5";const LEGACY_AGENT_PLAN_ID: &str = "legacy/unknown";pub(super) fn migrate(connection: &mut Connection) -> Result<(), ObservationStoreError> {+ migrate_inner(connection, true)+}+#[cfg(test)]+pub(super) fn migrate_legacy(connection: &mut Connection) -> Result<(), ObservationStoreError> {+ migrate_inner(connection, false)+}+fn migrate_inner(connection: &mut Connection, compact: bool) -> Result<(), ObservationStoreError> {+ if connection+ .query_row(+ "SELECT value FROM observation_meta WHERE key='schema_version'",+ [],+ |r| r.get::<_, String>(0),+ )+ .ok()+ .as_deref()+ == Some("5")+ {+ let tables:u64=connection.query_row("SELECT COUNT(*) FROM sqlite_schema WHERE type='table' AND name IN('logical_requests','observation_streams_v5','observation_events','observation_checkpoints','observation_feedback_v2','conversation_content_events_v2','content_blobs_v2','content_instances_v2')",[],|r|r.get(0)).map_err(|_|ObservationStoreError::Corrupt)?;+ if tables != 8 {+ return Err(ObservationStoreError::Corrupt);+ }+ connection+ .execute_batch(+ "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;",+ )+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ return create_schema_v5(connection);+ }connection.execute_batch(r#"@@ -32,7 +60,7 @@ pub(super) fn migrate(connection: &mut Connection) -> Result<(), ObservationStor|row| row.get(0),).map_err(|_| ObservationStoreError::Corrupt)?;- match version.as_str() {+ let result = match version.as_str() {"1" => {migrate_v1_to_v2(connection)?;migrate_v2_to_v3(connection)?;@@ -43,9 +71,70 @@ pub(super) fn migrate(connection: &mut Connection) -> Result<(), ObservationStormigrate_v3_to_v4(connection)}"3" => migrate_v3_to_v4(connection),- STORE_SCHEMA_VERSION => create_schema_v4(connection),+ "4" => create_schema_v4(connection),+ STORE_SCHEMA_VERSION => Ok(()),_ => Err(ObservationStoreError::Corrupt),+ };+ result?;+ if !compact || version == "5" {+ return Ok(());}+ let populated: bool = connection.query_row("SELECT EXISTS(SELECT 1 FROM observation_events) OR EXISTS(SELECT 1 FROM conversation_content_events_v2) OR EXISTS(SELECT 1 FROM observation_feedback_v2)",[],|r|r.get(0)).map_err(|_|ObservationStoreError::Corrupt)?;+ if populated {+ return Err(ObservationStoreError::Corrupt);+ }+ connection.execute_batch("DROP TABLE observation_events; DROP TABLE conversation_content_events_v2; DROP TABLE observation_feedback_v2;").map_err(|_|ObservationStoreError::ActivityUnavailable)?;+ connection+ .execute_batch(super::records::SCHEMA)+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ super::records::install_reference_triggers(connection)+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ connection+ .execute(+ "UPDATE observation_meta SET value='5' WHERE key='schema_version'",+ [],+ )+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ Ok(())+}++fn create_schema_v5(connection: &Connection) -> Result<(), ObservationStoreError> {+ fn omit(mut sql: String, header: &str) -> String {+ let start = sql.find(header).expect("known legacy definition");+ let end = start + sql[start..].find(';').expect("known complete definition") + 1;+ sql.replace_range(start..end, "");+ sql+ }+ let auxiliary = omit(+ AUXILIARY_SCHEMA_V2.to_owned(),+ "CREATE TABLE IF NOT EXISTS observation_events",+ );+ let content = omit(+ CONVERSATION_CONTENT_SCHEMA_V4.to_owned(),+ "CREATE TABLE IF NOT EXISTS conversation_content_events_v2",+ );+ let content = omit(+ content,+ "CREATE TABLE IF NOT EXISTS observation_feedback_v2",+ );+ let content = omit(+ content,+ "CREATE INDEX IF NOT EXISTS observation_feedback_v2_lookup",+ );+ for ddl in [+ BASE_SCHEMA_V2,+ VALUE_SCHEMA_V2,+ auxiliary.as_str(),+ EXECUTION_FACT_SCHEMA_V3,+ super::records::SCHEMA,+ content.as_str(),+ ] {+ connection+ .execute_batch(ddl)+ .map_err(|_| ObservationStoreError::ActivityUnavailable)?;+ }+ super::records::install_reference_triggers(connection)+ .map_err(|_| ObservationStoreError::ActivityUnavailable)}fn create_schema_v2(connection: &Connection) -> Result<(), ObservationStoreError> {diff --git a/crates/observation/src/store/tests/blob_gc.rs b/crates/observation/src/store/tests/blob_gc.rsindex d1ba26c3..e5c433ea 100644--- a/crates/observation/src/store/tests/blob_gc.rs+++ b/crates/observation/src/store/tests/blob_gc.rs@@ -95,11 +95,12 @@ fn blob_gc_live_lookup_stays_linear_and_existing_databases_gain_index() {store.connection.lock()- // Both current indexes can satisfy the workspace/digest lookup. Remove- // both to reproduce the original unindexed query, not a new fast path.+ // Each current index can satisfy this lookup. Remove all alternatives+ // so the negative control still represents the original unindexed path..execute_batch("DROP INDEX content_instances_v2_live_blob;- DROP INDEX content_instances_v2_search_blob;",+ DROP INDEX content_instances_v2_search_blob;+ DROP INDEX content_instances_v2_live_blob_global;",).unwrap();// Negative control for the original correlated lookup. The current mergediff --git a/crates/observation/src/store/tests/content_v2.rs b/crates/observation/src/store/tests/content_v2.rsindex 4f26c7ab..f1c0242c 100644--- a/crates/observation/src/store/tests/content_v2.rs+++ b/crates/observation/src/store/tests/content_v2.rs@@ -209,14 +209,15 @@ fn direction_stream_persists_exact_metadata_and_returns_replay_stable_rich_ack());let connection = fixture.store.connection.lock();- let metadata: String = connection+ let metadata: Vec<u8> = connection.query_row("SELECT metadata_json FROM conversation_content_events_v2 WHERE sequence=2",[],|row| row.get(0),).unwrap();- let metadata: serde_json::Value = serde_json::from_str(&metadata).unwrap();+ let metadata: serde_json::Value =+ serde_json::from_slice(&crate::store::records::decode(&metadata).unwrap()).unwrap();assert!(metadata.get("canonical_bytes_base64").is_none());assert!(metadata@@ -240,7 +241,10 @@ fn direction_stream_persists_exact_metadata_and_returns_replay_stable_rich_ack()).unwrap();assert_eq!((chunk.1, chunk.2), (0, 5));- assert_eq!(&std::fs::read(chunk.0).unwrap()[0..5], b"hello");+ assert_eq!(+ &std::fs::read(fixture.store.object_path(&chunk.0).unwrap()).unwrap()[0..5],+ b"hello"+ );drop(connection);let detail = fixture@@ -523,7 +527,7 @@ fn replay_revalidates_durable_chunk_bytes_instead_of_trusting_feedback_alone() {row.get(0)}).unwrap();- std::fs::write(path, b"z").unwrap();+ std::fs::write(fixture.store.object_path(&path).unwrap(), b"z").unwrap();assert_eq!(fixture.store.ingest_content(&append, &[]).unwrap_err(),ObservationStoreError::Corrupt,@@ -535,7 +539,7 @@ fn replay_revalidates_durable_chunk_bytes_instead_of_trusting_feedback_alone() {}#[test]-fn schema_three_migrates_once_to_the_single_v2_content_schema() {+fn legacy_schema_three_component_normalization_keeps_the_single_v2_content_schema() {let root = tempfile::tempdir().unwrap();let connection = Connection::open(root.path().join("activity.db")).unwrap();connection@@ -563,6 +567,12 @@ fn schema_three_migrates_once_to_the_single_v2_content_schema() {.unwrap();drop(connection);+ // Component coverage for the retained historical schema normalizer. Real+ // startup seals this old store unchanged instead of invoking conversion.+ let mut historical = Connection::open(root.path().join("activity.db")).unwrap();+ super::super::schema::migrate_legacy(&mut historical).unwrap();+ super::super::schema::migrate(&mut historical).unwrap();+ drop(historical);let store = LocalObservationStore::open(root.path(), DigestAuthority::new([7; 32])).unwrap();let connection = store.connection.lock();let version: String = connection@@ -572,7 +582,7 @@ fn schema_three_migrates_once_to_the_single_v2_content_schema() {|row| row.get(0),).unwrap();- assert_eq!(version, "4");+ assert_eq!(version, "5");let v2_tables: i64 = connection.query_row("SELECT COUNT(*) FROM sqlite_master WHERE type='table'@@ -613,7 +623,7 @@ fn schema_three_migrates_once_to_the_single_v2_content_schema() {|row| row.get::<_, String>(0),).unwrap(),- "4",+ "5",);}diff --git a/crates/observation/src/store/tests/value_contract.rs b/crates/observation/src/store/tests/value_contract.rsindex 6ee882ee..5553a831 100644--- a/crates/observation/src/store/tests/value_contract.rs+++ b/crates/observation/src/store/tests/value_contract.rs@@ -150,7 +150,7 @@ fn value_query_uses_required_half_open_range() {}#[test]-fn v1_value_schema_migrates_once_and_reopens_without_conflated_rollups() {+fn legacy_value_normalizer_preserves_frozen_amounts_without_conflated_rollups() {let temporary = tempdir().unwrap();let fixture = Fixture::new("migration");let activity_path = temporary.path().join("activity.db");@@ -232,6 +232,12 @@ fn v1_value_schema_migrates_once_and_reopens_without_conflated_rollups() {.unwrap();drop(connection);+ // Keep the historical ledger normalization arithmetic regression as a+ // component check. Production upgrade archives old observations unchanged.+ let mut historical = Connection::open(&activity_path).unwrap();+ super::super::schema::migrate_legacy(&mut historical).unwrap();+ super::super::schema::migrate(&mut historical).unwrap();+ drop(historical);let store = Arc::new(LocalObservationStore::open(temporary.path(), authority()).unwrap());let migrated = store.get_value(@@ -247,11 +253,11 @@ fn v1_value_schema_migrates_once_and_reopens_without_conflated_rollups() {assert_eq!(migrated.routing_savings_micros, None);assert_eq!(migrated.entitlement_savings_micros, Some(600));assert_eq!(migrated.estimated_total_savings_micros, None);- assert_eq!(schema_version(&store), "4");+ assert_eq!(schema_version(&store), "5");drop(store);let reopened = open_store(temporary.path());- assert_eq!(schema_version(&reopened), "4");+ assert_eq!(schema_version(&reopened), "5");let after_restart = reopened.get_value(&fixture.workspace,diff --git a/crates/observation/src/store/upgrade.rs b/crates/observation/src/store/upgrade.rsnew file mode 100644index 00000000..b8bf8a7c--- /dev/null+++ b/crates/observation/src/store/upgrade.rs@@ -0,0 +1,122 @@+//! Format admission only. Production startup seals legacy history before opening+//! the fresh current writer; historical rows and objects are never converted here.+#[cfg(test)]+mod tests;+use crate::writer::ObservationStoreError as Error;+use rusqlite::{Connection, OpenFlags, OptionalExtension};+use std::{+ fs,+ path::{Path, PathBuf},+};++pub(crate) fn version(root: &Path) -> Result<Option<String>, Error> {+ if !root.join("activity.db").exists() {+ return Ok(None);+ }+ let db =+ Connection::open_with_flags(root.join("activity.db"), OpenFlags::SQLITE_OPEN_READ_ONLY)+ .map_err(|_| Error::Corrupt)?;+ let exists: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name='observation_meta' AND type='table')", [], |r|r.get(0)).map_err(|_|Error::Corrupt)?;+ if !exists {+ return Err(Error::Corrupt);+ }+ db.query_row(+ "SELECT value FROM observation_meta WHERE key='schema_version'",+ [],+ |r| r.get(0),+ )+ .optional()+ .map_err(|_| Error::Corrupt)?+ .map(Some)+ .ok_or(Error::Corrupt)+}++fn paths(root: &Path) -> Result<(PathBuf, PathBuf, PathBuf), Error> {+ let parent = root.parent().ok_or(Error::Io)?;+ let name = root+ .file_name()+ .ok_or(Error::Io)?+ .to_str()+ .ok_or(Error::Io)?;+ Ok((+ parent.join(format!("{name}.pre-v5-source")),+ parent.join(format!("{name}.v5-target")),+ parent.join(format!("{name}.upgrade.json")),+ ))+}++pub(crate) fn required(root: &Path) -> Result<bool, Error> {+ match fs::symlink_metadata(root) {+ Ok(meta) if !meta.is_dir() || meta.file_type().is_symlink() => return Err(Error::Corrupt),+ Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(Error::Io),+ _ => {}+ }+ let (old, target, journal) = paths(root)?;+ if journal.exists() || target.exists() {+ // An earlier converter may have retained real target data. Preserve all+ // generations for whole-group recovery rather than guessing a reset.+ return Err(Error::Corrupt);+ }+ if old.exists() && !root.exists() {+ return Err(Error::Corrupt);+ }+ match version(root)?.as_deref() {+ None => {+ if root.exists() {+ for entry in fs::read_dir(root).map_err(|_| Error::Io)? {+ let entry = entry.map_err(|_| Error::Io)?;+ if entry.file_name() == ".DS_Store" {+ continue;+ }+ if entry.file_name() != "conversation-content"+ || !entry.file_type().map_err(|_| Error::Io)?.is_dir()+ {+ return Err(Error::Corrupt);+ }+ for child in fs::read_dir(entry.path()).map_err(|_| Error::Io)? {+ let child = child.map_err(|_| Error::Io)?;+ if !matches!(child.file_name().to_str(), Some("staging" | "blobs"))+ || !child.file_type().map_err(|_| Error::Io)?.is_dir()+ || fs::read_dir(child.path())+ .map_err(|_| Error::Io)?+ .next()+ .is_some()+ {+ return Err(Error::Corrupt);+ }+ }+ }+ }+ Ok(false)+ }+ Some("5") => {+ validate_tables(root, true)?;+ Ok(false)+ }+ Some("4") => {+ validate_tables(root, false)?;+ Ok(true)+ }+ Some("1" | "2" | "3") => Ok(true),+ _ => Err(Error::Corrupt),+ }+}++fn validate_tables(root: &Path, current: bool) -> Result<(), Error> {+ let db =+ Connection::open_with_flags(root.join("activity.db"), OpenFlags::SQLITE_OPEN_READ_ONLY)+ .map_err(|_| Error::Corrupt)?;+ let count:u64=db.query_row("SELECT COUNT(*) FROM sqlite_schema WHERE type='table' AND name IN('logical_requests','observation_events','observation_checkpoints','observation_feedback_v2','conversation_content_events_v2','content_blobs_v2','content_instances_v2')",[],|r|r.get(0)).map_err(|_|Error::Corrupt)?;+ if count != 7 {+ return Err(Error::Corrupt);+ }+ if current && !db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='observation_streams_v5')",[],|r|r.get::<_,bool>(0)).map_err(|_|Error::Corrupt)?{return Err(Error::Corrupt);}+ Ok(())+}++pub(crate) fn prepare(root: &Path) -> Result<(), Error> {+ if required(root)? {+ return Err(Error::Corrupt);+ }+ Ok(())+}diff --git a/crates/observation/src/store/upgrade/tests.rs b/crates/observation/src/store/upgrade/tests.rsnew file mode 100644index 00000000..0c42946a--- /dev/null+++ b/crates/observation/src/store/upgrade/tests.rs@@ -0,0 +1,153 @@+use super::*;++fn legacy(root: &Path, version: &str) {+ fs::create_dir_all(root).unwrap();+ let db = Connection::open(root.join("activity.db")).unwrap();+ db.execute_batch("CREATE TABLE observation_meta(key TEXT PRIMARY KEY,value TEXT); CREATE TABLE logical_requests(id TEXT); CREATE TABLE observation_events(id TEXT); CREATE TABLE observation_checkpoints(id TEXT); CREATE TABLE observation_feedback_v2(id TEXT); CREATE TABLE conversation_content_events_v2(id TEXT); CREATE TABLE content_blobs_v2(id TEXT); CREATE TABLE content_instances_v2(id TEXT);").unwrap();+ db.execute(+ "INSERT INTO observation_meta VALUES('schema_version',?1)",+ [version],+ )+ .unwrap();+ db.execute(+ "INSERT INTO observation_events VALUES('original history')",+ [],+ )+ .unwrap();+ fs::write(root.join("original-object"), b"retained bytes").unwrap();+}++#[test]+fn old_formats_are_classified_for_sealing_without_conversion_or_new_files() {+ for version in ["1", "2", "3", "4"] {+ let directory = tempfile::tempdir().unwrap();+ let root = directory.path().join("observation");+ legacy(&root, version);+ let before = fs::read(root.join("activity.db")).unwrap();+ assert!(required(&root).unwrap());+ assert!(prepare(&root).is_err());+ assert_eq!(fs::read(root.join("activity.db")).unwrap(), before);+ assert_eq!(+ fs::read(root.join("original-object")).unwrap(),+ b"retained bytes"+ );+ assert!(!root.join("conversation-content").exists());+ assert!(!paths(&root).unwrap().1.exists());+ }+}++#[test]+fn interrupted_converter_roots_and_unknown_formats_remain_preserved() {+ let directory = tempfile::tempdir().unwrap();+ let root = directory.path().join("observation");+ legacy(&root, "999");+ let before = fs::read(root.join("activity.db")).unwrap();+ assert!(prepare(&root).is_err());+ assert_eq!(fs::read(root.join("activity.db")).unwrap(), before);+ let (old, target, journal) = paths(&root).unwrap();+ fs::create_dir(&target).unwrap();+ fs::write(target.join("data"), b"real intermediate data").unwrap();+ fs::write(&journal, b"old converter progress").unwrap();+ assert!(required(&root).is_err());+ fs::rename(&root, &old).unwrap();+ assert!(prepare(&root).is_err());+ assert!(!root.exists());+ assert_eq!(+ fs::read(target.join("data")).unwrap(),+ b"real intermediate data"+ );+ assert_eq!(fs::read(old.join("activity.db")).unwrap(), before);+}++#[test]+fn missing_database_with_retained_files_is_never_fresh() {+ let directory = tempfile::tempdir().unwrap();+ let root = directory.path().join("observation");+ fs::create_dir_all(root.join("conversation-content/blobs")).unwrap();+ fs::write(+ root.join("conversation-content/blobs/retained"),+ b"do not reset",+ )+ .unwrap();+ assert!(prepare(&root).is_err());+ assert!(!root.join("activity.db").exists());+}++#[test]+fn current_history_and_capacity_policy_survive_reopen() {+ let directory = tempfile::tempdir().unwrap();+ let root = directory.path().join("observation");+ let authority = crate::DigestAuthority::new([7; 32]);+ let store = crate::LocalObservationStore::open(&root, authority.clone()).unwrap();+ store.set_capacity_limit(Some(20_000_000_000)).unwrap();+ store+ .connection+ .lock()+ .execute(+ "INSERT INTO observation_meta VALUES('retained_probe','original-current-data')",+ [],+ )+ .unwrap();+ drop(store);+ assert!(!required(&root).unwrap());+ let store = crate::LocalObservationStore::open(&root, authority).unwrap();+ assert_eq!(+ store+ .connection+ .lock()+ .query_row(+ "SELECT value FROM observation_meta WHERE key='retained_probe'",+ [],+ |row| row.get::<_, String>(0)+ )+ .unwrap(),+ "original-current-data"+ );+ assert_eq!(+ store.capacity_status().unwrap().max_active_bytes,+ Some(20_000_000_000)+ );+}++#[cfg(unix)]+#[test]+fn a_link_cannot_supply_the_live_observation_root() {+ let directory = tempfile::tempdir().unwrap();+ let original = directory.path().join("original");+ legacy(&original, "4");+ let root = directory.path().join("observation");+ std::os::unix::fs::symlink(&original, &root).unwrap();+ assert!(required(&root).is_err());+ assert_eq!(+ fs::read(original.join("original-object")).unwrap(),+ b"retained bytes"+ );+}++#[test]+fn fresh_initialization_preserves_unpublished_material_and_publishes_only_current_format() {+ let directory = tempfile::tempdir().unwrap();+ let root = directory.path().join("observation");+ let interrupted = directory.path().join(".observation-fresh-0000000000000000");+ fs::create_dir(&interrupted).unwrap();+ fs::write(+ interrupted.join("partial-db"),+ b"unpublished initialization",+ )+ .unwrap();+ let authority = crate::DigestAuthority::new([7; 32]);+ let store =+ crate::LocalObservationStore::open_fresh_after_seal(&root, authority.clone()).unwrap();+ assert_eq!(version(&root).unwrap().as_deref(), Some("5"));+ assert_eq!(+ fs::read(interrupted.join("partial-db")).unwrap(),+ b"unpublished initialization"+ );+ store.set_capacity_limit(Some(20_000_000_000)).unwrap();+ drop(store);+ let reopened = crate::LocalObservationStore::open_fresh_after_seal(&root, authority).unwrap();+ assert_eq!(+ reopened.capacity_status().unwrap().max_active_bytes,+ Some(20_000_000_000)+ );+}diff --git a/crates/observation/src/text_index/worker.rs b/crates/observation/src/text_index/worker.rsindex 820d5b9a..aa087f96 100644--- a/crates/observation/src/text_index/worker.rs+++ b/crates/observation/src/text_index/worker.rs@@ -206,7 +206,7 @@ fn next(store: &LocalObservationStore, after: &mut i64) -> Result<(Option<Buildilet Some((_, workspace, digest, path, expected_bytes, media)) = source else {return Ok((None, false));};- let file = File::open(path);+ let file = File::open(store.object_path(&path).map_err(|_| Error::Corrupt)?);store.connection.lock().execute("INSERT OR IGNORE INTO observation_text_index_v2(workspace,digest,state) VALUES(?1,?2,?3)",params![workspace,digest,if file.is_ok() {"building"} else {"failed"}],diff --git a/crates/product-session/src/commands.json b/crates/product-session/src/commands.jsonindex fc527cc6..24d80c23 100644--- a/crates/product-session/src/commands.json+++ b/crates/product-session/src/commands.json@@ -21,6 +21,23 @@"result": "Result<serde_json::Value, DesktopFailure>","host_only": false},+ {+ "name": "observation_capacity_get",+ "args": [],+ "result": "Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure>",+ "host_only": false+ },+ {+ "name": "observation_capacity_set",+ "args": [+ {+ "name": "input",+ "type": "hiroute_application_api::ObservationCapacityUpdateV1"+ }+ ],+ "result": "Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure>",+ "host_only": false+ },{"name": "test_classifier_decision","args": [diff --git a/crates/product-session/src/commands/dispatch.rs b/crates/product-session/src/commands/dispatch.rsindex 8c0d2e20..60dedb95 100644--- a/crates/product-session/src/commands/dispatch.rs+++ b/crates/product-session/src/commands/dispatch.rs@@ -249,6 +249,23 @@ pub async fn dispatch(.await?;serde_json::to_value(value).map_err(|_| "RESPONSE_DATA_INVALID".into())}+ "observation_capacity_get" => {+ let _: hiroute_application_api::ClientEmptyRequestV1 = serde_json::from_value(args)+ .map_err(|_| DesktopFailure::from("REQUEST_INVALID"))?;+ serde_json::to_value(observation_capacity_get(window, state).await?)+ .map_err(|_| "RESPONSE_DATA_INVALID".into())+ }+ "observation_capacity_set" => {+ #[derive(Deserialize)]+ #[serde(deny_unknown_fields)]+ struct Arguments {+ input: hiroute_application_api::ObservationCapacityUpdateV1,+ }+ let input: Arguments = serde_json::from_value(args)+ .map_err(|_| DesktopFailure::from("REQUEST_INVALID"))?;+ serde_json::to_value(observation_capacity_set(window, state, input.input).await?)+ .map_err(|_| "RESPONSE_DATA_INVALID".into())+ }"observation_delete" => {#[derive(Deserialize)]#[serde(deny_unknown_fields, rename_all = "camelCase")]diff --git a/crates/product-session/src/commands/observation.rs b/crates/product-session/src/commands/observation.rsindex b354e2a6..81fbfd83 100644--- a/crates/product-session/src/commands/observation.rs+++ b/crates/product-session/src/commands/observation.rs@@ -108,3 +108,45 @@ fn localized_confirmation()}}++pub async fn observation_capacity_get(+ window: &ClientContext<'_>,+ state: &CommandState<'_>,+) -> Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure> {+ observation_capacity_call(window, state, None).await+}+pub async fn observation_capacity_set(+ window: &ClientContext<'_>,+ state: &CommandState<'_>,+ input: hiroute_application_api::ObservationCapacityUpdateV1,+) -> Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure> {+ if input.max_active_bytes == Some(0) {+ return Err("REQUEST_INVALID".into());+ }+ observation_capacity_call(window, state, Some(input)).await+}+async fn observation_capacity_call(+ window: &ClientContext<'_>,+ state: &CommandState<'_>,+ update: Option<hiroute_application_api::ObservationCapacityUpdateV1>,+) -> Result<hiroute_domain::ObservationCapacityStatusV1, DesktopFailure> {+ main_window(window)?;+ let client = state+ .0+ .lock()+ .await+ .as_ref()+ .ok_or("RESIDENT_UNAVAILABLE")?+ .client+ .clone();+ let envelope = client+ .observation_capacity(&crate::random_id()?, update)+ .await?;+ if envelope.error.is_some() {+ let value = serde_json::to_value(envelope).map_err(|_| "RESPONSE_DATA_INVALID")?;+ return Err(DesktopFailure::backend(+ serde_json::from_value(value).map_err(|_| "RESPONSE_DATA_INVALID")?,+ ));+ }+ envelope.data.ok_or_else(|| "RESPONSE_DATA_MISSING".into())+}diff --git a/docs/code-map/architecture.md b/docs/code-map/architecture.mdindex 678975bc..93e7564c 100644--- a/docs/code-map/architecture.md+++ b/docs/code-map/architecture.md@@ -185,6 +185,24 @@ The shared product-session layer in `crates/product-session` owns Preview compos## Changing boundaries+Compact Observation format admission and fresh-generation initialization belong to+[`store/upgrade`](../../crates/observation/src/store/upgrade.rs) and the+[`store` factory](../../crates/observation/src/store/mod.rs); complete original-source+preservation remains in the existing+[startup coordinator](../../crates/local-storage/src/migrations/observation_source.rs).+The [sealed-source publisher](../../crates/local-storage/src/backup/source_snapshot/seal.rs)+uses same-filesystem directory publication without traversing historical contents.+The optional capacity threshold uses the+[existing maintenance lifecycle](../../crates/observation/src/maintenance.rs) and+[bounded retirement owner](../../crates/observation/src/store/capacity.rs).+Representative scenarios cover+[format admission](../../crates/observation/src/store/upgrade/tests.rs),+[interrupted sealing](../../crates/local-storage/src/backup/source_snapshot/seal/tests.rs),+[production startup/restart](../../crates/daemon/src/control/runtime/history_cutover_tests.rs),+and [shared survivors and rollback](../../crates/observation/src/store/capacity/tests.rs).+Operating and recovery details are in the+[Observation guide](../../crates/observation/README.md#compact-storage-and-capacity-retention).+Windows process defaults and private handle validation belong to[native security](../../crates/platform/src/windows/security.rs), with suspended-childinitialization in the [Job launcher](../../crates/platform/src/windows/job.rs).diff --git a/docs/release-contracts.md b/docs/release-contracts.mdindex 7b1a5154..11ba6d4a 100644--- a/docs/release-contracts.md+++ b/docs/release-contracts.md@@ -12,6 +12,37 @@ Do not downgrade an initialized data directory in place. Manual recovery requirecomplete pre-upgrade backup and the corresponding binary verified to read it; restorethe whole store group, not selected tables or files.+Observation format 5 changes the physical protocol carriers. On a supported+format 1–4 upgrade, startup preserves the original three-store group, keys,+auxiliary state and matching old package, then seals the stopped Observation+DB/WAL/SHM/object/staging directory with a same-filesystem rename. It starts a+fresh format-5 store with current configuration, credentials and routing. Old+observations are not converted or merged into current queries; history and+statistics begin at cutover. Current-format stores and their capacity policy are+preserved on ordinary restart.++Desktop reuses the verified previous package from its installation handoff. A+headless operator must retain the old package and provide its absolute directory+through `HIROUTE_RECOVERY_PACKAGE` on the first sealing startup. Missing keys or+recovery material stop startup without clearing data. Interrupted publication+resumes the original batch; missing or replaced archive material is not permission+to initialize an empty store. Unfinished older conversion generations remain+preserved for whole-group recovery.++Startup does not enumerate, hash, copy or normalize historical Observation+contents. The source manifest verifies copied recovery material and binds the+sealed directory's native identity; it does not claim eager full-content integrity+verification. This local recovery point still occupies disk and is not an+independent disaster-recovery copy. Protected sources/packages are excluded from+the active Observation threshold and are never deleted by its capacity policy.++For manual recovery, stop the daemon, preserve later new-format data separately,+check the old source offline, and restore its complete original storage group+and auxiliary state at the recorded original locations with the matching old+package, following the generated recovery guide. Never mix old business stores+with newer Observation data or restore only `activity.db`. An independent copy+and full-content check can be made offline, outside service startup.+Branch-route publications may contain `source_binding_id` in the persisted active,prepared or last-known-good Gateway aggregate. Version 0.3.1 and earlier strict readersreject this field, even though the enclosing contract remainsdiff --git a/docs/standalone-cli.md b/docs/standalone-cli.mdindex fe2f6691..06c92afd 100644--- a/docs/standalone-cli.md+++ b/docs/standalone-cli.md@@ -847,6 +847,34 @@ released v2 structured requests retain the priorresponse field set. Content-only deletion retains safe evidence; full fact deletionand retention remove it. These rates do not establish accuracy or model quality.+## Limit active local observation storage++The default detail retention remains seven days. Optionally save a capacity+threshold in bytes, or disable early capacity expiry:++```sh+hiroute observation storage show --output json+hiroute observation storage set --max-active-bytes 20000000000 --output json+hiroute observation storage set --disable --output json+```++These commands use the owner-authenticated local OS channel. A supplied delegated+or read-only grant cannot authorize a policy change.++The example is 20 decimal GB. Exceeding the threshold may expire the oldest+eligible completed-request detail before seven days. Active requests, live shared+content, replay/deletion barriers, non-content monetary aggregates and recovery+backups remain protected. Disabling stops future uncommitted early retirement;+it does not restore expired history.++This is a soft threshold: reclamation uses spaced, bounded batches and backs off+after contention or ineffective work. Initial inventory, pending cleanup or protected+data can delay physical release. `show` reports the saved limit, measured active+allocated bytes, measurement time and pending/blocked reason. Reusable SQLite+pages are not reported as released disk space. Protected source directories and+recovery packages have separate retention; their space must be budgeted for an+upgrade. See [whole-group recovery](release-contracts.md#upgrade-and-manual-recovery).+## Configure a Worker and delegate a taskWorker reuses the existing commands. Discovery does not install or select software:diff --git a/docs/standalone-cli.zh-CN.md b/docs/standalone-cli.zh-CN.mdindex 63a31bed..5704d5ab 100644--- a/docs/standalone-cli.zh-CN.md+++ b/docs/standalone-cli.zh-CN.md@@ -730,6 +730,28 @@ Session ID、v1 Receipt ID 与此定位符不同:且不得把未知金额或尚未形成账值行的 token 伪造为零成本。同一用户通过 Local Control 显式读取正文、搜索、catalog 与 ancestry 无需另取读取票据;显式传入的授权仍绑定准确操作和范围,运行范围受限的调用方仍保留原有边界。+## 设置本地观测存储容量++默认仍保留七天明细。可以额外设置容量阈值,单位为字节,也可以关闭容量提前回收:++```sh+hiroute observation storage show --output json+hiroute observation storage set --max-active-bytes 20000000000 --output json+hiroute observation storage set --disable --output json+```++这些命令通过操作系统认证的本地用户通道访问;显式传入的委派或只读授权不能修改策略。++示例为十进制 20 GB。超出后,最早的、已完成且允许回收的请求明细可能在七天前到期。+正在处理的请求、仍被引用的内容、防重放和删除屏障、非正文金额汇总以及恢复备份受到保护。+关闭后停止尚未提交的提前回收,不会恢复已经到期的历史。++这是软阈值:后台按间隔、分批回收,繁忙或无进展时延后重试,每轮工作有时间预算。+首次盘点、繁忙、待完成清理或受保护的数据均可能延迟磁盘释放。+`show` 返回有效配置、实测活动占用、测量时间及等待或阻塞原因;+数据库内部可复用页不计为已释放磁盘。旧源目录和恢复程序包单独保留,升级时需预留空间。+恢复必须遵循[完整存储组恢复规则](release-contracts.md#upgrade-and-manual-recovery)。+## 配置 Worker 并委派任务Worker 复用现有命令。发现不会安装或选择软件:diff --git a/e2e/product/schema/contract-set.v1.json b/e2e/product/schema/contract-set.v1.jsonindex 265fc50e..92beab84 100644--- a/e2e/product/schema/contract-set.v1.json+++ b/e2e/product/schema/contract-set.v1.json@@ -1,10 +1,10 @@{- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","files": {"dependency-gate.v1.json": "sha256:b8ac81155266dca921f159d1cfabec6605f2e4bab0320ad6b23a3dbee51e6fa1","product-action.v1.schema.json": "sha256:faf815b27fd292e564a7d51de20c5a2825abbe710315a2364a9932ae3b5e4b7e","product-oracle-evidence.v1.schema.json": "sha256:faab6582bf3fba307b0f9f2949c51726409a209b0fa62e847114e0c6301038c6",- "product-oracle-manifest.v1.json": "sha256:fccbfb2ac7d3dbae43b4e2886ab10e60e375f545ff40403020c13c9d0fa5e21d",+ "product-oracle-manifest.v1.json": "sha256:091693462ebf632d676e95112bb0020a63748302488a0578dcaf0fd62be5c24e","product-result.v1.schema.json": "sha256:72fc76be77572e40eed7516754911b45fdc9d55a8c089ba74c21427a33809def"},"schema_version": {diff --git a/e2e/product/schema/product-oracle-manifest.v1.json b/e2e/product/schema/product-oracle-manifest.v1.jsonindex c62af1b1..6da9aa4f 100644--- a/e2e/product/schema/product-oracle-manifest.v1.json+++ b/e2e/product/schema/product-oracle-manifest.v1.json@@ -1,5 +1,5 @@{- "descriptor_digest": "sha256:a07f0fba88c73d351be68cd0a845ac2499a22212246fe276fcf4973d424f0972",+ "descriptor_digest": "sha256:09627dbd802f299a49fbc1e104234cfbe91e2474fff7899a7c007bb3c0d1789a","expected_red_reasons": [],"planned_commands": ["console.open",@@ -93,6 +93,8 @@"sessions.delete.preview","sessions.delete.apply","value.show",+ "observation.storage.show",+ "observation.storage.set","observation.plan-quality.samples"],"planned_native_paths": [@@ -164,6 +166,8 @@"sessions.receipt","sessions.status","value.show",+ "observation.storage.show",+ "observation.storage.set","observation.plan-quality.samples"],"release_state": "released",diff --git a/issue-spec/specs/local-data-and-diagnostics/spec.md b/issue-spec/specs/local-data-and-diagnostics/spec.mdindex eee9ad1b..ac81f8a4 100644--- a/issue-spec/specs/local-data-and-diagnostics/spec.md+++ b/issue-spec/specs/local-data-and-diagnostics/spec.md@@ -122,6 +122,161 @@ continues through the existing three-store coordinator with platform-derived soufixtures; parser acceptance or a development-package upgrade is not proof of areleased Windows-to-Windows upgrade.+### Requirement: Share repeated observation storage without changing logical evidence++HiRoute SHALL store repeated retained content and common observation metadata through shared representations rather than independent complete copies for every occurrence. Equal content may share bytes only within the existing digest and authorization boundaries. Each request, message, content/event identity, original order, canonical digest, delivered-frame association, exact retained ACK/NACK meaning and actual model/usage attribution SHALL remain independently recoverable. Repeated input history SHALL retain distinct request references without inventing extra execution or usage. Physical sharing SHALL preserve the existing session-history, request-routing-history and usage-and-savings contracts, including authorized original-byte content access, tool identity/pairing, explicit gaps and provider-state visibility. Sharing SHALL NOT turn a partial observation into complete evidence or merge distinct conversations, attempts or workspaces.++#### Scenario: Repeated history shares storage++- **WHEN** many requests contain identical tool definitions, messages or tool results+- **THEN** the shared values are stored once per applicable storage identity with compact per-occurrence references, while each request retains its original order, identities, attribution and inspectable content.++#### Scenario: Stream events preserve their logical boundaries++- **WHEN** a delivered response contains small text/reasoning deltas and tool events+- **THEN** their physical representation may be shared or compacted but the original logical events, kinds, order, accepted-delivery associations and composed answer remain unchanged.++#### Scenario: Replay a retained event after compaction++- **WHEN** an accepted retained event is replayed after representation changes and restart+- **THEN** the receiver returns the same committed acknowledgement and checks its original digest/sequence identity; conflicting replays remain rejected and do not add usage or occurrences.++#### Scenario: Do not merge equal content across authority++- **WHEN** equal bytes appear in distinct logical conversations or authorization scopes+- **THEN** physical reuse does not merge identities, grant access, change visibility, or assign one occurrence the other occurrence's execution facts.++Source SPEC comments:+- https://github.com/higress-group/HiRoute/issues/151#issuecomment-6248901316++### Requirement: Compressed observations remain losslessly and boundedly accessible++HiRoute SHALL support lossless compressed local observation storage in independently readable bounded units. Original object bytes, content identifiers, canonical digests and original-byte search anchors SHALL remain stable independently of physical placement. An authorized bounded content read SHALL decode only the units necessary for its selected range rather than decompressing the entire history. Existing ingestion durability, nonblocking model delivery, reader admission, query deadlines and explicit incomplete/error behavior SHALL continue to apply. A damaged, truncated, unsupported or over-budget encoded unit SHALL produce an explicit affected-content/storage failure without fabricated empty content, successful integrity verification or loss of otherwise independent readable records. Compression SHALL preserve existing logical size limits and visibility rules; encoded length cannot authorize excessive decompression.++#### Scenario: Read an original range from compact data++- **WHEN** a user opens an existing text/tool-content reference or Unicode search anchor after its physical representation changed+- **THEN** the existing content interface returns the identical original range, digest, byte count and offsets under current authorization without scanning unrelated history.++#### Scenario: Read across a block boundary++- **WHEN** one logical object spans multiple compressed blocks+- **THEN** bounded continuation composes the exact original bytes in order, with no truncation, duplicated text or rewritten public identity.++#### Scenario: Interrupt compact publication++- **WHEN** the process stops between persisting compact bytes and committing their authoritative reference+- **THEN** acknowledged data in the previous committed representation remains recoverable, unpublished bytes never become successful evidence, and restart resumes or discards only owned incomplete work.++#### Scenario: Reject damaged or excessive encoded data++- **WHEN** a selected compressed block fails length, codec or integrity validation+- **THEN** the read reports its actual unavailable/corrupt state within existing resource bounds while unrelated valid objects remain readable; no oversized allocation follows an untrusted size header.++Source SPEC comments:+- https://github.com/higress-group/HiRoute/issues/151#issuecomment-6248901714++### Requirement: Reclaim obsolete observation representations through one lifecycle++HiRoute SHALL apply the existing confirmed deletion and seven-day detail-retention scope to every obsolete representation introduced or retained by observation storage, including content objects, event metadata, feedback/deduplication payloads, dead physical ranges where introduced, and rebuildable indices. Compact tombstones, sequence/checkpoint and conflict evidence needed to prevent replay resurrection SHALL remain authoritative. Reclamation SHALL release eligible physical storage through bounded resumable maintenance without waiting for an otherwise live stream or shared physical container to become wholly obsolete. It SHALL preserve every live reference and lawful longer-lived non-content monetary aggregate, distinguish content-only from facts-and-content deletion, and retain replay evidence until a provable retirement boundary permits its removal. Ineligibility or failed reclamation SHALL NOT be reported as completed physical release. Age alone SHALL NOT make active or unproven protocol evidence disposable, and compaction/replay SHALL NOT restore deleted bodies, membership links or precise expired detail.++#### Scenario: Delete one object in shared physical storage++- **WHEN** a confirmed deletion removes one request while other requests reference shared content or the same physical storage+- **THEN** deleted content becomes immediately inaccessible and maintenance eventually reclaims its eligible obsolete representation while retaining exact live bytes and current references.++#### Scenario: Expire old requests in an active stream++- **WHEN** some requests reach detail retention while later requests under the same producer stream remain retained+- **THEN** eligible old content/event/feedback/deduplication storage is reclaimed without waiting for the active stream to end, preserving required retirement evidence and later requests.++#### Scenario: Replay after retirement++- **WHEN** an old event or cached reference arrives after its body or facts were deleted or expired+- **THEN** existing barriers and retained retirement proof prevent resurrection or duplicate charging; missing retired content cannot receive a fabricated successful content acknowledgement.++#### Scenario: Content-only deletion retains monetary meaning++- **WHEN** the user clears body while retaining permitted request facts+- **THEN** body-bearing copies and references are removed from all relevant representations, while retained route facts, grouping and monetary evidence keep their existing semantics.++#### Scenario: Resume interrupted reclamation++- **WHEN** compact-storage publication, shared-object collection or physical page reclamation is interrupted or fails+- **THEN** logical visibility remains enforced and bounded maintenance resumes from durable state without deleting a live reference or falsely reporting released space.++Source SPEC comments:+- https://github.com/higress-group/HiRoute/issues/151#issuecomment-6248902164++### Requirement: Seal old observation history and start a fresh generation recoverably++HiRoute SHALL preserve supported old Observation history unchanged in a protected local archive and start a fresh current-format Observation store during upgrade. Startup SHALL retain the complete original storage group, keys, auxiliary state and matching old program before the cutover, preserve current configuration, credentials and routing, and avoid work proportional to old observation rows, body bytes or file count: no full historical copy, hashing, recompression, normalization or index rebuilding. The original Observation DB, WAL/SHM, objects and staging SHALL stay together through a durable same-filesystem namespace transition. Current history and statistics begin at cutover; old observations are available by whole-group matching-version recovery rather than silently combined into current queries. Current-format data and policy SHALL survive ordinary restart. Interrupted cutover SHALL resume the same original batch or expose actionable preserved-source recovery; missing recovery material, insufficient work space, unsupported/corrupt metadata or failed moves SHALL preserve the source and stop the affected startup. An old local archive is not an independent backup or freed disk space. Active capacity retention SHALL NOT reclaim it. Manual recovery SHALL preserve newer data separately and restore the complete corresponding old group with its matching old program.++#### Scenario: Upgrade a large old history++- **WHEN** a supported instance with retained old-format observations upgrades+- **THEN** the complete old history remains unchanged and recoverable, current configuration/credentials/routing continue, and service starts with a fresh current-format history without processing all historical rows or files; current history/statistics start at cutover.++#### Scenario: Interrupt the directory cutover++- **WHEN** the process stops before or after sealing the old directory or publishing its recovery material+- **THEN** restart uses the same durable batch and complete source, never mistakes a missing/partial live root for permission to discard history, and finishes the cutover or supplies actionable recovery guidance.++#### Scenario: Fail archive admission or publication++- **WHEN** recovery material or keys are unavailable, small-work space is insufficient, an input/phase is unsupported or corrupt, or the same-filesystem move fails+- **THEN** startup reports the failure, preserves existing source and progress, and does not fall back to a full historical copy or delete records/backups to proceed.++#### Scenario: Reopen a current-format store++- **WHEN** the upgraded instance restarts after recording new observations+- **THEN** it preserves those records and its current capacity policy, without resealing or clearing the current generation or scanning old archives.++#### Scenario: Recover old observations++- **WHEN** the owner chooses matching-version recovery+- **THEN** the complete original group, original locations and matching old program are restored together while newer data remains separately preserved; the original observations retain their bytes, identities and monetary facts.++Source SPEC comments:+- https://github.com/higress-group/HiRoute/issues/151#issuecomment-6248902605++### Requirement: Optional capacity retention uses restrained observation maintenance++HiRoute SHALL allow the local installation owner to save or disable one durable active-observation capacity threshold. The default threshold SHALL be 20 GB (20,000,000,000 bytes), including a fresh generation after old-history sealing. The owner may change it or explicitly disable capacity expiry; a disabled choice SHALL remain disabled across restart. Seven-day age retention remains independent. When enabled and exceeded, the existing lifecycle SHALL retire oldest eligible completed-request detail before seven days, preserving in-flight writes, live shared references, deletion/replay barriers, lawful non-content aggregates and recovery backups. Retained observations remain lossless; policy removals SHALL be distinct from representation savings and actual physical release. Accounting/reclamation SHALL use spaced bounded work, yielding and idle/failure/no-progress backoff rather than sustained high CPU or an unbounded catch-up loop. HiRoute SHALL report actual measured usage and pending/blocked release with its reason, without promising an instantaneous hard cap, reporting reusable pages as released, or dropping acknowledged writes. Changes/disabling affect uncommitted future retirement and do not restore already expired history.++#### Scenario: Default is 20 GB with independent age retention++- **WHEN** the owner has not saved a capacity preference, including after a fresh-generation upgrade+- **THEN** the effective threshold is 20,000,000,000 bytes and eligible detail may expire early when it is exceeded; ordinary seven-day expiry still applies independently.++#### Scenario: Explicit disable survives restart++- **WHEN** the owner disables capacity expiry and restarts HiRoute+- **THEN** the effective threshold remains disabled rather than reverting to 20 GB; capacity pressure alone does not authorize early expiry, while ordinary age retention continues.++#### Scenario: Young eligible history can expire++- **WHEN** a saved threshold is exceeded while completed eligible requests are younger than seven days+- **THEN** oldest eligible request detail is retired in spaced bounded batches through current visibility/reference rules while active requests, shared survivors, lawful aggregates and backups remain protected.++#### Scenario: Sustained pressure cannot busy-spin++- **WHEN** continued writes, contention or failed or ineffective release keeps usage above the threshold+- **THEN** maintenance yields and backs off between bounded passes, reports pending/blocked overage, and never continuously consumes CPU or drops acknowledged writes to force the limit.++#### Scenario: Physical release is reported honestly++- **WHEN** retired detail is hidden but shared survivors or reusable database pages prevent immediate file shrink+- **THEN** status distinguishes policy deletion from released bytes and does not count removed requests as a lossless compression gain.++#### Scenario: Restart and disable preserve policy meaning++- **WHEN** a saved valid policy survives restart and is later disabled+- **THEN** its effective value is restored on restart, further uncommitted early expiry stops after disabling, and ordinary retention/replay barriers remain without resurrecting already expired data.++Source SPEC comments:+- https://github.com/higress-group/HiRoute/issues/151#issuecomment-6252214214+## Implementation references- [Startup format admission](../../../crates/local-storage/src/migrations/startup_format.rs),