Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(observability): split error classification into focused modules

Steven Enamakel65be83e35c6c
diff --git a/crates/openhuman-core/src/core/observability.rs b/crates/openhuman-core/src/core/observability.rs
index 261e7a9f63..53a9f6645e 100644
--- a/crates/openhuman-core/src/core/observability.rs
+++ b/crates/openhuman-core/src/core/observability.rs
@@ -367,2985 +367,7 @@ pub enum ExpectedErrorKind {
WalletNotConfigured,
}
-pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
- let lower = message.to_ascii_lowercase();
- // Explicit host ownership/reportability metadata precedes text heuristics.
- let backend_error = crate::inference::registry::classify_backend_error(None, message);
- if backend_error
- .as_ref()
- .is_some_and(|classification| classification.backend_owned && !classification.reportable)
- {
- return Some(ExpectedErrorKind::BackendErrorCodeOwned);
- }
- // Explicitly reportable guard failures must bypass broader text demotions.
- if backend_error.as_ref().is_some_and(|classification| {
- classification.provider_owned
- && classification.reportable
- && matches!(
- classification.kind,
- Some(
- crate::inference::registry::BackendErrorKind::PayloadTooLarge
- | crate::inference::registry::BackendErrorKind::ContextLengthExceeded
- )
- )
- }) {
- return None;
- }
- // Check the Codex-CLI import envelope first: it is highly specific
- // (literal `codex cli auth` / `.codex/auth.json`) and carries no overlap
- // with the generic matchers below, so ordering is for clarity, not
- // precedence. See `ExpectedErrorKind::CodexCliAuthUnavailable`. The
- // keyring/persist failure path (`upsert_profile`) stringifies to generic
- // keychain / "auth profile" / SQLite text that contains neither anchor,
- // so a real defect in the import still falls through to capture.
- if lower.contains("codex cli auth") || lower.contains(".codex/auth.json") {
- return Some(ExpectedErrorKind::CodexCliAuthUnavailable);
- }
- // TAURI-RUST-5EH — `approval_decide` resolved a request_id whose row was
- // already decided / lazily expired / superseded (benign race per the
- // inline-approvals design spec). `rpc::approval_decide` emits this exact
- // `"no pending approval found"` wording ONLY after `store::get_decision`
- // confirms a persisted decision exists; the genuine never-registered case
- // raises `"no pending approval ever registered"` (matched by neither this
- // arm nor any below), so a real lost-registration defect still reaches
- // Sentry. See `ExpectedErrorKind::ApprovalNoPendingRace`.
- if lower.contains("no pending approval found") {
- return Some(ExpectedErrorKind::ApprovalNoPendingRace);
- }
- // TAURI-RUST-CGP — a remote MCP server answered the connect handshake with
- // HTTP 401 (`McpUnauthorizedError`). `connections::connect` already stores a
- // `needs_auth` flag so the UI prompts for OAuth sign-in (#3733 / #3719), but
- // the `mcp_clients_connect` RPC still re-raises the stringified error here.
- // It is preventable user-state (the server needs sign-in) with no
- // Sentry-actionable signal — demote it. Highly specific anchor; no overlap
- // with the generic matchers below. See `is_mcp_server_needs_auth_message`.
- if is_mcp_server_needs_auth_message(&lower) {
- return Some(ExpectedErrorKind::McpServerNeedsAuth);
- }
- // #5805 — a wallet-derived key was needed and the user has no wallet. An
- // optional feature nobody enabled is not a defect. Placed beside the other
- // highly-specific anchors: the needle is a full sentence produced by one
- // constant, so it cannot collide with the generic matchers below, and
- // ordering here is for clarity rather than precedence.
- if is_wallet_not_configured_message(&lower) {
- return Some(ExpectedErrorKind::WalletNotConfigured);
- }
- // TAURI-RUST-117K et al. — see `ExpectedErrorKind::ModuleUnavailable`.
- if is_module_unavailable_message(message) {
- return Some(ExpectedErrorKind::ModuleUnavailable);
- }
- if lower.contains("local ai is disabled") {
- return Some(ExpectedErrorKind::LocalAiDisabled);
- }
- // `"no api key is configured"` covers the composio direct-mode factory
- // bail (`client.rs`: "composio direct mode selected but no api key is
- // configured"). `composio_list_connections` now short-circuits that
- // state to an empty list, so the dominant 5 s-poll leak (TAURI-RUST-R4)
- // no longer reaches here — but other direct-mode ops the user invokes
- // explicitly (execute / authorize) can still surface it, and it is the
- // same user-config state with no Sentry-actionable signal.
- //
- // `"no api key supplied"` is Cohere's exact 401 body wording when the
- // hosted `/v2/embed` endpoint is called with an empty bearer (the BYO-key
- // embedding path with no key configured). The `CohereEmbedding::embed`
- // guard now fast-fails before issuing that request, but this arm demotes
- // any residual 401 of that shape — older clients, or a present-but-rejected
- // key — so the flood (TAURI-RUST-52S) stays out of Sentry either way.
- if is_api_key_unset_message(&lower) {
- return Some(ExpectedErrorKind::ApiKeyMissing);
- }
- // Check `ChannelSupervisorRestart` BEFORE `is_loopback_unavailable` and
- // `is_network_unreachable_message`: the supervisor wrapper contains
- // substrings (`error sending request for url`, OS-localized WSAETIMEDOUT
- // bodies, occasionally `connection refused`) that would otherwise classify
- // as `NetworkUnreachable` (which only demotes to `warn!` — still a Sentry
- // event) or `LoopbackUnavailable`. The supervisor's own restart loop
- // handles the condition; per-restart messages carry no actionable Sentry
- // signal (TAURI-RUST-15 / -BB). Sustained outages still surface via
- // `health.bus` / `FAIL_ESCALATE_THRESHOLD`, which is a separate path.
- if is_channel_supervisor_restart_message(&lower) {
- return Some(ExpectedErrorKind::ChannelSupervisorRestart);
- }
- // Check `is_loopback_unavailable` BEFORE `is_network_unreachable_message`:
- // a loopback `Connection refused` body shape would otherwise demote to the
- // broader `NetworkUnreachable` bucket and lose the boot-window vs.
- // user-environment distinction. Mirrors the `ProviderUserState`-before-
- // `BackendUserError` precedence pattern from #1795 (PR comment).
- if is_loopback_unavailable(&lower) {
- return Some(ExpectedErrorKind::LoopbackUnavailable);
- }
- // Check `is_ollama_user_config_rejection` BEFORE the generic network /
- // backend-error matchers: the GX "daemon unreachable at localhost" shape
- // contains a loopback host but no `Connection refused (os error …)`
- // marker, and the XS / MA / KM 400/404 shapes are pure user-config —
- // wrong model name, model not pulled, daemon opted-in but not running.
- // Route them to the dedicated arm so they share the `ProviderUserState`
- // bucket with the composio / OAuth user-state errors instead of falling
- // through to capture. See `is_ollama_user_config_rejection`.
- if is_ollama_user_config_rejection(&lower) {
- return Some(ExpectedErrorKind::ProviderUserState);
- }
- if is_network_unreachable_message(&lower) {
- return Some(ExpectedErrorKind::NetworkUnreachable);
- }
- // Check `is_upstream_edge_block_message` BEFORE the transient/backend
- // matchers: an HTML 403 gateway page from a provider embed call carries a
- // `403`/`forbidden` token that no other matcher claims, but routing it to
- // its own bucket keeps "edge/CDN block" distinct from genuine transient
- // 5xx and from the actionable JSON 4xx that must still page (TAURI-RUST-8S3).
- if is_upstream_edge_block_message(&lower) {
- return Some(ExpectedErrorKind::UpstreamEdgeBlock);
- }
- if is_transient_upstream_http_message(&lower) {
- return Some(ExpectedErrorKind::TransientUpstreamHttp);
- }
- if lower.contains("binary not found") {
- return Some(ExpectedErrorKind::LocalAiBinaryMissing);
- }
- // Check `is_provider_user_state_message` BEFORE `is_backend_user_error_message`:
- // composio's "Toolkit X is not enabled" lands as a 4xx that both would
- // match, and the more specific `ProviderUserState` bucket is the right
- // home — see the variant doc-comment for OPENHUMAN-TAURI-… coverage.
- if is_provider_user_state_message(&lower) {
- return Some(ExpectedErrorKind::ProviderUserState);
- }
- // TAURI-RUST-8FQ — the OpenAI ChatGPT/Codex OAuth access token expired with
- // no usable refresh token. The provider HTTP layer
- // (`provider::http_error::api_error` / `chat_via_responses`) already demotes its
- // own per-attempt event, but the same `anyhow::bail!` string re-raises here
- // at the RPC boundary (`jsonrpc` → `report_error_or_expected`); route it to
- // the shared `ProviderUserState` bucket so the re-report is demoted too
- // instead of leaking the event the emit-site already suppressed. User-state
- // — recovery is reconnecting OpenAI; Sentry has no remediation path. The
- // markers are distinct from the backend "invalid token" session-expiry
- // wording matched below, so this does not shadow that arm.
- if tinyinference_providers::is_openai_oauth_session_expired_message(message) {
- return Some(ExpectedErrorKind::ProviderUserState);
- }
- // TAURI-RUST-5MV — ollama.com hosted-inference 500 (`Internal Server Error
- // (ref: <uuid>)`) for `*:cloud` models. The provider HTTP layer
- // (`native_chat` / `streaming_chat` / `api_error`) already demotes its own
- // per-attempt event and re-raises the actionable
- // "Ollama cloud is temporarily unavailable …" string; this catches the
- // re-report at the agent / RPC boundary (`provider_chat` →
- // `report_error_or_expected`, the `domain=agent` half of the flood). Routed
- // to `TransientUpstreamHttp` — it is an external upstream 5xx the
- // reliable-provider layer retries + falls back over, with no client lever.
- // Delegates to the single-source provider matcher so the phrasing can't
- // drift. Distinct anchor from the matchers above, so it shadows nothing.
- if tinyinference_llm::failure::is_ollama_cloud_internal_500_message(message) {
- return Some(ExpectedErrorKind::TransientUpstreamHttp);
- }
- if is_backend_user_error_message(&lower) || is_plan_required_message(message) {
- return Some(ExpectedErrorKind::BackendUserError);
- }
- // Check `is_session_expired_message` BEFORE `is_embedding_backend_auth_failure`:
- // the OpenHuman-backend embedding 401 "Invalid token" envelope
- // (`Embedding API error (401 …): {"error":"Invalid token"}`) is a
- // recoverable session expiry (TAURI-RUST-4K5, #2786), not a generic
- // backend error. The broader `is_embedding_backend_auth_failure` matcher
- // below would otherwise demote that exact wire shape to `BackendUserError`
- // first and swallow the re-auth signal. `is_session_expired_message` is
- // narrowly anchored (parenthesised `(401` + the `"error":"Invalid token"`
- // envelope), so the bare-status `Embedding API error 401 …` shape and
- // BYO-key 401s still fall through to the matchers below.
- if is_session_expired_message(message) {
- return Some(ExpectedErrorKind::SessionExpired);
- }
- // The backend rejected the stored TinyHumans API key (401 on an api-key
- // credential, `BackendApiError::ApiKeyRejected`). User-state — the key was
- // revoked or mistyped and only a new key recovers it — so it shares the
- // credential-lapse bucket. Kept out of `is_session_expired_message` on
- // purpose: that predicate feeds the `SessionExpired` publish, which must
- // not clear a session because an API key failed.
- if is_api_key_rejected_message(message) {
- return Some(ExpectedErrorKind::SessionExpired);
- }
- if is_embedding_backend_auth_failure(&lower) {
- return Some(ExpectedErrorKind::SessionExpired);
- }
- // TAURI-RUST-5JR — a custom embeddings endpoint with no embeddings route
- // (the user pointed the Custom (OpenAI-compatible) provider at a chat-only
- // base URL, e.g. DeepSeek, which 404s every `/embeddings` POST).
- // Deterministic user-config state, re-emitted on every memory re-embed;
- // the embeddings settings UI surfaces an actionable "pick an
- // embeddings-capable provider" message. Demote to info. Scoped to 404/405
- // only so a real 500 from a valid embeddings endpoint stays in Sentry.
- if is_embedding_endpoint_absent(&lower) {
- return Some(ExpectedErrorKind::ProviderConfigRejection);
- }
- // TAURI-RUST-9SK — the user entered a non-embedding (chat) model id as the
- // embeddings model (e.g. an OpenRouter `…:free` chat model), so the
- // embeddings endpoint 400s `Model <id> does not exist` on every memory
- // re-embed (2205 events / 1 user). OpenRouter's bare `"does not exist"` +
- // integer `"code":400` body matches none of the chat-side phrases in
- // `is_provider_config_rejection_message` (which key on the OpenAI-native
- // `"does not exist or you do not have access"` / `model_not_found`), so
- // without this it reaches Sentry. Deterministic user-config state; the
- // settings UI surfaces an actionable "pick an embeddings-capable model"
- // remediation. Scoped to the 400 model-rejection body so a real 400
- // (oversized input, server fault) stays visible — same polarity contract as
- // `is_embedding_endpoint_absent`.
- if is_embedding_model_rejected(&lower) {
- return Some(ExpectedErrorKind::ProviderConfigRejection);
- }
- // Provider config-rejection (unknown model / abstract tier leaked to a
- // custom provider / model-specific temperature). Body-shape based and
- // intrinsically scoped to third-party providers — the OpenHuman
- // backend never emits these phrases. See the predicate's polarity
- // contract. Drops OPENHUMAN-TAURI-WJ / -QW / -HB / -NH re-reports
- // (#2079 / #2076 / #2202).
- if tinyinference_providers::is_provider_config_rejection_message(message) {
- return Some(ExpectedErrorKind::ProviderConfigRejection);
- }
- if is_local_ai_capability_unavailable_message(&lower) {
- return Some(ExpectedErrorKind::LocalAiCapabilityUnavailable);
- }
- if tinyinference_providers::is_budget_exhausted_message(message) {
- return Some(ExpectedErrorKind::BudgetExhausted);
- }
- if is_backend_unavailable_message(message) {
- return Some(ExpectedErrorKind::BackendUnavailable);
- }
- if is_prompt_injection_blocked_message(&lower) {
- return Some(ExpectedErrorKind::PromptInjectionBlocked);
- }
- // Context-window-exceeded re-report from a higher layer (agent /
- // web_channel). The provider api_error cascade suppresses its own
- // emit; this catches the re-raise. Delegates to the single-source
- // provider matcher so the phrasing can't drift. Runs last so a more
- // specific matcher always wins.
- if tinyinference_llm::failure::is_context_window_exceeded_message(message) {
- return Some(ExpectedErrorKind::ContextWindowExceeded);
- }
- if is_disk_full_message(&lower) {
- return Some(ExpectedErrorKind::DiskFull);
- }
- if is_windows_file_system_limitation_message(&lower) {
- return Some(ExpectedErrorKind::WindowsFileSystemLimitation);
- }
- if is_config_load_timed_out_message(&lower) {
- return Some(ExpectedErrorKind::ConfigLoadTimedOut);
- }
- // OS-level config-read denial on a file that exists (user-environment, zero
- // local lever). Keyed on the config-read anchor + an access-denied/locked io
- // signal; NotFound / unseen kinds fall through and keep paging. Requires the
- // loader to surface the full io chain (#3962).
- if is_config_read_io_failure_message(&lower) {
- return Some(ExpectedErrorKind::ConfigReadIoFailure);
- }
- // Empty-provider-response re-report from the web-channel layer. Runs
- // last so an earlier, more specific matcher always wins. See the
- // variant doc-comment and [`is_empty_provider_response_message`] for
- // the two-emit-site rationale (agent layer is handled by the typed
- // `AgentError::skips_sentry()` in PR #2790; this covers the
- // web_channel re-report where the type was flattened to a String).
- if is_empty_provider_response_message(&lower) {
- return Some(ExpectedErrorKind::EmptyProviderResponse);
- }
- // RPC-level filesystem path validation — explicit wire-shape anchors
- // (root_path / hosted path) prevent accidental demotion of unrelated
- // errors. See the variant doc-comment and
- // [`is_filesystem_user_path_invalid_message`] polarity contract.
- if is_filesystem_user_path_invalid_message(&lower) {
- return Some(ExpectedErrorKind::FilesystemUserPathInvalid);
- }
- None
-}
-
-/// Detect filesystem-out-of-space errors that bubble up from any syscall
-/// (`open`, `write`, `mkdir`, `rename`). Three platform-stable renderings:
-///
-/// - **POSIX `ENOSPC`** (Linux / macOS / BSD): `std::io::Error` renders as
-/// `"No space left on device (os error 28)"`. The errno-name substring is
-/// what we anchor on — case-folded to `"no space left on device"`.
-/// - **Windows `ERROR_DISK_FULL` (112)**: `std::io::Error` renders as
-/// `"There is not enough space on the disk. (os error 112)"`. Anchor on
-/// `"not enough space on the disk"`.
-/// - **Windows `ERROR_HANDLE_DISK_FULL` (39)**: same wire text but errno 39.
-/// The text anchor already covers it.
-///
-/// A fourth shape comes from call sites that render the `io::ErrorKind` debug
-/// name instead of the `io::Error` Display — notably the auth-profile
-/// lock-create annotation, which emits
-/// `"... (kind=Some(StorageFull), os_code=Some(28))"` (Sentry TAURI-RUST-4SZ).
-/// That string carries no "no space left on device" text, so anchor
-/// additionally on the cross-platform `StorageFull` ErrorKind token (std maps
-/// ENOSPC / `ERROR_DISK_FULL` / `ERROR_HANDLE_DISK_FULL` all to
-/// `ErrorKind::StorageFull`).
-///
-/// A fifth shape comes from SQLite itself. When the engine detects the
-/// disk-full condition during its own page bookkeeping (journal/WAL extension)
-/// before the next syscall surfaces an errno, rusqlite renders the `SQLITE_FULL`
-/// result code as `"database or disk is full"` (Sentry TAURI-RUST-B6N, hit
-/// while committing a local SQLite transaction). `SQLITE_FULL` has only two causes:
-/// genuine ENOSPC/ERROR_DISK_FULL (always the case in practice — the same
-/// burst always produces an os-error-28/112 sibling event) or a
-/// `max_page_count` PRAGMA cap (we set none).
-///
-/// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is
-/// the five words `"database or disk is full"` — Our local SQLite write
-/// call-sites wrap it with `format!("<verb>: {e}")` (e.g. `"commit tx: ..."`),
-/// so the phrase lands as the **suffix** of the local emit. The **extended**
-/// shape carries the full error-code envelope, `"database or disk is full:
-/// Error code 13: Insertion failed because database is full"` (Sentry
-/// TAURI-RUST-4R8, a local job-queue claim); here
-/// the canonical phrase sits mid-string, so the suffix anchor can't catch it.
-/// We detect this shape by requiring **both** local fragments together — the
-/// `"database or disk is full"` phrase AND the libsqlite3-sys `code_to_str`
-/// token `"insertion failed because database is full"` — which only rusqlite's
-/// own `SQLITE_FULL` Display emits as a pair. Requiring both (rather than the
-/// `code_to_str` token alone) keeps the silencer from matching a remote
-/// provider body that merely quotes the token half — e.g. an OpenAI-compatible
-/// `OpenAiEmbedding::embed` failure framed as `"Embedding API error (… Error
-/// code 13: Insertion failed because database is full)"` whose *server-side*
-/// SQLite is full is operator-actionable and must still surface to Sentry
-/// (codex CR on #3911). Both arms anchor on local-emit fragments rather than a
-/// bare `contains("database or disk is full")` so the silencer does not match a
-/// non-2xx backend response body whose payload happens to mention the phrase
-/// (e.g. a managed-backend 5xx whose server-side SQLite is full). Non-2xx
-/// backend bodies are framed by
-/// `integrations::client::post` / `composio::client` as `"Backend returned
-/// <status> <reason> for <METHOD> <url>: <detail>"` — an operator-actionable
-/// server/storage failure that must still surface to Sentry. As
-/// defense-in-depth for the edge case where the backend body itself ends with
-/// the phrase, reject any message that also carries the `"backend returned "`
-/// envelope prefix (codex CR on #3672, mirrors the precedent set by
-/// [`is_backend_user_error_message`]).
-///
-/// This is defense-in-depth for the genuinely
-/// unpreventable **write** paths (a write can't succeed on a full disk); the
-/// read path no longer emits this error at all (it degrades to a lock-free
-/// read — see `AuthProfilesStore::load`).
-fn is_disk_full_message(lower: &str) -> bool {
- if lower.contains("no space left on device")
- || lower.contains("not enough space on the disk")
- || lower.contains("storagefull")
- {
- return true;
- }
- // Two SQLITE_FULL renderings — see the fifth-shape section above. The
- // **bare** shape lands the phrase as a suffix (after trimming trailing
- // whitespace / punctuation that closures + JSON wrappers append). The
- // **extended** shape (TAURI-RUST-4R8) puts the phrase mid-string followed
- // by `: Error code 13: Insertion failed because database is full`; require
- // BOTH local fragments so we match only rusqlite's own `SQLITE_FULL` Display
- // and never a remote provider body that merely quotes the `code_to_str`
- // half (codex CR on #3911). The negative `"backend returned "` guard
- // rejects the remote 5xx envelope as a further line of defense.
- let trimmed = lower.trim_end_matches(|c: char| {
- c.is_ascii_whitespace() || matches!(c, '.' | ',' | ';' | ':' | '"' | '\'')
- });
- let bare_suffix = trimmed.ends_with("database or disk is full");
- let extended_local = lower.contains("database or disk is full")
- && lower.contains("insertion failed because database is full");
- (bare_suffix || extended_local) && !lower.contains("backend returned ")
-}
-
-/// Detect Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665) —
-/// a host-filesystem condition (fragmentation, USN journal overflow,
-/// filter-driver resource cap) that is persistent, locale-independent
-/// in the `(os error N)` suffix, and unrecoverable by the app.
-///
-/// Anchor on the locale-stable `(os error 665)` suffix. The Italian
-/// locale rendering is `"Impossibile completare l'operazione a causa
-/// di un limite del file system (os error 665)"` (Sentry
-/// TAURI-RUST-QT0, 6,050 events / 1 user). The `(os error 665)` suffix
-/// is the same across every locale, so no locale-detection is needed.
-///
-/// Polarity contract — only match when `(os error 665)` appears in
-/// the message body. This is a standard `std::io::Error` Display
-/// rendering, so it will never appear in a remote backend body
-/// accidentally.
-fn is_windows_file_system_limitation_message(lower: &str) -> bool {
- lower.contains("(os error 665)") && !lower.contains("backend returned ")
-}
-
-/// Detect the literal `"Config loading timed out"` string produced by
-/// [`crate::config::ops::load_config_with_timeout`] /
-/// [`crate::config::ops::reload_config_snapshot_with_timeout`]
-/// when `tokio::time::timeout` elapses around `Config::load_or_init` /
-/// `Config::load_from_config_path`.
-fn is_config_load_timed_out_message(lower: &str) -> bool {
- lower.contains("config loading timed out")
-}
-
-/// Detect a config-file READ that failed because the operating system refused
-/// access to a file that **exists** — i.e. an unpreventable user-environment
-/// condition with zero local lever, not an OpenHuman defect.
-///
-/// `Config::load_or_init` (`impl_load.rs`) takes its read branch only after
-/// `config_path.exists()` returns true, then `read_to_string` is retried 5× and
-/// still fails. On a healthy install that never happens; in the wild a user's
-/// `config.toml` can be ACL-denied, held open by another process (antivirus /
-/// backup agent), or a OneDrive "files on demand" placeholder that won't
-/// hydrate. We cannot unlock or re-ACL a foreign-held file, so the per-poll
-/// re-report (TAURI-RUST-DME: `inference_downloads_progress` re-loads config on
-/// every poll → 36k events / 1 user) carries no Sentry-actionable signal.
-///
-/// Polarity contract — demote **only** when BOTH hold:
-/// 1. an OpenHuman config-read context anchor is present — either
-/// `"failed to read config file"` (`load_or_init` retry path,
-/// `impl_load.rs`, the DME surface) or `"reading config.toml from"`
-/// (`load_from_config_path` snapshot-reload path) — AND
-/// 2. an OS-level *access-denied / locked* signal is present.
-///
-/// `NotFound` (`os error 2` / "cannot find the file"), "is a directory", and any
-/// io kind not enumerated here are deliberately EXCLUDED: a file that vanished
-/// after the `exists()` check is a TOCTOU race / app defect and MUST keep
-/// paging. This matcher is only meaningful once the loader surfaces the full
-/// chain (`{:#}`, #3962) — the io fragment lives in the source, not the top
-/// `with_context` line.
-fn is_config_read_io_failure_message(lower: &str) -> bool {
- let has_config_read_anchor =
- lower.contains("failed to read config file") || lower.contains("reading config.toml from");
- if !has_config_read_anchor {
- return false;
- }
- // A directory (or otherwise non-regular file) at the config path is a
- // bad-install / corruption signal that MUST keep paging. On Windows reading
- // a directory surfaces the same `Access is denied. (os error 5)` shape as a
- // genuine ACL denial, so the io-signal check below cannot tell them apart;
- // the read site (`impl_load.rs`) now fails a directory fast with this
- // distinct wording, and we belt-and-braces exclude it here too. (Codex P2.)
- if lower.contains("is a directory") || lower.contains("not a file") {
- return false;
- }
- // The file is owned by a uid other than the one reading it. That is an
- // OpenHuman defect, not user-environment state: *we* pick the runtime uid
- // (Dockerfile) and *we* write the config at 0600 (`impl_load.rs`), and the
- // container entrypoint chowns only the workspace directory — so a
- // stale-uid `config.toml` inside a reused volume denies every config RPC
- // (including the sign-in `auth_store_session`) while `/health` stays green.
- // There IS a local lever here, so it must keep paging. Genuine ACL /
- // antivirus / OneDrive denials carry no marker and stay demoted.
- //
- // Referencing the loader's constant (rather than a copied literal) makes
- // the producer/consumer coupling a compile-time one: the marker is already
- // lowercase, so it matches `lower` verbatim.
- if lower.contains(crate::config::schema::CONFIG_OWNER_MISMATCH_MARKER) {
- return false;
- }
- lower.contains("access is denied")
- || lower.contains("permission denied")
- || lower.contains("being used by another process")
- || lower.contains("cannot access the file")
- || lower.contains("(os error 5)")
- || lower.contains("(os error 32)")
-}
-
-// (The whatsapp_data SQLite busy/corrupt matchers were removed with the
-// store's move to the Tauri shell — the core no longer produces the
-// `[whatsapp_data] ingest failed:` envelope they anchored on.)
-
-fn is_embedding_backend_auth_failure(lower: &str) -> bool {
- tinyinference_embeddings::probe::is_embedding_backend_auth_failure(lower)
-}
-
-/// Detect a custom embeddings endpoint that exposes **no embeddings API** —
-/// the `OpenAiEmbedding` client POSTed `/embeddings` and the host answered
-/// `404 Not Found` (route absent) or `405 Method Not Allowed`. Canonical wire
-/// shape from `tinyinference-embeddings/src/cloud.rs`:
-///
-/// ```text
-/// Embedding API error (404 Not Found): <body>
-/// Embedding API error (405 Method Not Allowed): <body>
-/// ```
-///
-/// Deterministic user-config state: the user pointed the Custom
-/// (OpenAI-compatible) embeddings provider at a base URL whose host has no
-/// embeddings endpoint (e.g. a chat-only provider like DeepSeek). Every memory
-/// re-embed re-emits it (TAURI-RUST-5JR, ~2685 events / 9 users) and the
-/// settings UI surfaces an actionable remediation — Sentry has no fix to make.
-///
-/// Polarity (important): scoped to **404/405 only**. A `500` from a valid
-/// embeddings endpoint is a real server fault and must keep reaching Sentry; a
-/// `400` (e.g. oversized input) is prevented at source by the chunk cap
-/// (#3598) and likewise stays visible. Reused by
-/// `inference::embedding_host::rpc::update_settings` as the save-time hard-block signal so the
-/// two never drift.
-pub(crate) fn is_embedding_endpoint_absent(lower: &str) -> bool {
- tinyinference_embeddings::probe::is_embedding_endpoint_absent(lower)
-}
-
-/// Detect a custom/cloud embeddings endpoint that IS an embeddings API but
-/// **rejected the configured model id** — the user pasted a non-embedding
-/// (chat/reasoning) model into the embeddings model field. Canonical wire shape
-/// from `tinyinference-embeddings/src/cloud.rs` (TAURI-RUST-9SK, ~2205 events):
-///
-/// ```text
-/// Embedding API error (400 Bad Request): {"error":{"message":"Model nvidia/nemotron-3-super-120b-a12b does not exist","code":400}}
-/// ```
-///
-/// Deterministic user-config state, re-emitted on every memory re-embed; the
-/// embeddings settings UI surfaces an actionable "pick an embeddings-capable
-/// model" remediation (appended to the message at the emit site). The
-/// OpenRouter body — bare `"does not exist"` with an integer `"code":400` —
-/// matches none of the chat-side phrases in
-/// `tinyinference_providers::is_provider_config_rejection_message` (those key on the
-/// OpenAI-native `"does not exist or you do not have access"` /
-/// `model_not_found`), so this dedicated matcher is what demotes it.
-///
-/// Polarity (important): scoped to **400** + a model-rejection body. A bare
-/// 400 (oversized input — prevented at source by the chunk cap #3598) or a 500
-/// from a valid embeddings endpoint is a real fault and must keep reaching
-/// Sentry, so this never fires on them.
-fn is_embedding_model_rejected(lower: &str) -> bool {
- tinyinference_embeddings::probe::is_embedding_model_rejected(lower)
-}
-
-/// Whether `msg` is the hosted backend refusing a feature the account's plan
-/// does not include (#8081): a `[…_PLAN_REQUIRED]` code in the head
-/// TinyMemory gives a hosted failure, `[CODE] memory API {route} …`. Matched
-/// as that delimited code, so a description that merely mentions the code
-/// is not a plan refusal. User state, never an expired session.
-pub fn is_plan_required_message(msg: &str) -> bool {
- // The code runs from the nearest `[` before the head and is all
- // `A-Z0-9_`, as TinyMemory cleans it.
- msg.match_indices(PLAN_REQUIRED_HEAD).any(|(at, _)| {
- msg[..at].rsplit_once('[').is_some_and(|(_, code)| {
- code.chars()
- .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_')
- })
- })
-}
-
-/// What follows a plan code in TinyMemory's hosted failure head.
-const PLAN_REQUIRED_HEAD: &str = "_PLAN_REQUIRED] memory API ";
-
-/// Detect **app-session-expired** boundary errors that bubble up from any
-/// backend-touching call site (agent, web channel, cron, integrations).
-///
-/// This is also the JSON-RPC dispatch-site classifier. Keep it stricter than
-/// a bare "401 + unauthorized" pair: OpenAI / Anthropic BYO-key failures,
-/// Composio scope failures, and channel-provider 401s are actionable scoped
-/// errors, not proof that the user's OpenHuman app session expired.
-///
-/// The canonical OpenHuman session-expired wire shapes:
-///
-/// - `"OpenHuman API error (401 Unauthorized): {…\"Session expired. Please
-/// log in again.\"…}"` — emitted by `provider::http_error::api_error` from the
-/// OpenHuman backend and re-raised through `agent::run_single` /
-/// `web_chat::run_chat_task` (OPENHUMAN-TAURI-26). The
-/// `"session expired"` substring anchors the match to the OpenHuman
-/// backend's session-renewal body, not the bare numeric status.
-/// - `"OpenHuman API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
-/// — same emit site, same wire shape as the `Session expired` body, but the
-/// OpenHuman backend swaps in `"Invalid token"` for the JWT-validity
-/// rejection branch (vs. the explicit session-renewal branch).
-/// OPENHUMAN-TAURI-4P0. The conjunctive anchor — `"OpenHuman API error
-/// (401"` **and** the envelope-shaped `"\"error\":\"Invalid token\""` —
-/// keeps the #2286 contract intact: bare `"Invalid token"`, OpenAI /
-/// Anthropic BYO-key 401s, Discord upstream-bot-token rejections, and
-/// provider scope errors still route to Sentry as actionable.
-/// - `"Embedding API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
-/// — TAURI-RUST-4K5 (~118 events, escalating on 0.56.0). Same OpenHuman
-/// backend session-expired envelope as 4P0, but the embedding client at
-/// `tinyinference-embeddings/src/cloud.rs` wraps it with the
-/// `"Embedding API error"` prefix instead of `"OpenHuman API error"`.
-/// Uses the same conjunctive-anchor pattern so BYO-key embedding 401s
-/// from third-party providers (OpenAI / Voyage / Cohere) still escalate
-/// — guarded by `does_not_classify_embedding_byo_key_401_as_session_expired`.
-/// - `"OpenHuman streaming API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
-/// — TAURI-RUST-1EE (~110 events, ongoing on 0.56.0). Same envelope as
-/// 4P0, wrapped by the streaming-chat path at
-/// `inference/provider/compatible.rs:949` with the
-/// `"OpenHuman streaming API error"` prefix. The `streaming` token means
-/// the 4P0 anchor doesn't match, so it needs its own prefix arm; BYO-key
-/// streaming 401s still escalate — guarded by
-/// `does_not_classify_streaming_byo_key_401_as_session_expired`.
-/// - `"SESSION_EXPIRED: backend session not active — sign in to resume LLM work"`
-/// — the `llm_gate::is_signed_out` sentinel from
-/// `providers::openhuman_backend::resolve_bearer`.
-/// - `"no backend session token; run auth_store_session first"` and
-/// `"session JWT required"` — local pre-flight guards that fire when the
-/// stored profile is empty (`#1465`-ish onboarding spam) or has been
-/// cleared by a previous 401 cycle. Both shapes are OpenHuman-specific.
-///
-/// At the JSON-RPC dispatch boundary the same strict match controls
-/// `DomainEvent::SessionExpired` publication, so downstream/provider 401s stay
-/// recoverable and do not clear the stored app session.
-pub fn is_session_expired_message(msg: &str) -> bool {
- // #8081: tinymemory's hosted mapping words every 403, including
- // `[MEMORY_PLAN_REQUIRED]` (memory not in the account's plan), as "the
- // session expired or the API key was rejected (HTTP 403 …)". A 403 or a
- // plan refusal never means the session expired, and signing out on one
- // logged a valid user out on opening a Memory tab.
- if is_plan_required_message(msg) || msg.contains("(HTTP 403") {
- return false;
- }
- let lower = msg.to_ascii_lowercase();
- lower.contains("session expired")
- || lower.contains("no backend session token")
- || lower.contains("session jwt required")
- || msg.contains("SESSION_EXPIRED")
- || (msg.contains("OpenHuman API error (401") && msg.contains("\"error\":\"Invalid token\""))
- || (msg.contains("Embedding API error (401") && msg.contains("\"error\":\"Invalid token\""))
- // OPENHUMAN-TAURI-4P0 — OpenHuman backend's "Invalid token" 401
- // envelope. Both anchors must be present: the OpenHuman-scoped
- // `"OpenHuman API error (401"` prefix (so a third-party provider's
- // `"OpenAI API error (401 Unauthorized): invalid_api_key"` cannot
- // match), AND the envelope-shaped `"\"error\":\"Invalid token\""`
- // (so bare prose mentions of "invalid token" — Discord OAuth
- // failures, generic upstream errors covered by #2286 — stay
- // actionable in Sentry).
- || (msg.contains("OpenHuman API error (401")
- && msg.contains("\"error\":\"Invalid token\""))
- // TAURI-RUST-4K5 — same OpenHuman backend "Invalid token" envelope
- // wrapped by `tinyinference-embeddings/src/cloud.rs` with the
- // `"Embedding API error"` prefix instead of `"OpenHuman API error"`.
- // Same conjunctive-anchor pattern as 4P0: the embedding-scoped
- // prefix gates the match so a third-party BYO-key embedding 401
- // (e.g. OpenAI/Voyage/Cohere rejecting the user's own API key)
- // stays actionable — guarded by
- // `does_not_classify_embedding_byo_key_401_as_session_expired`.
- || (msg.contains("Embedding API error (401")
- && msg.contains("\"error\":\"Invalid token\""))
- // TAURI-RUST-1EE — same OpenHuman backend "Invalid token" envelope
- // wrapped by the streaming-chat path at
- // `inference/provider/compatible.rs:949` with the
- // `"OpenHuman streaming API error"` prefix. The `streaming` token
- // between `OpenHuman` and `API error` means the 4P0 anchor
- // (`"OpenHuman API error (401"`) does not match it, so the
- // streaming path needs its own prefix arm. Same conjunctive-anchor
- // pattern keeps third-party BYO-key streaming 401s
- // (`"OpenAI streaming API error (401): invalid_api_key"`)
- // escalating — guarded by
- // `does_not_classify_streaming_byo_key_401_as_session_expired`.
- || (msg.contains("OpenHuman streaming API error (401")
- && msg.contains("\"error\":\"Invalid token\""))
- // TAURI-RUST-N — same OpenHuman backend "Invalid token" envelope
- // wrapped by the tinyagents `ProviderError::Display` format
- // (`TinyAgentsError::Provider(Box<ProviderError>)`). The historical
- // `OpenHumanBackendModel` delegated to `OpenAiModel`
- // which formats errors as `"{provider} returned HTTP {status}: {body}"`,
- // producing `"OpenHuman returned HTTP 401: ..."` — different from the
- // `"OpenHuman API error (401"` prefix the classic `api_error` path emits.
- // Same conjunctive-anchor pattern: scoped to `"OpenHuman returned HTTP
- // {status}"` (so a third-party BYO-key 401 from a provider whose label
- // contains "OpenHuman" cannot match) AND the envelope-shaped
- // `"\"error\":\"Invalid token\""` (so bare prose mentions of "invalid
- // token" stay actionable).
- || (msg.contains("OpenHuman returned HTTP 401")
- && msg.contains("\"error\":\"Invalid token\""))
-}
-
-/// Detect a remote MCP server's connect-time 401 — the user must sign in to
-/// that server (OAuth), not a code defect. Anchored on the canonical
-/// TinyMCP unauthorized error text, which renders as
-/// `"mcp unauthorized for \`<endpoint>\` (HTTP 401…)"`.
-///
-/// Conjunctive match — both anchors must hit (input already lower-cased):
-///
-/// 1. `"mcp unauthorized for "` — the stable module error prefix. Scopes the match to
-/// the MCP transport's own 401 so an unrelated "unauthorized" / "401" from
-/// another domain cannot borrow this demotion.
-/// 2. `"(http 401"` — the parenthesised status the module message always emits
-/// (with or without the trailing `resource metadata:` discovery hint).
-///
-/// `connections::connect` already classifies this and stores a `needs_auth`
-/// flag so the UI prompts for sign-in; this predicate keeps the parallel
-/// `mcp_clients_connect` RPC re-report out of Sentry. See
-/// [`ExpectedErrorKind::McpServerNeedsAuth`].
-fn is_mcp_server_needs_auth_message(lower: &str) -> bool {
- lower.contains("mcp unauthorized for ") && lower.contains("(http 401")
-}
-
-/// Detect the wallet's "not set up yet" sentinel, anywhere in the message.
-///
-/// Substring rather than equality **on purpose**: the same condition arrives
-/// bare from a direct RPC and context-wrapped from any caller that does
-/// `format!("{context}: {e}")`. Both are the same user-state and both must be
-/// demoted, so the predicate must not care how many layers wrapped it.
-///
-/// The needle is the shared producer constant, not a copy, so a wording change
-/// moves both sides at once. `wallet_not_configured_sentinel_is_ascii_lowercase`
-/// pins the remaining assumption — that the constant is already lowercase, so
-/// it can be compared against the pre-lowercased haystack without allocating.
-/// See [`ExpectedErrorKind::WalletNotConfigured`].
-fn is_wallet_not_configured_message(lower: &str) -> bool {
- lower.contains(crate::web3::wallet::WALLET_NOT_CONFIGURED_MESSAGE)
-}
-
-/// Detect the "a configured provider has no API key" user-config state.
-///
-/// Single source of truth for the `ApiKeyMissing` wording so the
-/// `expected_error_kind` demotion arm and the cron scheduler's halt-on-first
-/// classifier (`is_api_key_unset_failure`, TAURI-RUST-HCK) can never drift
-/// apart. The phrasing is emitted deterministically, before any HTTP, by the
-/// credential guards:
-/// - `inference/provider/compatible_request.rs::credential_for_request`
-/// ("<provider> API key not set. Configure via the web UI …")
-/// - the embeddings credential guards (cohere/openai) + the composio
-/// direct-mode factory bail ("no api key is configured" / "no api key
-/// supplied").
-///
-/// Distinct from a *rejected* key (a provider 401 "Invalid API key"): that is a
-/// present-but-wrong key and stays actionable in Sentry — this matcher is for
-/// the **absent** key only, which has no Sentry remediation path.
-pub fn is_api_key_unset_message(text: &str) -> bool {
- let lower = text.to_ascii_lowercase();
- lower.contains("api key not set")
- || lower.contains("missing api key")
- || lower.contains("no api key is configured")
- || lower.contains("no api key supplied")
-}
-
-/// Does this error text describe a **local** LLM provider (loopback host, e.g.
-/// LM Studio / Ollama / llama.cpp on `127.0.0.1:<port>` or `localhost:<port>`)
-/// refusing the connection because its server isn't running?
-///
-/// Single-source wrapper over [`is_loopback_unavailable`] so callers outside
-/// this module (the cron scheduler's retry-halt guard) key off the exact same
-/// matcher the [`expected_error_kind`] classifier uses — the phrasing cannot
-/// drift between the source demotion and the cron suppression. Lowercases the
-/// input to match the classifier's internal contract (TAURI-RUST-12K).
-pub fn is_local_provider_unreachable_message(text: &str) -> bool {
- is_loopback_unavailable(&text.to_ascii_lowercase())
-}
-
-/// Detect the in-process-core boot-window shape: a sibling component
-/// (frontend RPC relay, integrations / composio HTTP clients) tried to
-/// reach the embedded core's `127.0.0.1:<port>` listener before it finished
-/// binding, so the kernel returned `Connection refused`. The condition
-/// self-resolves once startup completes — Sentry has no remediation path.
-///
-/// Conjunctive match — both anchors must hit:
-///
-/// 1. **Loopback host with port**: substring `127.0.0.1:` or `localhost:` so
-/// a doc URL or unrelated mention without a port (`localhost`,
-/// `127.0.0.1\n`) does not match. Pinned to the colon+port pattern
-/// because every observed shape from reqwest / hyper / our own
-/// `IntegrationClient` wraps the host as `<host>:<port>` in the URL the
-/// error chain renders.
-/// 2. **Connection refused with platform errno**: `connection refused (os
-/// error 61)` (macOS / BSD), `connection refused (os error 111)`
-/// (Linux), or `connection refused (os error 10061)` (Windows
-/// `WSAECONNREFUSED`). Pinning to `(os error N)` keeps the matcher from
-/// swallowing higher-level wrappers that merely mention "connection
-/// refused" in prose.
-///
-/// 3. **Locale-independent fallback**: on non-English OS locales the kernel
-/// renders the `ECONNREFUSED` / `WSAECONNREFUSED` text translated (e.g. a
-/// zh-CN Windows host emits `由于目标计算机积极拒绝,无法连接。 (os error
-/// 10061)`), so the English `connection refused` prefix in (2) is absent
-/// and a genuine loopback-refused body would leak past this matcher into
-/// the broad [`is_network_unreachable_message`] bucket (TAURI-RUST-12K).
-/// Recover it by pairing the reqwest/hyper-stable `tcp connect error`
-/// marker with the connect-refused errno alone. Scoped to the three
-/// connect-refused errnos only — the distinct timeout errnos (`60` / `110`
-/// / `10060`, `WSAETIMEDOUT`) are NOT matched, so a loopback *timeout*
-/// still classifies as its own shape rather than being mislabelled
-/// "refused".
-///
-/// Drops OPENHUMAN-TAURI-R5 (~2.5k events, `integrations.get` emit site)
-/// and OPENHUMAN-TAURI-R6 (~2.5k events, the `rpc.invoke_method` re-wrap of
-/// the same trace). Both share `trace_id=6ebf5b62748d5144e541e2cddeabbbd0`
-/// and the canonical body shape:
-///
-/// ```text
-/// error sending request for url (http://127.0.0.1:18474/integrations/composio/connections)
-/// → client error (Connect) → tcp connect error → Connection refused (os error 61)
-/// ```
-///
-/// Without this matcher the body falls through to
-/// [`is_network_unreachable_message`] and demotes as `NetworkUnreachable`,
-/// which conflates an internal lifecycle race with user-environment problems
-/// (VPN drop, captive portal, ISP block) and makes the "what's spiking?"
-/// question un-answerable. See [`ExpectedErrorKind::LoopbackUnavailable`].
-fn is_loopback_unavailable(lower: &str) -> bool {
- let has_loopback_host = lower.contains("127.0.0.1:") || lower.contains("localhost:");
- if !has_loopback_host {
- return false;
- }
- // (2) English errno-prefixed form.
- if lower.contains("connection refused (os error 61)")
- || lower.contains("connection refused (os error 111)")
- || lower.contains("connection refused (os error 10061)")
- {
- return true;
- }
- // (3) Locale-independent fallback: the OS-refused text may be translated,
- // leaving only the errno. Require the transport-layer `tcp connect error`
- // marker so a non-connect loopback failure cannot match, and pin to the
- // connect-refused errnos (not the timeout errnos 60 / 110 / 10060).
- lower.contains("tcp connect error")
- && (lower.contains("(os error 61)")
- || lower.contains("(os error 111)")
- || lower.contains("(os error 10061)"))
-}
-
-/// Detect Ollama embed call sites that surface a user-config rejection from
-/// the local Ollama daemon — pure user-state errors the UI already surfaces
-/// (toast / settings page warning) where Sentry has no remediation path.
-///
-/// Several canonical wire shapes are covered, all emitted by the TinyAgents
-/// Ollama embedder and the host embed service fallback path:
-///
-/// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the
-/// embedder at a chat / vision model id with a temperature suffix (e.g.
-/// `qwen3-vl:[email protected]`) which Ollama parses as malformed. Wire shape:
-/// `ollama embed failed with status 400 Bad Request: {"error":"invalid model name"}`.
-/// - **OPENHUMAN-TAURI-MA / -KM** (deferred follow-up from PR #2216), and
-/// **TAURI-RUST-K** (~1990 events) / **TAURI-RUST-8K** (~411 events) on
-/// self-hosted Sentry: user configured a model id that the local Ollama
-/// daemon hasn't pulled yet. Wire shape:
-/// `ollama embed failed with status 404 Not Found: {"error":"model \"<id>\" not found, try pulling it first"}`.
-/// (Self-hosted Sentry events still flow from older client releases that
-/// predate this matcher; they drop off naturally as users upgrade.)
-/// - **TAURI-RUST-3X / -8WA** (~982 events on 0.57.52): 501 "embeddings not
-/// supported". Two bodies — the model is chat/vision-only
-/// (`{"error":"this model does not support embeddings"}`) or the Ollama
-/// daemon was started without embed support
-/// (`{"error":"This server does not support embeddings. Start it with `--embeddings`"}`).
-/// - **TAURI-RUST-3E** (~249 events): 401 auth-required Ollama endpoint with
-/// no credentials configured. Wire shape:
-/// `ollama embed failed with status 401 Unauthorized: {"error": "unauthorized"}`.
-/// - **OPENHUMAN-TAURI-GX**: user opted into Ollama embeddings but the
-/// daemon isn't running on `localhost:11434`, so the embed service falls
-/// back to cloud embeddings for the session. Wire shape:
-/// `ollama embeddings opted-in but daemon unreachable at http://localhost:11434; falling back to cloud embeddings for this session`.
-///
-/// All are user-config: the user picked the wrong model id, forgot to pull
-/// it, ran a daemon without embed support, omitted credentials, or forgot to
-/// start the daemon. The remediation is "fix the model id in Settings" /
-/// "run `ollama pull <id>`" / "start ollama with `--embeddings`" / "add a
-/// key" / "start ollama" — none of which Sentry can do for them.
-///
-/// Each arm is anchored on the `"ollama embed"` prefix
-/// (`"ollama embed failed"` for the failed-request shapes,
-/// `"ollama embeddings opted-in"` for the daemon-unreachable fallback) so
-/// unrelated errors elsewhere in the codebase that happen to contain
-/// `"invalid model name"`, `"not found"`, or `"does not support embeddings"`
-/// substrings are not silenced.
-///
-/// Routes to [`ExpectedErrorKind::ProviderUserState`] — the same bucket that
-/// holds the composio / gmail / OAuth user-state errors. We deliberately do
-/// **not** introduce a dedicated Ollama enum variant: the demotion semantics
-/// (drop to `info` log, skip Sentry capture) are identical and adding a new
-/// variant for every provider would balloon the enum without changing
-/// behavior.
-fn is_ollama_user_config_rejection(lower: &str) -> bool {
- // XS — 400-status user-config (invalid model name, including the
- // temperature-suffix shape `qwen3-vl:[email protected]` Ollama parses as malformed).
- if lower.contains("ollama embed failed") && lower.contains("invalid model name") {
- return true;
- }
-
- // MA / KM — 404-status pull-required. The wire shape is JSON-escaped
- // (`\"<model-id>\" not found`); after lower-casing we still see the
- // backslash-quoted form. Anchor on `model \"` + `\" not found` so an
- // unrelated 404 that merely contains `"model"` and `"not found"` is not
- // swallowed. The `\\"` byte pair in Rust source matches the literal
- // `\"` sequence in the wire shape.
- if lower.contains("ollama embed failed")
- && lower.contains("model \\\"")
- && lower.contains("\\\" not found")
- {
- return true;
- }
-
- // 3X / 8WA — 501-status "embeddings not supported". Ollama emits two
- // bodies for this: the model is chat/vision-only
- // (`{"error":"this model does not support embeddings"}`) or the daemon
- // itself was started without embedding support
- // (`{"error":"This server does not support embeddings. Start it with `--embeddings`"}`,
- // TAURI-RUST-8WA, ~982 events on 0.57.52). Both are user-side Ollama
- // config the app can't fix — it can neither swap the user's model nor
- // restart their daemon with `--embeddings`. Anchor on the shared
- // `does not support embeddings` phrase (still gated by the
- // `ollama embed failed` prefix) so a future qualifier wording still demotes.
- if lower.contains("ollama embed failed") && lower.contains("does not support embeddings") {
- return true;
- }
-
- // TAURI-RUST-3E (~249 events) — 401-status auth failure from Ollama
- // (user pointed the embedder at an authenticated Ollama endpoint
- // without configuring credentials, e.g. self-hosted Ollama behind an
- // auth proxy or Ollama Cloud without API key). Body shape:
- // `{"error": "unauthorized"}`. Anchor on `ollama embed failed`
- // + `status 401` so unrelated 401s from other call sites (provider
- // chat, backend API) aren't silenced.
- if lower.contains("ollama embed failed") && lower.contains("status 401") {
- return true;
- }
-
- // GX — daemon-unreachable opt-in state. The wire shape is emitted by
- // the embed service when the user has opted into Ollama in settings
- // but the daemon isn't responding, so the service falls back to cloud
- // embeddings for the session. Anchor on the full prefix to keep the
- // matcher from colliding with unrelated `"daemon unreachable"`
- // messages from other domains (e.g. backend connection-health logs).
- if lower.contains("ollama embeddings opted-in but daemon unreachable at") {
- return true;
- }
-
- false
-}
-
-/// Detect transport-level connection failures that fire before any HTTP status
-/// is observed — DNS resolution failures, TCP connect refused/reset, TLS
-/// handshake failures, or ISP/firewall blocks. The canonical shape is
-/// reqwest's `"error sending request for url (…)"`, which surfaces from any
-/// HTTP call site (provider chat, embeddings, backend RPC) when the request
-/// can't reach the server at all.
-///
-/// These are user-environment problems — VPN drop, captive portal, ISP-level
-/// block (OPENHUMAN-TAURI-32: user in RU couldn't reach the managed backend),
-/// firewall — that no amount of retry / fallback on our side can resolve.
-/// Sentry has no signal to act on (no status, no trace, no payload), so each
-/// occurrence is pure noise. Classify them as expected so the report site
-/// logs a breadcrumb rather than spawning an error event.
-///
-/// Loopback `127.0.0.1:<port>` `Connection refused` shapes are routed
-/// through [`is_loopback_unavailable`] *before* this matcher so the
-/// boot-window race against the embedded core keeps its own bucket — see
-/// the precedence comment in [`expected_error_kind`].
-///
-/// Three additional substrings cover wire-shape variants observed in
-/// Wave 4 that the original `"dns error"` / status-code matchers miss:
-///
-/// - `"failed to lookup address"` / `"nodename nor servname"` —
-/// `getaddrinfo()` failure renderings on macOS / BSD libc and POSIX
-/// resolvers (`OPENHUMAN-TAURI-44` ~50 events,
-/// `[socket] Connection failed: WebSocket connect: IO error: failed to
-/// lookup address information: nodename nor servname provided, or not
-/// known`).
-/// - `"http error: 200 ok"` — tungstenite's `WsError::Http(200)` render
-/// when a corporate proxy / captive portal intercepts the WebSocket
-/// handshake and returns a plain HTML 200 page (`OPENHUMAN-TAURI-4P`
-/// ~66 events). Tungstenite-only — reqwest renders HTTP 200 as
-/// `"HTTP status server error (200)"`, so this can't collide with the
-/// regular HTTP call path.
-/// - `"unexpected eof during handshake"` — `native-tls`'s render when the
-/// peer (or an intercepting firewall / antivirus / corporate TLS proxy)
-/// closes the TCP connection mid-TLS-handshake, surfacing as
-/// `"TLS error: native-tls error: unexpected EOF during handshake"`
-/// wrapped by `socket::ws_loop::run_connection` into
-/// `"WebSocket connect: …"` (`TAURI-RUST-4ZD`, first seen on
-/// `[email protected]`, Windows). The existing `"tls handshake"` anchor
-/// misses it because the words aren't contiguous (`"tls error"` …
-/// `"during handshake"`). Same user-environment shape as the other
-/// handshake-stage entries — the socket supervisor already retries with
-/// exponential backoff and Sentry has no actionable signal.
-/// - `"http version must be 1.1 or higher"` — tungstenite's
-/// `ProtocolError::WrongHttpVersion` render. Fires when a server (or
-/// intermediary proxy / HTTP/2-only edge) responds to the WebSocket
-/// upgrade with HTTP/2+, which the WS spec forbids — the handshake
-/// requires HTTP/1.1 (`CORE-RUST-DP`, ~2 events / 24h, first seen on
-/// `[email protected]`). Same shape as the existing handshake-stage
-/// entries: a user-environment / infra misconfiguration that the
-/// client cannot fix; Sentry has no actionable signal beyond what the
-/// socket supervisor's exponential backoff already provides.
-fn is_network_unreachable_message(lower: &str) -> bool {
- lower.contains("error sending request for url")
- || lower.contains("dns error")
- || lower.contains("failed to lookup address")
- || lower.contains("nodename nor servname")
- || lower.contains("connection refused")
- || lower.contains("connection reset")
- // OPENHUMAN-TAURI-EM (128 events): the channel supervisor wraps
- // `discord_listen()`'s anyhow chain as `format!("Channel {} error:
- // {e:#}; restarting", ...)`, which lands as
- // `"Channel discord error: IO error: Operation timed out (os error
- // 60); restarting"`. The discord gateway TCP/WebSocket connection
- // timing out is transient network state, not a code bug — the
- // supervisor already retries with exponential backoff. Same shape
- // surfaces on every channel (slack/telegram/...) once the
- // underlying socket hits ETIMEDOUT, so we match on the platform-
- // agnostic phrase, symmetric with `"connection reset"` /
- // `"connection refused"` above. Errno renderings are not pinned
- // because `(os error 60)` (BSD/macOS), `(os error 110)` (Linux),
- // `(os error 10060)` (Windows `WSAETIMEDOUT`), and bare prose
- // `"operation timed out"` (hyper / tungstenite / std::io) all
- // share the same lowercase substring.
- || lower.contains("operation timed out")
- || lower.contains("network is unreachable")
- || lower.contains("no route to host")
- || lower.contains("tls handshake")
- || lower.contains("unexpected eof during handshake")
- || lower.contains("certificate verify failed")
- || lower.contains("http error: 200 ok")
- || lower.contains("http version must be 1.1 or higher")
-}
-
-/// Detect the canonical supervisor-wrap shape emitted by
-/// `channels::runtime::supervision::spawn_supervised_listener` —
-/// `"Channel <name> error: <inner>; restarting"`. Language-agnostic
-/// (anchored on the Rust wrapper, not the inner error wording) so it
-/// covers OS-localized variants (TAURI-RUST-BB Chinese-Windows
-/// WSAETIMEDOUT body) that escape the English-only network anchors in
-/// [`is_network_unreachable_message`].
-///
-/// The supervisor restarts the listener with its own exponential backoff;
-/// sustained outages surface via separate `health.bus` events /
-/// `FAIL_ESCALATE_THRESHOLD`. Per-restart messages carry no actionable
-/// Sentry signal — Sentry has no remediation path beyond what the
-/// supervisor already does (TAURI-RUST-15 ~11.4 k events / -BB ~815
-/// events on self-hosted `tauri-rust`).
-///
-/// Anchors on three substrings together to avoid false positives:
-/// - leading `"channel "` (with trailing space disambiguates from
-/// unrelated mentions like `"channels"` or `"channel-runtime"`)
-/// - `" error:"` (the wrapper's literal separator)
-/// - `"; restarting"` (the wrapper's literal trailer)
-///
-/// A bare `"…; restarting"` log line without the `"Channel <name> error:"`
-/// preamble must NOT classify — that's a generic restart note from some
-/// other subsystem and Sentry signal there may still be actionable.
-fn is_channel_supervisor_restart_message(lower: &str) -> bool {
- lower.starts_with("channel ") && lower.contains(" error:") && lower.contains("; restarting")
-}
-
-/// Detect transient upstream HTTP failures that have bubbled up out of the
-/// provider layer and into higher-level domains (`agent`, `web_channel`, …).
-///
-/// The reliable-provider stack already retries / falls back on
-/// [`TRANSIENT_PROVIDER_HTTP_STATUSES`] (408/429/502/503/504), and the
-/// `before_send` filter drops the per-attempt provider events that carry
-/// `domain=llm_provider`. But the same error is *also* returned via
-/// `Result::Err` and re-reported by callers that wrap the provider — e.g.
-/// `agent.run_single` (OPENHUMAN-TAURI-5Z), `web_channel.run_chat_task`,
-/// scheduler tick handlers — under a different `domain` tag, escaping the
-/// provider-scoped filter and producing one Sentry event per failed turn.
-///
-/// The canonical wire format from `provider::http_error::api_error` is:
-/// `"<provider> API error (<status>): <sanitized>"` — e.g.
-/// `"OpenHuman API error (504 Gateway Timeout): error code: 504"`. Pin the
-/// match to that exact `"api error (<status>"` prefix so an unrelated message
-/// that merely mentions "504" (a log line, a doc URL) is not silenced.
-///
-/// Also matches the second canonical wire shape: tungstenite's
-/// `WsError::Http(response)` Display, which renders as `"HTTP error: <status>"`
-/// (and which `socket::ws_loop::run_connection` wraps as
-/// `"WebSocket connect: HTTP error: 502 Bad Gateway"`). Per
-/// OPENHUMAN-TAURI-5P (~110 events) and -EZ (~51 events), backend
-/// staging/production load balancers emit HTTP 502/504 during the WebSocket
-/// upgrade handshake; tungstenite surfaces those as `WsError::Http` and the
-/// socket reconnect loop already handles them via exponential backoff. Each
-/// `FAIL_ESCALATE_THRESHOLD` escalation fires `report_error_or_expected` with
-/// the formatted reason, which would land in Sentry as `domain=socket`
-/// noise without this matcher (the existing `domain=integrations`
-/// before_send filter scopes too narrowly).
-///
-/// Three separator variants cover every observed shape: trailing space
-/// (`"HTTP error: 502 Bad Gateway"`), trailing newline (`"HTTP error: 502\n…"`
-/// from chained errors), and trailing colon (`"HTTP error: 502: …"`). Bare
-/// `"HTTP error: 502"` at end-of-string is not matched on purpose — the
-/// status integer alone could collide with unrelated log lines containing
-/// `"HTTP error: 5023"` (port number, runbook ID).
-fn is_transient_upstream_http_message(lower: &str) -> bool {
- tinyinference_llm::failure::TRANSIENT_PROVIDER_HTTP_STATUSES
- .iter()
- .any(|code| {
- lower.contains(&format!("api error ({code}"))
- || lower.contains(&format!("api error {code} "))
- || lower.contains(&format!("http error: {code} "))
- || lower.contains(&format!("http error: {code}\n"))
- || lower.contains(&format!("http error: {code}:"))
- })
-}
-
-/// Detect a non-2xx **HTML 403/Forbidden gateway page** returned to a provider
-/// embedding call — the signature of an edge/CDN/WAF or regional block sitting
-/// in front of the provider API, where the request never reached the provider
-/// app and the body is a generic gateway error page rather than the provider's
-/// JSON error envelope.
-///
-/// Canonical wire shape (TAURI-RUST-8S3, `CohereEmbedding::embed` emit site):
-/// `"Cohere embed API error (403 Forbidden): <!doctype html>…<title>403</title>403 Forbidden"`.
-/// The same shape also covers the OpenAI/Voyage and custom OpenAI-compatible
-/// embed paths when their upstream is fronted by the same edge tier, so we do
-/// not pin to a single provider prefix.
-///
-/// Two conditions must BOTH hold:
-///
-/// 1. A **403 token** is present — either the `403` status inside the embed
-/// error prefix or a bare `403 forbidden` in the body.
-/// 2. An **HTML gateway marker** is present (`<!doctype html`, `<html`,
-/// `<title>403`) — proving the body is a gateway page, not the provider's
-/// structured error.
-///
-/// And the body must **not** look like a JSON envelope (no `{` … `"message"` /
-/// `"error"` JSON shape). This is the discrimination guard: Cohere's real JSON
-/// 403 (`{"message": "invalid api key"}`) and any genuine request-shape bug in
-/// our client that round-trips a JSON 4xx stay classified `None` and keep
-/// reaching Sentry. Only the bodiless edge HTML page is demoted.
-///
-/// `genuinely-unpreventable`: endpoint correct, key sent, retry can't clear an
-/// edge policy keyed on the user's network reputation / geo / IP — Sentry has
-/// no remediation path.
-fn is_upstream_edge_block_message(lower: &str) -> bool {
- // Must originate from an embedding call so we don't silence an HTML 403
- // surfaced by some unrelated path.
- let is_embed = lower.contains("embed");
- if !is_embed {
- return false;
- }
- // A JSON envelope means the provider's app answered with a structured
- // error — actionable, must page. Don't demote.
- let looks_like_json = lower.contains('{')
- && (lower.contains("\"message\"")
- || lower.contains("\"error\"")
- || lower.contains("\"detail\""));
- if looks_like_json {
- return false;
- }
- let has_403 = lower.contains("(403 ") || lower.contains("403 forbidden");
- let has_html_marker =
- lower.contains("<!doctype html") || lower.contains("<html") || lower.contains("<title>403");
- has_403 && has_html_marker
-}
-
-/// Detect non-2xx HTTP failures returned from the backend integrations / composio
-/// clients that are by definition user-input or user-auth-state problems — not
-/// bugs Sentry can act on.
-///
-/// The canonical wire format from
-/// [`crate::integrations::client::IntegrationClient::post`] / `get`
-/// and [`crate::integrations::composio::client::ComposioClient`] is:
-/// `"Backend returned <status> <reason> for <METHOD> <url>: <detail>"` — e.g.
-/// `"Backend returned 400 Bad Request for POST https://api.example.com/integrations/composio/authorize: Composio authorization failed: 400 …"`
-/// (OPENHUMAN-TAURI-BC: user submitted SharePoint authorize without filling in
-/// the required Tenant Name field). The backend correctly returned a 4xx; the
-/// UI already surfaces the structured error to the user via toast — Sentry has
-/// no remediation path because the request was malformed *by the user's
-/// input*, not by our code.
-///
-/// We pin the match to the `"backend returned "` prefix so an unrelated
-/// message merely mentioning "400" (a log line, doc URL) is not silenced.
-///
-/// We classify only 4xx codes, with **two exclusions**:
-/// - `408 Request Timeout` and `429 Too Many Requests` are *transient* — they
-/// are surfaced via [`is_transient_upstream_http_message`] for the provider
-/// path and stay actionable for the backend path so a sustained 429 (rate
-/// limit cliff) still pages.
-///
-/// 5xx is intentionally **not** classified here — server-side failures from
-/// our backend are real bugs that should reach Sentry. The transient
-/// 502/503/504 deduplication is handled by the threshold logic in callers
-/// (see e.g. `crate::platform::socket::ws_loop::FAIL_ESCALATE_THRESHOLD`).
-fn is_backend_user_error_message(lower: &str) -> bool {
- let Some(rest) = lower.split_once("backend returned ").map(|(_, r)| r) else {
- return false;
- };
- let status_digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
- let Ok(status) = status_digits.parse::<u16>() else {
- return false;
- };
- // 4xx (except transient 408 / 429 which are handled separately).
- matches!(status, 400..=499) && status != 408 && status != 429
-}
-
-/// Detect third-party provider validation failures that bubble up as
-/// user-state errors — composio trigger registry mismatch, toolkit not
-/// enabled, OAuth scopes missing, required fields left blank.
-///
-/// Unlike [`is_backend_user_error_message`], this classifier is **body-text
-/// shape-based** rather than HTTP-status-based, so it catches the cases
-/// where the composio backend wraps a Composio API 4xx as a 500 with the
-/// real validation message embedded in the body (OPENHUMAN-TAURI-3R / -3S
-/// / -97 — `"Backend returned 500 … Trigger type GITHUB_PUSH_EVENT not
-/// found"`, `"Backend returned 500 … Missing required fields: Your
-/// Subdomain"`). These would otherwise escape the 4xx-only matcher and
-/// fire as actionable Sentry events even though the underlying condition
-/// is user-state (the trigger slug isn't in composio's registry, the
-/// toolkit wasn't enabled by the user, the form field was left blank, …).
-///
-/// Also handles the gmail-sync 403 (OPENHUMAN-TAURI-33) where the
-/// composio sync loop surfaces the upstream Google OAuth scopes error as
-/// `"HTTP 403: Request had insufficient authentication scopes."`. The
-/// remediation is "user re-authorizes with the right scope" — nothing
-/// Sentry can act on.
-///
-/// All matches are substring-based against the lower-cased message so the
-/// classifier survives caller wrapping (rpc.invoke_method, agent.run_single,
-/// `[composio:gmail]` prefixes, anyhow chains, …).
-fn is_provider_user_state_message(lower: &str) -> bool {
- // TAURI-RUST-HXF: a direct BYO provider (groq `on_demand` free tier)
- // rejected a *single* request whose token count exceeds the account's
- // tokens-per-minute cap — `413 Payload Too Large … Request too large …
- // tokens per minute (TPM): Limit 8000, Requested 42084`. It is permanently
- // non-viable on the current tier (not a burst that retry/backoff clears)
- // and OpenHuman cannot raise a third-party account's TPM tier, so it is
- // user-config state, not a product bug. NOTE: a *managed-backend*
- // `PAYLOAD_TOO_LARGE` guard-leak is force-captured (returns `None`) earlier
- // in `expected_error_kind`, before this matcher runs, so this arm only ever
- // sees direct-provider TPM rejections. Shared matcher so the wording can't
- // drift.
- if tinyinference_llm::failure::is_provider_rate_cap_exceeded_message(lower) {
- return true;
- }
-
- // OPENHUMAN-TAURI-3R / -3S: composio enable_trigger when the slug isn't
- // in the trigger registry (e.g. user clicked a stale UI option).
- // Backend returns 500 with `"Trigger type GITHUB_PUSH_EVENT not found"`.
- // Also covers the alternate phrasing `"Cannot enable trigger … not found"`.
- if (lower.contains("trigger type ") && lower.contains("not found"))
- || (lower.contains("cannot enable trigger") && lower.contains("not found"))
- {
- return true;
- }
-
- // OPENHUMAN-TAURI-34: composio rejected a tool call because the user
- // hasn't enabled the toolkit yet. Wire shape:
- // `Backend returned 400 … Toolkit "get" is not enabled`.
- if lower.contains("toolkit ") && lower.contains("is not enabled") {
- return true;
- }
-
- // OPENHUMAN-TAURI-XX: custom_openai upstream rejected the request with
- // its own 400. Wire shape produced by
- // `inference/provider/compatible.rs::is_custom_openai_upstream_bad_request_http_400`:
- //
- // custom_openai API error (400 Bad Request): {"error":{
- // "message":"Bad request to upstream provider",
- // "type":"upstream_error","status":400}}
- //
- // Anchored to the `custom_openai api error (400` prefix so this can't
- // silence unrelated errors that happen to mention both
- // "bad request to upstream provider" and "upstream_error" elsewhere
- // (e.g. a future provider whose envelope reuses one of those strings).
- if lower.contains("custom_openai api error (400")
- && lower.contains("bad request to upstream provider")
- && lower.contains("upstream_error")
- {
- return true;
- }
-
- // OPENHUMAN-TAURI-97: composio authorize with a blank required field —
- // SharePoint Subdomain, WhatsApp WABA ID, Tenant Name, etc.
- // Backend returns 500 with `"Missing required fields: …"` body.
- //
- // **Intentionally broad** — unlike the trigger/toolkit arms, this is a
- // single substring with no second anchor. Composio's wire shape varies
- // per provider (`Missing required fields: Tenant Name`, `Missing
- // required fields: Your Subdomain (example: 'your-subdomain' for…)`,
- // `Missing required fields: WABA ID (WhatsApp Business Account ID…)`)
- // and embedding every variant would be brittle. Accepted false-positive
- // surface: a non-composio caller whose error happens to contain
- // `"missing required fields"` (e.g. `"Internal error: missing required
- // fields in config"`) will also demote to info. This is fine — every
- // current emit site routed through `report_error_or_expected` is scoped
- // to composio / integrations envelopes, so a stray collision would have
- // to come from a brand-new call site that explicitly opts in.
- // See `unrelated_missing_required_fields_classifies_as_accepted_false_positive`
- // for the documented surface.
- if lower.contains("missing required fields") {
- return true;
- }
-
- // OPENHUMAN-TAURI-33: gmail sync hit an OAuth scope wall —
- // `HTTP 403: Request had insufficient authentication scopes.`
- // (or any sibling OAuth scope rejection from composio's toolkits).
- if lower.contains("insufficient authentication scopes") {
- return true;
- }
-
- // OPENHUMAN-TAURI-S7: provider policy rejection on Kimi's coding
- // endpoint when requests are not sent from an approved coding-agent
- // client. Canonical body contains `access_terminated_error` and:
- // "currently only available for Coding Agents ...".
- if lower.contains("access_terminated_error")
- || lower.contains("currently only available for coding agents")
- {
- return true;
- }
-
- // TAURI-RUST-X9 (#1166): direct-mode composio call against the user's
- // personal Composio v3 tenant rejected with a 401 because the stored
- // API key is invalid / revoked / has the wrong prefix. The canonical
- // wire shape rendered by
- // `crates/openhuman-core/src/integrations/composio/tools/impl/network/composio.rs::response_error`
- // and the various direct-mode op wrappers is:
- //
- // `[composio-direct] list_connections failed: Composio v3
- // connected_accounts failed: HTTP 401: Invalid API key: ak_…`
- //
- // The "Invalid API key" body is rendered for every direct-mode
- // endpoint (list_connections / list_tools / authorize / etc.), so we
- // gate on the **`[composio-direct]` prefix** + either of the two
- // anchors that prove the failure came from the v3 auth wall:
- // - `HTTP 401` (the status the v3 wall returns)
- // - `Invalid API key` (the body Composio puts in the JSON)
- //
- // Requiring the `[composio-direct]` prefix keeps this from
- // accidentally swallowing unrelated bugs — backend-mode 401s from
- // `integrations/composio/*` still carry the `Backend returned 401`
- // shape (handled by the failure-tag flow with `status="401"`),
- // not the `HTTP 401: Invalid API key` shape.
- //
- // Remediation is purely user-state: the user must rotate / re-enter
- // their Composio key via Settings → Composio → Direct mode. Sentry
- // has no actionable signal — the UI surfaces the "Invalid API key"
- // toast and the polling layer already retries every 5 s.
- //
- // Drops Sentry TAURI-RUST-X9 (~15.7 k events / ~22 h, single user,
- // release [email protected]+c25fc8e5fd3e).
- if lower.contains("[composio-direct]")
- && (lower.contains("http 401") || lower.contains("invalid api key"))
- {
- return true;
- }
-
- // TAURI-RUST-K27 — the set-key sibling of X9. `composio_set_api_key`'s
- // validate-before-store probe (added in #4318) rejects an obviously invalid
- // BYO key *before* persisting it and returns its own user-facing prose,
- // `COMPOSIO_INVALID_API_KEY_USER_MESSAGE` ("Invalid Composio API key. Re-enter
- // a valid key in Connections > Composio."). That string carries
- // neither the `[composio-direct]` prefix nor an `HTTP 401` token, and the word
- // "Composio" splits the `invalid … api key` sequence — so the X9 arm above
- // never claims it and the RPC-boundary `report_error` leaked as a Sentry error
- // (473 events / 53 users, starting ~5 h after #4318 merged). Same user-state as
- // X9: an invalid/revoked BYO key with zero client-side lever to make it valid;
- // the Settings UI already surfaces the actionable re-entry copy. Anchor on the
- // distinctive `COMPOSIO_INVALID_API_KEY_ANCHOR` phrase — it can only be produced
- // by our own const (a genuine set-path defect renders a different `store_…
- // failed` / `save config failed` body that still pages). Keyed off the shared
- // anchor const (not a copied literal) and coupled to the typed source by
- // `demotes_composio_set_key_invalid_key_rejection` so a reword that drops the
- // phrase fails CI instead of silently re-opening the leak.
- if lower.contains(crate::integrations::composio::direct_auth::COMPOSIO_INVALID_API_KEY_ANCHOR) {
- return true;
- }
-
- // TAURI-RUST-34H — composio backend endpoint (e.g.
- // `/integrations/composio/connections`) wraps an upstream
- // Cloudflare anti-bot challenge as `Backend returned 500 Internal
- // Server Error … 403 <!DOCTYPE html>…<title>Just a moment...</title>…`.
- // The CF interstitial is keyed by the user's network reputation /
- // geo / cookie state — there is nothing in `openhuman_core` that
- // can act on it. Backend ops or the user's network is the
- // remediation path; Sentry has no signal.
- //
- // Double-anchor on the Cloudflare challenge title + the literal
- // "cloudflare" token to avoid colliding with unrelated bodies that
- // merely mention "Just a moment" in a different context.
- //
- // Drops ~8.9 k events / 14d (TAURI-RUST-34H, sibling -32G / -34J /
- // -323 share the same cascade).
- if lower.contains("just a moment...") && lower.contains("cloudflare") {
- return true;
- }
-
- // OPENHUMAN-TAURI-YJ: `inference/provider/models/catalog_listing.rs::list_configured_models` probed a
- // user-configured custom-provider's `/models` endpoint and the upstream
- // server returned 404. Wire shape emitted at `ops.rs:118-122`:
- //
- // "provider returned 404: {\"error\":\"path \\\"/api/v1/models\\\" not found\"}"
- //
- // (the trailing body is whatever the upstream server wrote — `{"error":...}`,
- // `{"detail":...}`, bare HTML, etc.; we only anchor on the `provider returned
- // 404` prefix). The semantic is unambiguous: the user pointed a custom
- // OpenAI-compatible provider at a base URL that does not host a `/models`
- // listing endpoint (wrong base, model-only proxy, typo'd path). The model
- // dropdown already surfaces the failure inline — Sentry has no remediation.
- //
- // **404 only**. Other 4xx from the same emit site stay actionable:
- // - 401 / 403: BYO-key auth wall — actionable misconfiguration; the
- // `does_not_classify_byo_key_provider_401_as_session_expired` contract
- // (#2286) intentionally keeps these in Sentry.
- // - 400: typically request-shape bugs in OUR client; must escalate.
- // - 429 / 5xx: transient — handled by other matchers / retry policy.
- //
- // No `inference/provider/models/catalog_listing.rs::list_configured_models` other than this site emits
- // the `provider returned NNN` prefix (verified via grep), so the prefix
- // alone is a sufficient anchor.
- //
- // TAURI-RUST-8X3: anchor to the position where `provider returned 404` is
- // the formatted *error prefix* — never to any occurrence in the response
- // body. The primary fix classifies the *raw* error at the source
- // (`inference/ops.rs::inference_list_models`) before any log prefix is
- // applied, so the raw shape always starts with `provider returned 404:`.
- // The one historically-observed prefixed re-report path is the
- // `inference/ops.rs` `error!("[inference::ops] list_models:error: {err}")`
- // log line — handled below as an explicit prefixed shape.
- //
- // A bare `contains` would mis-fire: a genuine 400/500 list-models failure
- // formats as `provider returned 500: <body>`, and if `<body>` merely
- // relays an upstream phrase like `upstream provider returned 404 ...`, the
- // loose substring would demote that real 4xx/5xx defect out of Sentry —
- // exactly the failures the discrimination guard
- // (`does_not_classify_non_404_list_models_failures_as_user_state`) says
- // must still escalate. So we require the anchor to be the prefix, not buried
- // text. (Mirrors the parenthesised `(401` anchoring in
- // `is_session_expired_message`.)
- if lower.starts_with("provider returned 404")
- || lower.contains("list_models:error: provider returned 404")
- {
- return true;
- }
-
- false
-}
-
-/// Detect "<capability> is disabled / unavailable for this RAM tier" errors
-/// emitted by the local-AI service when the user's hardware tier doesn't
-/// support a capability (OPENHUMAN-TAURI-3B: vision asset download invoked
-/// on a 0–4 GB tier). These are pure user-state conditions — the local-AI
-/// service surfaces them so the UI can prompt the user to switch tiers —
-/// and carry no remediable signal for Sentry.
-///
-/// The two canonical wire shapes today both contain `"for this ram tier"`:
-///
-/// - `"Vision is disabled for this RAM tier. Switch to the 4-8 GB tier or
-/// above to enable it."` — from `inference/local/service/assets.rs::ensure_capability_ready`
-/// - `"vision summaries are unavailable for this RAM tier. Use OCR-only
-/// summarization or switch to a higher local AI tier."` —
-/// from `inference/local/service/vision_embed.rs::summarize`
-///
-/// Anchor the classifier to that exact substring so an unrelated message
-/// that merely mentions "RAM tier" out of context is not silenced.
-fn is_local_ai_capability_unavailable_message(lower: &str) -> bool {
- lower.contains("for this ram tier")
-}
-
-/// Detect prompts rejected by the in-process prompt-injection guard.
-///
-/// Both enforcement actions that produce a user-visible error — `Blocked`
-/// (score ≥ 0.70) and `ReviewBlocked` (score ≥ 0.55) — share a unique
-/// prefix that cannot appear in any other error path. Anchored to the exact
-/// strings emitted by `prompt_guard_user_message` in
-/// `crates/openhuman-core/src/inference/local/ops.rs`.
-fn is_prompt_injection_blocked_message(lower: &str) -> bool {
- lower.contains("prompt flagged for security review")
- || lower.contains("prompt blocked by security policy")
-}
-
-/// Detect an RPC-level filesystem path validation failure from user input.
-///
-/// Anchored on the two known wire shapes — both emitted at the RPC entry
-/// boundary when a user typed/picked a path that doesn't resolve to an
-/// existing directory:
-///
-/// - `"root_path is not a directory: <path>"` — historically emitted by the
-/// now-removed knowledge-vault `vault_create` path when the chosen folder
-/// didn't exist or pointed at a file (Sentry TAURI-RUST-4QH). Kept as a
-/// classifier fixture since the wire shape may recur from other callers.
-/// - `"hosted path is not a directory: <path>"` —
-/// `openhuman_rpc::http_host::path_utils` when an HTTP host config
-/// references a missing directory. Not yet observed in Sentry but
-/// shares the same user-input failure mode; preempts a future ID.
-///
-/// Both are deterministic Err returns at the validation gate of an RPC
-/// handler, BEFORE any side-effect happens. The UI already surfaces the
-/// typed error and Sentry has no remediation path.
-///
-/// **Polarity contract** — explicit wire-shape anchors prevent accidental
-/// demotion of future errors whose bodies happen to contain "path is not
-/// a directory:" in a different context:
-///
-/// - `skills::ops_install` emits `"{path} is not a directory — refusing
-/// to remove"` (em-dash separator, no "root_path" or "hosted path"
-/// prefix). That is an `rm -rf` safety guard catching an UNEXPECTED
-/// state, not user input — it must STAY actionable.
-/// - A generic `"input config path is not a directory: /etc/foo"` from a
-/// future provider/wallet/storage error would NOT match (no known
-/// prefix) and would reach Sentry as intended.
-///
-/// All matches are substring-based against the lower-cased message so
-/// the classifier survives caller wrapping (`rpc.invoke_method`,
-/// anyhow context chains, …).
-fn is_filesystem_user_path_invalid_message(lower: &str) -> bool {
- lower.contains("root_path is not a directory:")
- || lower.contains("hosted path is not a directory:")
-}
-
-/// Detect the agent harness's empty-provider-response bail.
-///
-/// Anchored on the literal user-facing string emitted at
-/// `agent::session_host::turn` —
-/// `"The model returned an empty response. Please try again."` — which is
-/// preserved verbatim as the provider/model returns a body with
-/// `text_chars=0 thinking_chars=0 tool_calls=0`.
-///
-/// This catches the **web-channel re-report** (Sentry TAURI-RUST-4Z1):
-/// `web_chat::run_chat_task` wraps the failure as
-/// `"run_chat_task failed client_id=… error=The model returned an empty
-/// response. Please try again."` and routes it through
-/// `report_error_or_expected` after the typed
-/// `AgentError::EmptyProviderResponse` was flattened to a `String` at the
-/// native-bus boundary (so the agent-layer `skips_sentry()` suppression
-/// from PR #2790 can't reach it).
-///
-/// Anchored on `"model returned an empty response"` (not the looser
-/// `"empty response"`) so the sibling phrases stay actionable:
-/// `"summarizer returned empty response, falling through"`
-/// (`payload_summarizer`) and `"provider returned an empty response;
-/// returning empty extraction"` (`subagent_host::extract_tool`) are
-/// internal fall-through paths with different wording and are NOT
-/// silenced.
-fn is_empty_provider_response_message(lower: &str) -> bool {
- lower.contains("model returned an empty response")
-}
-
-/// Capture an error to Sentry with structured tags.
-///
-/// `domain` and `operation` are required and become tags `domain:<…>` and
-/// `operation:<…>`. `extra` is an optional list of extra tag pairs. The error
-/// itself is rendered via `Display` and emitted as a `tracing::error!` event,
-/// which the Sentry tracing layer turns into a Sentry event under the active
-/// scope.
-///
-/// Use stable, low-cardinality values for tag keys/values so Sentry can group
-/// and aggregate. High-cardinality data (full IDs, payloads) belongs in the
-/// error message body, not in tags.
-pub fn report_error<E: Display + ?Sized>(
- err: &E,
- domain: &str,
- operation: &str,
- extra: &[Tag<'_>],
-) {
- // Use the alternate format specifier so `anyhow::Error` renders its full
- // context chain (outer context + every wrapped cause, joined by ": ").
- // Plain `Display` impls fall back to the standard representation. Without
- // this, anyhow's default `to_string()` only emits the outermost context
- // and the underlying cause (e.g. a `toml::de::Error` with line/column) is
- // dropped — making the captured Sentry event undiagnosable. See
- // OPENHUMAN-TAURI-B2 for an instance where this masked the real failure.
- let message = format!("{err:#}");
- report_error_message(&message, domain, operation, extra);
-}
-
-/// Report an error unless it is an expected user-state/config condition.
-///
-/// Expected conditions are logged at `info` or `warn` so the Sentry tracing
-/// layer records at most a breadcrumb, not an error event.
-pub fn report_error_or_expected<E: Display + ?Sized>(
- err: &E,
- domain: &str,
- operation: &str,
- extra: &[Tag<'_>],
-) {
- let message = format!("{err:#}");
- if let Some(kind) = expected_error_kind(&message) {
- report_expected_message(kind, &message, domain, operation);
- return;
- }
- report_error_message(&message, domain, operation, extra);
-}
-
-fn report_expected_message(kind: ExpectedErrorKind, message: &str, domain: &str, operation: &str) {
- match kind {
- ExpectedErrorKind::LocalAiDisabled => {
- tracing::info!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected local-ai disabled error: {message}"
- );
- }
- ExpectedErrorKind::ApiKeyMissing => {
- tracing::warn!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected API-key configuration error: {message}"
- );
- }
- ExpectedErrorKind::NetworkUnreachable => {
- tracing::warn!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected network-unreachable error: {message}"
- );
- }
- ExpectedErrorKind::TransientUpstreamHttp => {
- tracing::warn!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped transient upstream HTTP error: {message}"
- );
- }
- ExpectedErrorKind::LocalAiBinaryMissing => {
- // User-state condition: the piper binary (local TTS) isn't
- // installed on this host; OpenHuman never installs it. The error message itself is
- // the user-facing instruction ("Set PIPER_BIN or install
- // piper.") — Sentry has nothing to act on, since we can't
- // install the binary for them. OPENHUMAN-TAURI-9N is the
- // canonical instance: `local_ai_tts` fails immediately
- // (elapsed_ms=1) on a Windows host without piper installed.
- tracing::info!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected local-ai binary-missing error: {message}"
- );
- }
- ExpectedErrorKind::BackendUserError => {
- // 4xx from the integrations / composio backend client —
- // user-input or auth-state failure that the backend already
- // surfaced to the user via the structured error toast.
- // OPENHUMAN-TAURI-BC: SharePoint authorize 400 because the
- // user didn't fill in the required Tenant Name field.
- tracing::warn!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected backend user-error response: {message}"
- );
- }
- ExpectedErrorKind::ProviderUserState => {
- // Third-party provider (composio, gmail OAuth, …) rejected the
- // request for a user-state reason: trigger slug missing from
- // composio's registry (OPENHUMAN-TAURI-3R / -3S), toolkit not
- // enabled (OPENHUMAN-TAURI-34), OAuth scopes missing
- // (OPENHUMAN-TAURI-33), or a required form field was left blank
- // (OPENHUMAN-TAURI-97). The UI already surfaces the actionable
- // error to the user — Sentry has no remediation path.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "provider_user_state",
- error = %message,
- "[observability] {domain}.{operation} skipped expected provider-user-state error: {message}"
- );
- }
- ExpectedErrorKind::McpServerNeedsAuth => {
- // A remote MCP server rejected the connect handshake with HTTP 401:
- // it needs OAuth sign-in. `mcp::registry::connections::connect`
- // already stores a `needs_auth` flag and the UI prompts the user to
- // authenticate (#3733 / #3719) — but `mcp_clients_connect` re-raises
- // the stringified error to the RPC dispatcher, where it was being
- // captured as a full Sentry error (TAURI-RUST-CGP: ~1.2k events / 79
- // users). Preventable user-state with no Sentry-actionable signal;
- // demote to info so the breadcrumb survives but no error event fires.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "mcp_server_needs_auth",
- error = %message,
- "[observability] {domain}.{operation} skipped expected MCP needs-auth (401) error: {message}"
- );
- }
- ExpectedErrorKind::WalletNotConfigured => {
- // The user has not set up a wallet. That is the default state of an
- // optional feature, the UI already prompts for setup, and no
- // core-side change makes the call succeed — so demote to info: the
- // breadcrumb survives for correlation, no error event fires.
- // See `ExpectedErrorKind::WalletNotConfigured` (#5805).
- //
- // The raw message is deliberately NOT logged, and that follows from
- // how this kind is matched. The classifier accepts the sentinel
- // anywhere in the string so the demotion survives context wrapping —
- // which means everything *around* the sentinel is arbitrary
- // caller-supplied text. Today's wrappers are tame (`self_identity
- // key_status: …`), but nothing constrains a future one, and a
- // wrapper is exactly where an id, a path or a pasted value ends up.
- // The permissive matcher is the right trade for correctness; paying
- // for it with a careful log is the other half of that trade.
- //
- // Nothing is lost: the only part of the body this arm can vouch for
- // is the sentinel itself, and it is a constant. `domain` and
- // `operation` carry the correlation, which is what a breadcrumb is
- // for. Same reasoning as the param-validation skip in
- // `openhuman-rpc/src/server/http/rpc_handler.rs`, which redacts because its messages embed
- // caller-supplied param names.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "wallet_not_configured",
- "[observability] {domain}.{operation} skipped expected wallet-not-configured \
- error (message withheld: the wrapper around the sentinel is caller-supplied)"
- );
- }
- ExpectedErrorKind::ProviderConfigRejection => {
- // User-config state: a custom cloud provider rejected the
- // request because of the user's model / parameter setup — an
- // OpenHuman abstract tier alias leaked to a provider that only
- // speaks its native ids (#2079), an unknown / stale model pin
- // (#2202), or a model-specific temperature constraint (#2076,
- // Moonshot Kimi K2). The provider HTTP layer already demoted
- // its own per-attempt event; this is the re-report raised
- // again by agent.run_single / web_channel.run_chat_task. The
- // UI surfaces an actionable "fix your model/provider settings"
- // error — Sentry has no remediation path
- // (OPENHUMAN-TAURI-WJ / -QW / -HB / -NH).
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "provider_config_rejection",
- error = %message,
- "[observability] {domain}.{operation} skipped expected provider config-rejection error: {message}"
- );
- }
- ExpectedErrorKind::LocalAiCapabilityUnavailable => {
- // User-state condition: the local-AI service refused a
- // capability (vision summarization, vision asset download)
- // because the user's RAM tier doesn't support it. The
- // error message itself is the user-facing remediation
- // ("Switch to the 4-8 GB tier or above to enable it.") —
- // Sentry has nothing to act on. OPENHUMAN-TAURI-3B: 28
- // hits in 4 days from `local_ai_download_asset` on a
- // 0–4 GB tier requesting vision.
- tracing::info!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected local-ai capability-unavailable error: {message}"
- );
- }
- ExpectedErrorKind::BudgetExhausted => {
- // User-state condition: the backend reports the user is out of
- // budget / credits / balance (HTTP 400 from the OpenHuman backend,
- // surfaced by `providers::is_budget_exhausted_message`). The UI
- // already surfaces this as an actionable toast — Sentry would
- // turn each affected turn into noise (OPENHUMAN-TAURI-3M / -12 /
- // -13). Demote to info so it still appears in breadcrumbs but
- // never spawns a Sentry error event.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "budget",
- error = %message,
- "[observability] {domain}.{operation} skipped expected budget-exhausted error: {message}"
- );
- }
- ExpectedErrorKind::ModuleUnavailable => log_module_unavailable(domain, operation, message),
- ExpectedErrorKind::BackendUnavailable => {
- // Build-state condition: no hosted backend is registered, so
- // the hosted backend is unreachable by construction. Nothing to
- // fix in Sentry — the host chose a backend-less core.
- tracing::debug!(
- domain = domain,
- operation = operation,
- kind = "backend_unavailable",
- error = %message,
- "[observability] {domain}.{operation} skipped expected backend-unavailable error: {message}"
- );
- }
- ExpectedErrorKind::SessionExpired => {
- // Auth-boundary condition: the user's JWT expired (or was never
- // present). The JSON-RPC dispatch layer already handles the
- // teardown — `Err` propagation publishes `DomainEvent::SessionExpired`
- // which clears the stored token and flips the scheduler-gate
- // signed-out override so background workers stand down — and the
- // UI re-auths the user. The per-attempt error event from the
- // upstream call site (agent.run_single, web_channel.run_chat_task)
- // adds noise without signal: every mid-conversation 401 would
- // emit one event before the cascade dampener kicks in
- // (OPENHUMAN-TAURI-26, and the same upstream gap that
- // OPENHUMAN-TAURI-1T's #1516 cascade fix dampened but did not
- // close). Demote to info so the breadcrumb survives for trace
- // correlation but Sentry sees no error event.
- tracing::info!(
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} skipped expected session-expired error: {message}"
- );
- }
- ExpectedErrorKind::LoopbackUnavailable => {
- // In-process-core boot-window condition: a sibling component
- // tried to reach `127.0.0.1:<port>` before the embedded core's
- // HTTP listener finished binding (OPENHUMAN-TAURI-R5 / -R6).
- // Self-resolves once startup completes. Demote at `debug!` —
- // lower than the `warn!` we use for NetworkUnreachable because
- // this isn't a user-environment problem; it's an internal
- // lifecycle race that always recovers. We deliberately drop the
- // raw `message` from the structured fields and format string and
- // log only `domain` / `operation` / `kind` — the body adds no
- // remediation signal (the URL is always loopback, the error is
- // always "Connection refused") and keeping the breadcrumb sparse
- // mirrors the per-#1719 review feedback (metadata over raw text
- // for noise demotions).
- tracing::debug!(
- domain = domain,
- operation = operation,
- kind = "loopback_unavailable",
- "[observability] {domain}.{operation} skipped expected loopback-unavailable error"
- );
- }
- ExpectedErrorKind::PromptInjectionBlocked => {
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "prompt_injection_blocked",
- "[observability] {domain}.{operation} skipped expected prompt-injection-blocked error"
- );
- }
- ExpectedErrorKind::ContextWindowExceeded => {
- // Request too long for the model's context window. The provider
- // api_error cascade already demotes its own emit; this is the
- // higher-layer re-report. Deterministic user-state — the UI
- // shows the retry message and the user trims / starts a new
- // chat. Demote to `warn!` (breadcrumb only) — same tier as the
- // other usage-state conditions.
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "context_window_exceeded",
- error = %message,
- "[observability] {domain}.{operation} skipped expected context-window-exceeded error: {message}"
- );
- }
- ExpectedErrorKind::DiskFull => {
- // Host filesystem out of space. The user must free space on
- // their machine — Sentry can't help. Demote at `warn!` so a
- // sustained spike still shows up in operator dashboards
- // without turning every affected user-session into a Sentry
- // error event. Drops TAURI-RUST-H4.
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "disk_full",
- "[observability] {domain}.{operation} skipped expected disk-full error"
- );
- }
- ExpectedErrorKind::WindowsFileSystemLimitation => {
- // Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665) —
- // caused by fragmentation, USN journal overflow, or a
- // filesystem filter driver bottleneck. The user must restart
- // their machine, run a defrag, or free disk space — Sentry
- // has no remediation path. Demote at `warn!` so a sustained
- // spike shows up in dashboards without flooding Sentry.
- // Drops TAURI-RUST-QT0 (6,050 events / 1 user).
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "windows_file_system_limitation",
- "[observability] {domain}.{operation} skipped expected Windows file-system-limitation error (os error 665)"
- );
- }
- ExpectedErrorKind::FilesystemUserPathInvalid => {
- // User-input validation failure surfaced at the RPC
- // boundary — e.g. `openhuman.vault_create` called with a
- // `root_path` that doesn't exist. The typed error is
- // already shown to the user; Sentry has no remediation
- // path. Demote to `info!` — same tier as
- // `PromptInjectionBlocked`, which is the closest severity
- // class ("user input we already surfaced a typed error for";
- // not operator-actionable like `DiskFull` / `NetworkUnreachable`).
- //
- // **Do not include the raw `message` here.** The message
- // body embeds the user's local filesystem layout (username,
- // project name, document directory, …) and
- // `sentry_tracing_layer` in `core::logging` maps
- // `Level::INFO` to `EventFilter::Breadcrumb` — so any
- // formatted body would be attached as a breadcrumb to
- // every subsequent Sentry event from this hub, leaking
- // user paths into unrelated reports. Log only `domain` /
- // `operation` / `kind` (no PII), matching the
- // `LoopbackUnavailable` arm above ("metadata over raw text
- // for noise demotions", per the #1719 review feedback).
- // Full-path diagnostics for local debugging stay available
- // via `RUST_LOG=…=debug` since `Level::DEBUG` / `TRACE`
- // are mapped to `EventFilter::Ignore`.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "filesystem_user_path_invalid",
- "[observability] {domain}.{operation} skipped expected filesystem path validation error"
- );
- }
- ExpectedErrorKind::EmptyProviderResponse => {
- // Model/user-config condition — the provider returned a
- // completely empty body and the agent harness bailed with the
- // user-facing retry message. The agent layer already suppresses
- // this via the typed `AgentError::skips_sentry()` (PR #2790);
- // this arm covers the `web_channel.run_chat_task` re-report
- // where the type was flattened to a String. Demote to `warn!`
- // (breadcrumb only) — same tier as `MaxIterationsExceeded`,
- // the other deterministic agent-state outcome surfaced to the
- // user via the `chat_error` event.
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "empty_provider_response",
- error = %message,
- "[observability] {domain}.{operation} skipped expected empty-provider-response error: {message}"
- );
- }
- ExpectedErrorKind::ChannelSupervisorRestart => {
- // Channel supervisor caught a transient error from a channel
- // listener (`spawn_supervised_listener`) and restarted it. The
- // wrapper is language-agnostic — anchored on the Rust supervisor
- // shape, not the inner error wording — so this catches both the
- // English Discord-gateway body (TAURI-RUST-15 ~11.4 k events) and
- // OS-localized variants (TAURI-RUST-BB Chinese WSAETIMEDOUT,
- // ~815 events) that the English-only `NetworkUnreachable`
- // matchers miss. Self-resolving via the supervisor's exponential
- // backoff — Sentry has no remediation path. Sustained outages
- // still surface through `health.bus` / `FAIL_ESCALATE_THRESHOLD`
- // (separate code path, not affected by this demotion). Demote to
- // `info!` so the breadcrumb survives for trace correlation but
- // Sentry sees no error or warn event.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "channel_supervisor_restart",
- error = %message,
- "[observability] {domain}.{operation} skipped expected channel-supervisor restart: {message}"
- );
- }
- ExpectedErrorKind::ConfigLoadTimedOut => {
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "config_load_timed_out",
- error = %message,
- "[observability] {domain}.{operation} skipped expected config-load timeout: {message}"
- );
- }
- ExpectedErrorKind::ConfigReadIoFailure => {
- // OS refused to read an existing config.toml (ACL-denied, locked by
- // another process, OneDrive placeholder). User-environment state —
- // we cannot make the file readable, and the same poll re-reports it
- // every cycle (TAURI-RUST-DME). Demote at `warn!` so it stays in the
- // local log for support without paging on every poll.
- // Metadata-only: the raw message embeds the absolute config path
- // (username / home dir). Keep this arm PII-free like the other
- // path-sensitive demotions — domain/operation/kind are enough to
- // see the condition without leaking the path into local logs.
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "config_read_io_failure",
- "[observability] {domain}.{operation} skipped expected config-read io failure (OS access denied/locked)"
- );
- }
- ExpectedErrorKind::CodexCliAuthUnavailable => {
- // User-state condition: the Codex CLI login at `~/.codex/auth.json`
- // is missing / unparseable / has no tokens. The import RPC already
- // returned the actionable "Run `codex login` first" string and the
- // UI surfaces it inline — Sentry has nothing to act on. Demote at
- // `info!` (mirrors `LocalAiBinaryMissing`). Do NOT include the raw
- // `message`: it embeds the absolute `~/.codex/auth.json` path
- // (home dir / username). Log only domain/operation/kind — no PII.
- tracing::info!(
- domain = domain,
- operation = operation,
- kind = "codex_cli_auth_unavailable",
- "[observability] {domain}.{operation} skipped expected codex-cli auth-unavailable error"
- );
- }
- ExpectedErrorKind::BackendErrorCodeOwned => {
- // The classifier's canonical token aids triage without re-parsing
- // provider-specific strings in this generic module.
- let code = crate::inference::registry::classify_backend_error(None, message)
- .and_then(|classification| classification.token)
- .unwrap_or_default();
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "backend_error_code",
- error_code = %code,
- "[observability] {domain}.{operation} skipped backend-owned errorCode={code} error: {message}"
- );
- }
- ExpectedErrorKind::UpstreamEdgeBlock => {
- // Provider embed call hit an edge/CDN/WAF or regional 403 block —
- // the body is a generic HTML gateway page, not the provider's JSON
- // error. Endpoint correct, key sent, retry can't clear an edge
- // policy; the embedding caller degrades gracefully and the UI
- // surfaces it. Demote at `warn!` so the breadcrumb retains the
- // shape for triage without spawning an event (TAURI-RUST-8S3).
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "upstream_edge_block",
- error = %message,
- "[observability] {domain}.{operation} skipped upstream edge/CDN 403 block: {message}"
- );
- }
- ExpectedErrorKind::ApprovalNoPendingRace => {
- // Benign approval race (TAURI-RUST-5EH): the request_id was already
- // decided / lazily expired / superseded — `store::get_decision`
- // confirmed a persisted decision exists, so this is a double-tap,
- // two-operator, or expiry-while-live race classified benign by the
- // inline-approvals design spec, not a lost registration. Demote at
- // `warn!` so a sustained spike still shows in operator dashboards
- // without paging. The genuine never-registered case takes the
- // capture path (distinct wording) and stays a Sentry signal.
- tracing::warn!(
- domain = domain,
- operation = operation,
- kind = "approval_no_pending_race",
- "[observability] {domain}.{operation} skipped benign already-decided/expired approval race: {message}"
- );
- }
- }
-}
-
-/// Distinct `tracing::Metadata::target()` we set on the diagnostic
-/// `tracing::error!` emitted from [`report_error_message`].
-///
-/// Sentry capture for this helper happens via an explicit
-/// `sentry::capture_message` call below — not via the `sentry-tracing`
-/// layer scooping up the `tracing::error!` event. The production
-/// `sentry_tracing_layer()` in `core::logging` filters events with this
-/// target to `EventFilter::Ignore` so we never double-report (one direct
-/// `capture_message`, one tracing-bridge capture of the same condition).
-///
-/// Why direct capture instead of relying on the bridge: the bridge worked
-/// in steady-state but flaked under parallel test scheduling
-/// (`thread_not_found_rpc_error_does_not_report_to_sentry` repeatedly hit
-/// `events.len() == 0` in CI even with a thread-default subscriber wired
-/// up — likely a Linux-only thread-local ordering quirk in
-/// `sentry-tracing`'s `Hub::current()` lookup at event-emit time). Direct
-/// `sentry::capture_message` synchronously routes through the active hub
-/// and is deterministic, which keeps both production reporting and tests
-/// honest.
-pub const REPORT_ERROR_TRACING_TARGET: &str = "openhuman::observability::report_error";
-
-/// Cooldown period for rate-limiting repeated filesystem-limit errors.
-/// Within this window, the same (domain, operation) pair that carries
-/// `(os error 665)` is suppressed — Sentry never receives a duplicate
-/// Sentry event, and only a `debug!` log line is emitted.
-#[cfg(feature = "crash-reporting")]
-const FS_ERROR_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(300); // 5 min
-
-/// In-process state tracking the last-reported time of `ERROR_FILE_SYSTEM_LIMITATION`
-/// (os error 665) events, keyed by `(domain, operation)`. This prevents
-/// unthrottled retry loops from flooding Sentry (TAURI-RUST-QT0: 6,050 events /
-/// 1 user).
-#[cfg(feature = "crash-reporting")]
-static LAST_FS_LIMIT_REPORT: std::sync::LazyLock<
- std::sync::Mutex<std::collections::HashMap<(String, String), std::time::Instant>>,
-> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new()));
-
-/// Returns `true` when `(domain, operation)` was already reported with
-/// an error-665 message within [`FS_ERROR_COOLDOWN`]. Cleans up stale
-/// entries older than the cooldown on every call.
-#[cfg(feature = "crash-reporting")]
-fn was_recently_reported(domain: &str, operation: &str) -> bool {
- let mut guard = LAST_FS_LIMIT_REPORT
- .lock()
- .unwrap_or_else(|e| e.into_inner());
- let now = std::time::Instant::now();
- // Prune stale entries older than cooldown (single-pass to keep map bounded).
- guard.retain(|_, last| now.duration_since(*last) < FS_ERROR_COOLDOWN);
- let key = (domain.to_string(), operation.to_string());
- if let Some(last) = guard.get(&key) {
- if now.duration_since(*last) < FS_ERROR_COOLDOWN {
- return true;
- }
- }
- guard.insert(key, now);
- false
-}
-
-pub(crate) fn report_error_message(
- message: &str,
- domain: &str,
- operation: &str,
- extra: &[Tag<'_>],
-) {
- // Redact secret-looking spans (bearer tokens, API keys, `sk-` keys) before
- // `message` reaches any log sink or Sentry event. The parallel `tracing`
- // log line below is emitted in every build — including slim builds with no
- // `crash-reporting` `before_send` hook — so scrub once, up front.
- let scrubbed = crate::core::log_redaction::scrub_secrets(message);
- let message = scrubbed.as_str();
- // Rate-limit Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665)
- // events: skip the Sentry capture if the same (domain, operation) pair
- // was already reported within the last 5 minutes. This prevents
- // unthrottled retry loops from flooding Sentry (TAURI-RUST-QT0:
- // 6,050 events / 1 user). The `before_send` chain also filters these,
- // but this early-exit avoids the Sentry scope overhead entirely.
- #[cfg(feature = "crash-reporting")]
- if message.contains("os error 665") && was_recently_reported(domain, operation) {
- tracing::debug!(
- target: REPORT_ERROR_TRACING_TARGET,
- domain = domain,
- operation = operation,
- "[observability] {domain}.{operation} rate-limited os error 665 (reported within cooldown)"
- );
- return;
- }
- // Sentry-touching behaviour is gated behind `crash-reporting`. The
- // diagnostic `tracing::error!` stays compiled in both builds (see the
- // `#[cfg(not(...))]` companion below) so stderr / file appenders keep the
- // record even in a sentry-free build.
- #[cfg(feature = "crash-reporting")]
- sentry::with_scope(
- |scope| {
- scope.set_tag("domain", domain);
- scope.set_tag("operation", operation);
- for (k, v) in extra {
- scope.set_tag(k, v);
- }
- },
- || {
- // Direct, synchronous Sentry capture — see
- // `REPORT_ERROR_TRACING_TARGET` for why we don't rely on the
- // `sentry-tracing` layer for this call site.
- sentry::capture_message(message, sentry::Level::Error);
- // Diagnostic log line for stderr / file appenders. Tagged with
- // the marker target so the production sentry-tracing layer
- // skips it (no double Sentry event).
- tracing::error!(
- target: REPORT_ERROR_TRACING_TARGET,
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} failed: {message}"
- );
- },
- );
- #[cfg(not(feature = "crash-reporting"))]
- {
- // Sentry compiled out: `extra` tags have no scope to attach to, so
- // discard them explicitly to avoid an unused-variable warning while
- // still emitting the diagnostic log line.
- let _ = extra;
- tracing::error!(
- target: REPORT_ERROR_TRACING_TARGET,
- domain = domain,
- operation = operation,
- error = %message,
- "[observability] {domain}.{operation} failed: {message}"
- );
- }
-}
-
-/// Capture a message to Sentry at **warning** severity with structured tags.
-///
-/// Mirror of [`report_error_message`] but at `sentry::Level::Warning`: the
-/// event is still recorded in Sentry (so it stays available for triage and
-/// dashboards) while warning-severity events do not trip the error-rate
-/// alert/paging rules that `Level::Error` events do (see the
-/// `sentry_tracing_layer` mapping in `core::logging`, where `ERROR` becomes a
-/// captured `Event` and `WARN`/`INFO` only a `Breadcrumb`). Use this for
-/// transport-boundary conditions worth seeing in aggregate that are never an
-/// actionable core defect — e.g. unrecognised RPC method names (#3567).
-///
-/// Like [`report_error_message`], capture is an explicit, synchronous
-/// `sentry::capture_message` rather than the `sentry-tracing` bridge; the
-/// accompanying diagnostic line is tagged with [`REPORT_ERROR_TRACING_TARGET`]
-/// so the production layer ignores it and we never double-report.
-// Its sole caller is the `http-server`-gated RPC handler (unrecognised-method
-// reporting, #3567), so it has no caller in a slim build (#5048). Kept compiled
-// for the crash-reporting carve-out; the allow keeps the disabled build quiet.
-#[cfg_attr(not(feature = "http-server"), allow(dead_code))]
-pub fn report_warning_message(message: &str, domain: &str, operation: &str, extra: &[Tag<'_>]) {
- // Redact secret-looking spans before `message` reaches any log sink or
- // Sentry event — see the note in `report_error_message`.
- let scrubbed = crate::core::log_redaction::scrub_secrets(message);
- let message = scrubbed.as_str();
- // Sentry-touching behaviour is gated behind `crash-reporting`; the
- // diagnostic `tracing::warn!` stays compiled in both builds (see the
- // `#[cfg(not(...))]` companion below).
- #[cfg(feature = "crash-reporting")]
- sentry::with_scope(
- |scope| {
- scope.set_tag("domain", domain);
- scope.set_tag("operation", operation);
- for (k, v) in extra {
- scope.set_tag(k, v);
- }
- },
- || {
- sentry::capture_message(message, sentry::Level::Warning);
- tracing::warn!(
- target: REPORT_ERROR_TRACING_TARGET,
- domain = domain,
- operation = operation,
- message = %message,
- "[observability] {domain}.{operation} warning: {message}"
- );
- },
- );
- #[cfg(not(feature = "crash-reporting"))]
- {
- let _ = extra;
- tracing::warn!(
- target: REPORT_ERROR_TRACING_TARGET,
- domain = domain,
- operation = operation,
- message = %message,
- "[observability] {domain}.{operation} warning: {message}"
- );
- }
-}
-
-/// Returns true when a Sentry event is a per-attempt provider HTTP failure
-/// that the reliable-provider layer already handles via retry + fallback.
-///
-/// The primary suppression lives at the call site
-/// (`crate::inference::provider::http_error::should_report_provider_http_failure`),
-/// which short-circuits transient codes before `report_error` ever fires.
-/// This helper is intended for use inside the `sentry::ClientOptions`
-/// `before_send` hook as defense-in-depth — it catches any future call
-/// site that emits a `tracing::error!` with the same shape but bypasses
-/// the classifier.
-///
-/// Match criteria (all required):
-/// - tag `domain == "llm_provider"` — pins the filter to provider-originated
-/// events so an unrelated subsystem emitting `failure=non_2xx`/`status=503`
-/// for its own reasons doesn't get silently dropped
-/// - tag `failure == "non_2xx"` (the marker set by `ops::api_error`)
-/// - tag `status` parses to one of [`TRANSIENT_PROVIDER_HTTP_STATUSES`]
-#[cfg(feature = "crash-reporting")]
-pub fn is_transient_provider_http_failure(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some("llm_provider") {
- return false;
- }
- if tags.get("failure").map(String::as_str) != Some("non_2xx") {
- return false;
- }
- let Some(status_u16) = tags.get("status").and_then(|s| s.parse::<u16>().ok()) else {
- return false;
- };
- tinyinference_llm::failure::TRANSIENT_PROVIDER_HTTP_STATUSES.contains(&status_u16)
-}
-
-/// Defense-in-depth `before_send` filter for managed-backend `errorCode`
-/// events (#870): drops any Sentry event whose message/exception text carries a
-/// backend `errorCode` that the backend owns (F2/F4) — *except* a backend-flagged
-/// malformed `BAD_REQUEST`, which the client caused and so still pages (F8).
-///
-/// Primary suppression lives at the emit sites (`api_error` /
-/// `compatible_*` streaming gates) and at the higher-layer re-report
-/// classifier (`expected_error_kind` → [`ExpectedErrorKind::BackendErrorCodeOwned`]).
-/// This catches any future call site that re-emits the same flattened error
-/// without routing through those funnels. Delegates the decision to the
-/// single-source [`crate::inference::provider::managed_error_skips_sentry`]
-/// (managed-envelope gated, so a BYO payload carrying an `errorCode`-shaped
-/// field is not wrongly dropped) so the layers can't drift.
-#[cfg(feature = "crash-reporting")]
-pub fn is_backend_error_code_event(event: &sentry::protocol::Event<'_>) -> bool {
- let direct = event.message.as_deref();
- let from_logentry = event.logentry.as_ref().map(|log| log.message.as_str());
- let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
- [direct, from_logentry, from_exception]
- .into_iter()
- .flatten()
- .any(|message| {
- crate::inference::registry::classify_backend_error(None, message).is_some_and(
- |classification| classification.backend_owned && !classification.reportable,
- )
- })
-}
-
-/// Defense-in-depth `before_send` filter for transient streaming **transport**
-/// failures (F7): drops `domain=llm_provider, failure=transport` events whose
-/// body is a transient transport phrase (timeout / reset / TLS-handshake EOF /
-/// "error sending request"). Flaky-network blips on the streaming send are
-/// recovered by retry/fallback and carry no actionable Sentry signal; a
-/// non-transient transport failure (DNS misconfig, unexpected protocol error)
-/// is not matched and still pages.
-///
-/// Primary gate lives at the two streaming emit sites in
-/// `compatible_provider_impl.rs` (`stream_chat` / `stream_chat_history`); this
-/// catches any future site that reports the same shape without gating.
-///
-/// Scoped to the **streaming** operations on purpose (CodeRabbit): only the
-/// `stream_chat` / `stream_chat_history` transport emits are meant to be
-/// suppressed. A non-streaming `domain=llm_provider, failure=transport` event
-/// carries a different `operation` tag and must keep paging, so the
-/// observability blind spot stays as narrow as F7 intends.
-#[cfg(feature = "crash-reporting")]
-pub fn is_transient_provider_transport_failure(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some("llm_provider") {
- return false;
- }
- if tags.get("failure").map(String::as_str) != Some("transport") {
- return false;
- }
- if !matches!(
- tags.get("operation").map(String::as_str),
- Some("stream_chat") | Some("stream_chat_history")
- ) {
- return false;
- }
- event_has_transient_transport_phrase(event)
-}
-
-/// Defense-in-depth filter for aggregate provider exhaustion events where the
-/// aggregate only restates transient attempt failures.
-///
-/// Keep ordinary `failure=all_exhausted` events: they are the useful "every
-/// fallback failed" signal. Drop only the narrow shape observed in #3542,
-/// where the aggregate body starts with the reliable-provider exhaustion
-/// prefix and contains transient HTTP/transport wording already classified by
-/// [`is_transient_message_failure`].
-#[cfg(feature = "crash-reporting")]
-pub fn is_all_transient_provider_exhaustion_event(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some("llm_provider") {
- return false;
- }
- if tags.get("failure").map(String::as_str) != Some("all_exhausted") {
- return false;
- }
-
- let direct = event.message.as_deref();
- let from_logentry = event.logentry.as_ref().map(|log| log.message.as_str());
- let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
- [direct, from_logentry, from_exception]
- .into_iter()
- .flatten()
- .any(all_provider_attempts_are_transient)
-}
-
-#[cfg(feature = "crash-reporting")]
-fn all_provider_attempts_are_transient(message: &str) -> bool {
- let Some(attempts) = message.strip_prefix("All providers/models failed. Attempts:") else {
- return false;
- };
- let mut saw_attempt = false;
- for attempt in attempts.split(';').map(str::trim).filter(|s| !s.is_empty()) {
- saw_attempt = true;
- if !is_transient_message_failure(attempt) {
- return false;
- }
- }
- saw_attempt
-}
-
-/// Returns true when a Sentry event's message/exception text contains the
-/// canonical max-tool-iterations cap phrase (see
-/// `tinyagents_harness::agent_error::MAX_ITERATIONS_ERROR_PREFIX`).
-///
-/// Defense-in-depth filter for the Sentry `before_send` hook: the primary
-/// suppression lives at the call sites in `agent::session_host::
-/// runtime::run_single`, `channels::runtime::dispatch`, and
-/// `web_chat::run_chat_task`, all of which now skip
-/// `report_error` when this variant is detected. This filter catches any
-/// future call site that re-emits the message without going through those
-/// funnels — e.g. a new wrapper that calls `tracing::error!` directly with
-/// the typed error rendering — and keeps OPENHUMAN-TAURI-99 / -98
-/// permanently off Sentry without requiring touch-ups at each new site.
-///
-/// Match strategy: scans `event.message` first (the path used by
-/// `report_error_message` → `sentry::capture_message`) and falls back to
-/// the last exception's `value` (the shape `sentry-tracing` produces when
-/// stacktraces are attached). Both fields are checked for the canonical
-/// prefix so the filter stays robust to future Sentry plumbing changes.
-#[cfg(feature = "crash-reporting")]
-pub fn is_max_iterations_event(event: &sentry::protocol::Event<'_>) -> bool {
- let direct = event.message.as_deref();
- let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
- [direct, from_exception]
- .into_iter()
- .flatten()
- .any(tinyagents_harness::agent_error::is_max_iterations_error)
-}
-
-/// Tag + body classifier for the `before_send` chain — drops Sentry events
-/// emitted at the OpenHuman backend / rpc layers for "401 Session
-/// expired" or the pre-flight "no session token stored" guards.
-///
-/// Pairs with [`is_session_expired_message`] (which classifies the
-/// message body at the emit site via `report_error_or_expected`). This
-/// fn runs in `before_send` so it catches any future call site that
-/// re-emits the same shape without routing through the classifier —
-/// keeps OPENHUMAN-TAURI-25 / -1Q / -27 / -1G permanently off Sentry
-/// (~185 events/day combined).
-///
-/// Scope: only the three domains that surface session-expired today
-/// (`llm_provider`, `backend_api`, `rpc`). Composio's OAuth-state 401
-/// is excluded — that's actionable and must reach Sentry.
-#[cfg(feature = "crash-reporting")]
-pub fn is_session_expired_event(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- let Some(domain) = tags.get("domain").map(String::as_str) else {
- return false;
- };
- if !matches!(domain, "llm_provider" | "backend_api" | "rpc") {
- return false;
- }
-
- let status_is_401 = tags
- .get("status")
- .and_then(|s| s.parse::<u16>().ok())
- .is_some_and(|code| code == 401);
-
- let direct = event.message.as_deref();
- let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
- let body_matches = [direct, from_exception]
- .into_iter()
- .flatten()
- .any(is_session_expired_message);
-
- if status_is_401 && body_matches {
- return true;
- }
-
- // Pre-flight rpc guard has no status tag — accept on body alone,
- // scoped to the rpc dispatcher (other domains don't emit the
- // "no session token stored" sentinel).
- if domain == "rpc" && body_matches {
- return true;
- }
-
- false
-}
-
-pub fn is_transient_http_status(status: &str) -> bool {
- TRANSIENT_HTTP_STATUSES.contains(&status)
-}
-
-pub fn is_transient_http_status_code(status: u16) -> bool {
- let status = status.to_string();
- is_transient_http_status(status.as_str())
-}
-
-pub fn contains_transient_transport_phrase(message: &str) -> bool {
- let lower = message.to_ascii_lowercase();
- TRANSIENT_TRANSPORT_PHRASES
- .iter()
- .any(|phrase| lower.contains(phrase))
-}
-
-pub fn is_updater_transient_http_status(status: u16) -> bool {
- UPDATER_TRANSIENT_HTTP_STATUSES.contains(&status)
-}
-
-pub fn is_updater_transient_message(message: &str) -> bool {
- let lower = message.to_ascii_lowercase();
- UPDATER_TRANSIENT_MESSAGE_PHRASES
- .iter()
- .any(|phrase| lower.contains(phrase))
-}
-
-#[cfg(feature = "crash-reporting")]
-fn event_has_transient_transport_phrase(event: &sentry::protocol::Event<'_>) -> bool {
- event
- .message
- .as_deref()
- .is_some_and(contains_transient_transport_phrase)
- || event
- .logentry
- .as_ref()
- .is_some_and(|log| contains_transient_transport_phrase(&log.message))
- || event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(contains_transient_transport_phrase)
- })
-}
-
-#[cfg(feature = "crash-reporting")]
-fn event_has_updater_transient_message(event: &sentry::protocol::Event<'_>) -> bool {
- event
- .message
- .as_deref()
- .is_some_and(is_updater_transient_message)
- || event
- .logentry
- .as_ref()
- .is_some_and(|log| is_updater_transient_message(&log.message))
- || event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(is_updater_transient_message)
- })
-}
-
-#[cfg(feature = "crash-reporting")]
-fn event_has_updater_domain(event: &sentry::protocol::Event<'_>) -> bool {
- matches!(
- event.tags.get("domain").map(String::as_str),
- Some("update") | Some("update.check_releases") | Some("updater")
- )
-}
-
-#[cfg(feature = "crash-reporting")]
-fn is_transient_domain_failure(event: &sentry::protocol::Event<'_>, domain: &str) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some(domain) {
- return false;
- }
-
- match tags.get("failure").map(String::as_str) {
- Some("non_2xx") => tags
- .get("status")
- .is_some_and(|status| is_transient_http_status(status)),
- Some("transport") => event_has_transient_transport_phrase(event),
- _ => false,
- }
-}
-
-/// Transient backend API failures (gateway hiccups, scheduled downtime).
-/// Match by event tags written by report_error at the authed_json call site.
-#[cfg(feature = "crash-reporting")]
-pub fn is_transient_backend_api_failure(event: &sentry::protocol::Event<'_>) -> bool {
- is_transient_domain_failure(event, "backend_api")
-}
-
-/// Defense-in-depth `before_send` filter for skill-install fetch 4xx statuses.
-///
-/// A user/catalog-supplied `SKILL.md` URL returning 4xx means the remote skill
-/// path is missing, private, or otherwise unavailable to that user. The install
-/// RPC still returns the error so the UI can surface it, but Sentry should keep
-/// reporting server-side and transport failures only.
-#[cfg(feature = "crash-reporting")]
-pub fn is_skill_install_user_fetch_failure(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some("skills") {
- return false;
- }
- if tags.get("operation").map(String::as_str) != Some("install_fetch") {
- return false;
- }
- if tags.get("failure").map(String::as_str) != Some("non_2xx") {
- return false;
- }
-
- tags.get("status")
- .and_then(|status| status.parse::<u16>().ok())
- .is_some_and(|status| (400..500).contains(&status))
-}
-
-/// Transient integrations / Composio failures (timeout, connection reset,
-/// gateway hiccups).
-///
-/// Accepts both `domain="integrations"` (the shared
-/// [`crate::integrations::IntegrationClient`] HTTP wrapper that
-/// fronts every backend-proxied integration) and `domain="composio"` (errors
-/// reported from the Composio op layer in
-/// [`crate::integrations::composio::ops`]). Composio routes through the same
-/// `IntegrationClient`, so the failure shape is identical — but op-level
-/// reporters that wrap and re-emit those errors with their own domain tag
-/// would otherwise escape the integrations-scoped filter (OPENHUMAN-TAURI-35
-/// ~139ev, -2H ~26ev: `[composio] list_connections failed: Backend returned
-/// 502 …` events that landed in Sentry under `domain=composio`).
-#[cfg(feature = "crash-reporting")]
-pub fn is_transient_integrations_failure(event: &sentry::protocol::Event<'_>) -> bool {
- is_transient_domain_failure(event, "integrations")
- || is_transient_domain_failure(event, "composio")
-}
-
-/// Skill-install fetch **client errors** (4xx, esp. 404/410).
-///
-/// `install_workflow_from_url_with_home` fetches a user/catalog-supplied
-/// `SKILL.md`; a 4xx means the requested URL is gone or wrong — expected
-/// user-input state surfaced to the UI as "skill not found", not a
-/// Sentry-actionable defect. The primary suppression lives at that emit site
-/// (it no longer calls `report_error` for 4xx); this is the defense-in-depth
-/// net mirroring the `is_transient_*` filters, catching any future skills call
-/// site that reports a 4xx. Matched by the tags `report_error` writes:
-/// `domain=skills`, `failure=non_2xx`, and a 4xx `status`. A 5xx is a genuine
-/// remote failure and stays reportable. Drops TAURI-RUST-CGE (~1,446 events /
-/// 72 users on `[email protected]`).
-#[cfg(feature = "crash-reporting")]
-pub fn is_skills_install_client_error_event(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("domain").map(String::as_str) != Some("skills") {
- return false;
- }
- if tags.get("failure").map(String::as_str) != Some("non_2xx") {
- return false;
- }
- tags.get("status")
- .and_then(|status| status.parse::<u16>().ok())
- .is_some_and(|code| (400..500).contains(&code))
-}
-
-/// Transient updater failures from GitHub release probes/downloads.
-///
-/// Core-side reports carry structured tags (`domain=update`, often
-/// `operation=check_releases`, plus `failure/status`). Tauri's updater plugin
-/// can also emit message-only events such as
-/// `"failed to check for updates: error sending request for url (...latest.json)"`.
-/// Match both shapes, but never drop an arbitrary update-domain event unless
-/// it also has a transient status/transport marker.
-#[cfg(feature = "crash-reporting")]
-pub fn is_updater_transient_event(event: &sentry::protocol::Event<'_>) -> bool {
- if event_has_updater_transient_message(event) {
- return true;
- }
-
- if !event_has_updater_domain(event) {
- return false;
- }
-
- match event.tags.get("failure").map(String::as_str) {
- Some("non_2xx") => event
- .tags
- .get("status")
- .and_then(|status| status.parse::<u16>().ok())
- .is_some_and(is_updater_transient_http_status),
- Some("transport") => event_has_transient_transport_phrase(event),
- _ => false,
- }
-}
-
-/// String tokens that mark a formatted error message as a transient HTTP
-/// failure. Used at upstream emit sites (`rpc.invoke_method`,
-/// `web_channel.run_chat_task`) where the error has already been stringified
-/// and the original `status` / `failure` tag context is gone.
-///
-/// Each token combines a status code with a non-numeric anchor (parenthesis
-/// or canonical reason phrase) so bare numeric coincidences ("process 502
-/// exited") do not match.
-const TRANSIENT_STATUS_MESSAGE_TOKENS: &[&str] = &[
- "(408 ",
- "(429 ",
- "(502 ",
- "(503 ",
- "(504 ",
- "(520 ",
- "408 request timeout",
- "429 too many requests",
- "502 bad gateway",
- "503 service unavailable",
- "504 gateway timeout",
- "520 <unknown status code>",
-];
-
-/// Returns true when a formatted error message describes a transient HTTP
-/// or transport-layer failure that has already been demoted further down the
-/// stack. Use at upstream re-emit sites (`rpc.invoke_method`,
-/// `web_channel.run_chat_task`) where `report_error` is called with the
-/// stringified downstream error and no `failure` / `status` tag context.
-pub fn is_transient_message_failure(msg: &str) -> bool {
- let lower = msg.to_ascii_lowercase();
- TRANSIENT_STATUS_MESSAGE_TOKENS
- .iter()
- .any(|token| lower.contains(token))
- || contains_transient_transport_phrase(&lower)
-}
-
-/// Sentinel prefix stamped on a team-usage probe error that the
-/// failure-backoff in `crate::integrations::client::budget_gate` short-circuited — i.e. an
-/// already-reported repeat within the backoff window. The FIRST failure of a
-/// streak propagates its real error string and reports normally; only the
-/// suppressed repeats carry this prefix so the JSON-RPC boundary can demote
-/// them (no re-report) instead of re-flooding Sentry. See GH #4153.
-///
-/// Single source of truth: the producer (`team::ops::get_usage_with_cache`)
-/// builds its sentinel from this constant, and [`is_suppressed_usage_probe_backoff`]
-/// matches it — coupled by a unit test so the two cannot drift.
-pub const USAGE_PROBE_BACKOFF_PREFIX: &str = "USAGE_PROBE_BACKOFF:";
-
-/// Returns true when a message is the usage-probe failure-backoff sentinel
-/// (see [`USAGE_PROBE_BACKOFF_PREFIX`]). Anchored on the exact prefix so a real
-/// backend error string can never match.
-pub fn is_suppressed_usage_probe_backoff(msg: &str) -> bool {
- msg.starts_with(USAGE_PROBE_BACKOFF_PREFIX)
-}
-
-/// Returns true when a Sentry event is a budget-exhausted 400 that should be
-/// dropped from `before_send`.
-///
-/// Match criteria (all required):
-/// - tag `failure == "non_2xx"`
-/// - tag `status == "400"`
-/// - the event message or any exception value contains one of the tight
-/// budget-exhaustion phrases
-///
-/// Note: `domain` is intentionally not gated here as defense-in-depth over
-/// the emit-site classifier — any non_2xx/400 event that carries the
-/// budget-exhausted phrasing is dropped regardless of which domain produced
-/// it, so a future re-emitter under a different tag still gets filtered.
-#[cfg(feature = "crash-reporting")]
-pub fn is_budget_event(event: &sentry::protocol::Event<'_>) -> bool {
- let tags = &event.tags;
- if tags.get("failure").map(String::as_str) != Some("non_2xx") {
- return false;
- }
- if tags.get("status").map(String::as_str) != Some("400") {
- return false;
- }
- event_contains_budget_exhausted_message(event)
-}
-
-/// Whether a raw error / message string is a provider **insufficient-credits
-/// 402** — the BYO account (e.g. OpenRouter) genuinely lacks the balance to
-/// satisfy the request. Anchored on BOTH a 402-status shape AND a credit
-/// phrase, so a bare `402` (or a non-402 error whose body merely contains the
-/// digits `402`) is not swallowed and keeps reaching Sentry.
-///
-/// Single source of truth shared by the message-level cron halt
-/// (`crate::cron::scheduler`'s `is_insufficient_credits_failure`, which
-/// stops retrying a permanent 402 and skips its `report_error`) and the
-/// event-level `before_send` filter [`is_insufficient_credits_event`] — the
-/// same split as [`is_session_expired_message`] ↔ [`is_session_expired_event`].
-/// TAURI-RUST-514 / -C62.
-pub fn is_insufficient_credits_message(text: &str) -> bool {
- let lower = text.to_ascii_lowercase();
- // Anchor the 402 to a status shape — the emit sites format the message
- // as "<provider> API error (402 Payment Required): <body>". Matching a
- // bare "402" would false-positive on body digits (e.g. a 400 error
- // whose body says "can only afford 402 tokens"), which is NOT this
- // user-state and must keep reaching Sentry.
- let is_402_status = lower.contains("(402") || lower.contains("402 payment required");
- if !is_402_status {
- return false;
- }
- // Check the credit/balance signal against the BODY only. The status prefix
- // "(402 Payment Required)" itself contains the phrase "payment required",
- // which `body_indicates_insufficient_credits` matches — so feeding it the
- // whole string would classify ANY 402 (even one whose body is an unrelated
- // condition) as insufficient-credits and suppress it (codex P2 on #3913).
- // Slice off everything up to and including the formatted "): " status
- // separator first; fall back to the whole text when the separator is
- // absent (non-standard shape) so a credit phrase there still matches.
- let body = lower
- .split_once("): ")
- .map_or(lower.as_str(), |(_, body)| body);
- tinyinference_llm::failure::body_indicates_insufficient_credits(body)
-}
-
-/// Defense-in-depth `before_send` filter for **insufficient-credits 402**
-/// provider events (TAURI-RUST-C62): the user's own BYO provider account
-/// (e.g. OpenRouter) is out of balance — a billing state OpenHuman has no
-/// lever over once the request already caps `max_tokens`.
-///
-/// The primary emit-site demotion lives in the `Provider::chat()` native_chat
-/// cascade (`is_provider_insufficient_credits_402`), but the compatible
-/// provider reports the same failure from several other paths
-/// (`chat_with_system`, `chat_with_history`, the streaming gates, and the
-/// shared `api_error` helper) that don't run that cascade. This filter is the
-/// single outermost net that catches all of them, keyed on the formatted
-/// message rather than tags so it matches regardless of which path emitted it.
-///
-/// Match criteria (all required):
-/// - the event message or any exception value names a 402 / payment-required
-/// failure (`"402"` or `"payment required"`), AND
-/// - that same text carries an insufficient-credits phrase
-/// (`provider::body_indicates_insufficient_credits`).
-#[cfg(feature = "crash-reporting")]
-pub fn is_insufficient_credits_event(event: &sentry::protocol::Event<'_>) -> bool {
- if event
- .message
- .as_deref()
- .is_some_and(is_insufficient_credits_message)
- {
- return true;
- }
- event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(is_insufficient_credits_message)
- })
-}
-
-/// Message-level matcher for a provider **monthly-quota / usage-limit
-/// exhausted** failure. Status-agnostic by design — unlike
-/// [`is_insufficient_credits_message`] it does NOT anchor on a 402 status,
-/// because the Kiro IDE proxy wraps its 402 inside a 500 envelope
-/// (TAURI-RUST-C9A). Delegates to the single-source quota-phrase set in
-/// [`tinyinference_llm::failure::body_indicates_quota_exhausted`], so
-/// the emit-site guard and this `before_send` net can't drift. Shared with the
-/// event-level filter [`is_quota_exhausted_event`].
-#[cfg(any(feature = "crash-reporting", test))]
-fn is_quota_exhausted_message(body: &str) -> bool {
- tinyinference_llm::failure::body_indicates_quota_exhausted(body)
-}
-
-/// Defense-in-depth `before_send` filter for provider **monthly-quota
-/// exhausted** events (TAURI-RUST-C9A): the user's third-party plan has spent
-/// its allotment for the period — a billing/plan state OpenHuman has no lever
-/// over.
-///
-/// The primary emit-site demotion lives in the `Provider::chat()` native_chat
-/// cascade and the shared `api_error` helper (`is_provider_quota_exhausted`),
-/// but the compatible provider reports the same failure from several other
-/// paths that don't run those guards. This filter is the single outermost net
-/// that catches all of them, keyed on the formatted message rather than tags so
-/// it matches regardless of which path emitted it (and regardless of whether
-/// the upstream wrapped the 402 in a 500 envelope).
-#[cfg(feature = "crash-reporting")]
-pub fn is_quota_exhausted_event(event: &sentry::protocol::Event<'_>) -> bool {
- if event
- .message
- .as_deref()
- .is_some_and(is_quota_exhausted_message)
- {
- return true;
- }
- event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(is_quota_exhausted_message)
- })
-}
-
-/// Whether a raw error / message string is an Ollama **Cloud** hosted-inference
-/// `500` (`Internal Server Error (ref: <uuid>)`). Matches either the raw emit
-/// shape (`ollama API error (500 …): {"error":"Internal Server Error (ref: …)"}`)
-/// or the actionable re-raise the emit sites swap in
-/// (`is_ollama_cloud_internal_500_message`). The raw arm requires BOTH the
-/// `ollama` provider name and the `internal server error (ref:` envelope, so a
-/// generic 500 from another provider, or a local Ollama daemon crash (which
-/// carries no `ref:` UUID), still reaches Sentry.
-pub fn is_ollama_cloud_internal_500_message_any(text: &str) -> bool {
- if tinyinference_llm::failure::is_ollama_cloud_internal_500_message(text) {
- return true;
- }
- let lower = text.to_ascii_lowercase();
- lower.contains("ollama") && lower.contains("internal server error (ref:")
-}
-
-/// Defense-in-depth `before_send` filter for **Ollama Cloud hosted-inference
-/// 500s** (TAURI-RUST-5MV): ollama.com's `*:cloud` models intermittently
-/// return an opaque `Internal Server Error (ref: <uuid>)` with no client lever
-/// (non-deterministic, byte-identical request succeeds when healthy), retried +
-/// fallen-back by the reliable-provider layer.
-///
-/// The primary demotion lives at the `native_chat` / `streaming_chat` /
-/// `api_error` emit sites, and the agent re-report is demoted via
-/// `expected_error_kind` → `TransientUpstreamHttp`. This is the single outermost
-/// net for any other compatible-provider path (`chat_with_system`,
-/// `chat_with_history`, the non-native cascades) that reports the same body,
-/// keyed on the message rather than tags so it matches regardless of emitter.
-#[cfg(feature = "crash-reporting")]
-pub fn is_ollama_cloud_internal_500_event(event: &sentry::protocol::Event<'_>) -> bool {
- if event
- .message
- .as_deref()
- .is_some_and(is_ollama_cloud_internal_500_message_any)
- {
- return true;
- }
- event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(is_ollama_cloud_internal_500_message_any)
- })
-}
-
-/// Defense-in-depth `before_send` filter for Windows `ERROR_FILE_SYSTEM_LIMITATION`
-/// (os error 665) — a host-filesystem condition (fragmentation, USN journal
-/// overflow, filter-driver resource cap) that is persistent, locale-independent
-/// in the `(os error N)` suffix, and unrecoverable by the app.
-///
-/// The primary suppression lives at the emit site via `expected_error_kind` →
-/// `ExpectedErrorKind::WindowsFileSystemLimitation` (called by
-/// `report_error_or_expected`). This filter catches any future call site that
-/// bypasses `report_error_or_expected` and emits the error via `report_error`
-/// or `tracing::error!` directly — keeping TAURI-RUST-QT0 (6,050 events /
-/// 1 user) permanently off Sentry.
-///
-/// Also catches `OS error 665` from the Tauri shell side
-/// (`crates/openhuman-app/`) which is compiled into a separate crate and therefore
-/// cannot route through the core's `expected_error_kind` classifier.
-#[cfg(feature = "crash-reporting")]
-pub fn is_windows_file_system_limitation_event(event: &sentry::protocol::Event<'_>) -> bool {
- let check = |text: &str| -> bool {
- let lower = text.to_ascii_lowercase();
- lower.contains("(os error 665)")
- };
- if event.message.as_deref().is_some_and(check) {
- return true;
- }
- if event
- .logentry
- .as_ref()
- .map(|log| log.message.as_str())
- .is_some_and(check)
- {
- return true;
- }
- event
- .exception
- .values
- .iter()
- .any(|exception| exception.value.as_deref().is_some_and(check))
-}
-
-#[cfg(feature = "crash-reporting")]
-fn event_contains_budget_exhausted_message(event: &sentry::protocol::Event<'_>) -> bool {
- if event
- .message
- .as_deref()
- .is_some_and(tinyinference_providers::is_budget_exhausted_message)
- {
- return true;
- }
-
- event.exception.values.iter().any(|exception| {
- exception
- .value
- .as_deref()
- .is_some_and(tinyinference_providers::is_budget_exhausted_message)
- })
-}
-
-/// Defense-in-depth `before_send` filter for **user-config provider error
-/// patterns** — 4xx client errors, model-not-found, subscription/payment
-/// issues, and other misconfigurations that are not application bugs.
-///
-/// Matches on `event.message` or exception values against known user-config
-/// patterns. The primary suppression lives at the `report_error` / emit sites;
-/// this catches any future call site that bypasses those classifiers.
-///
-/// Returns true (should be dropped) when the event message matches any known
-/// user-config provider pattern. Target: ~22 Sentry issues / ~26k events.
-#[cfg(feature = "crash-reporting")]
-pub fn is_user_config_provider_event(event: &sentry::protocol::Event<'_>) -> bool {
- // Collect all text sources to check against
- let texts: Vec<&str> = {
- let mut v = Vec::with_capacity(2 + event.exception.values.len());
- if let Some(msg) = event.message.as_deref() {
- v.push(msg);
- }
- if let Some(log) = event.logentry.as_ref().map(|l| l.message.as_str()) {
- v.push(log);
- }
- for exc in &event.exception.values {
- if let Some(val) = exc.value.as_deref() {
- v.push(val);
- }
- }
- v
- };
- if texts.is_empty() {
- return false;
- }
-
- // Provider 4xx patterns — user config errors like:
- // "HTTP 400", "HTTP 401", "HTTP 403", "HTTP 404"
- // matching against llm_provider domain events
- let tags = &event.tags;
- let domain = tags.get("domain").map(String::as_str);
- if domain == Some("llm_provider") {
- if let Some(status) = tags.get("status") {
- // 4xx user config errors. 401/403/404 are unambiguously
- // auth/permission/config issues and safe to drop blanket.
- // 400 (Bad Request) may also be a client-side serialization
- // bug — only drop when the message content confirms it's
- // a user config error (handled by message patterns below
- // and the earlier `is_budget_event` / `is_insufficient_credits_event`
- // filters in the before_send chain).
- if matches!(status.as_str(), "401" | "403" | "404") {
- return true;
- }
- }
- // Message-level patterns
- let lower = texts.join(" ").to_ascii_lowercase();
- if lower.contains("401 payment required")
- || lower.contains("subscription")
- || lower.contains("max monthly spend")
- || lower.contains("context length exceeded")
- || lower.contains("context size exceeded")
- {
- return true;
- }
- }
-
- // Embedding API 4xx / unauthorized — user's embedding provider config
- if domain == Some("llm_provider") || domain == Some("local_ai") {
- let lower = texts.join(" ").to_ascii_lowercase();
- if (lower.contains("embed") || lower.contains("embedding"))
- && (lower.contains("401")
- || lower.contains("404")
- || lower.contains("unauthorized")
- || lower.contains("invalid model"))
- {
- return true;
- }
- }
-
- false
-}
-
-#[cfg(feature = "crash-reporting")]
-include!("observability_connectivity.rs");
-
+include!("observability_expected_error_classification.rs");include!("observability_plan_and_session_errors.rs");include!("observability_provider_local_errors.rs");include!("observability_network_provider_errors.rs");include!("observability_reporting.rs");include!("observability_event_classification.rs");include!("observability_backend_event_classification.rs");
#[cfg(test)]
#[path = "observability_tests.rs"]
mod tests;
diff --git a/crates/openhuman-core/src/core/observability_backend_event_classification.rs b/crates/openhuman-core/src/core/observability_backend_event_classification.rs
new file mode 100644
index 0000000000..41de461bdf
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_backend_event_classification.rs
@@ -0,0 +1,478 @@
+pub fn is_transient_backend_api_failure(event: &sentry::protocol::Event<'_>) -> bool {
+ is_transient_domain_failure(event, "backend_api")
+}
+
+/// Defense-in-depth `before_send` filter for skill-install fetch 4xx statuses.
+///
+/// A user/catalog-supplied `SKILL.md` URL returning 4xx means the remote skill
+/// path is missing, private, or otherwise unavailable to that user. The install
+/// RPC still returns the error so the UI can surface it, but Sentry should keep
+/// reporting server-side and transport failures only.
+#[cfg(feature = "crash-reporting")]
+pub fn is_skill_install_user_fetch_failure(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some("skills") {
+ return false;
+ }
+ if tags.get("operation").map(String::as_str) != Some("install_fetch") {
+ return false;
+ }
+ if tags.get("failure").map(String::as_str) != Some("non_2xx") {
+ return false;
+ }
+
+ tags.get("status")
+ .and_then(|status| status.parse::<u16>().ok())
+ .is_some_and(|status| (400..500).contains(&status))
+}
+
+/// Transient integrations / Composio failures (timeout, connection reset,
+/// gateway hiccups).
+///
+/// Accepts both `domain="integrations"` (the shared
+/// [`crate::integrations::IntegrationClient`] HTTP wrapper that
+/// fronts every backend-proxied integration) and `domain="composio"` (errors
+/// reported from the Composio op layer in
+/// [`crate::integrations::composio::ops`]). Composio routes through the same
+/// `IntegrationClient`, so the failure shape is identical — but op-level
+/// reporters that wrap and re-emit those errors with their own domain tag
+/// would otherwise escape the integrations-scoped filter (OPENHUMAN-TAURI-35
+/// ~139ev, -2H ~26ev: `[composio] list_connections failed: Backend returned
+/// 502 …` events that landed in Sentry under `domain=composio`).
+#[cfg(feature = "crash-reporting")]
+pub fn is_transient_integrations_failure(event: &sentry::protocol::Event<'_>) -> bool {
+ is_transient_domain_failure(event, "integrations")
+ || is_transient_domain_failure(event, "composio")
+}
+
+/// Skill-install fetch **client errors** (4xx, esp. 404/410).
+///
+/// `install_workflow_from_url_with_home` fetches a user/catalog-supplied
+/// `SKILL.md`; a 4xx means the requested URL is gone or wrong — expected
+/// user-input state surfaced to the UI as "skill not found", not a
+/// Sentry-actionable defect. The primary suppression lives at that emit site
+/// (it no longer calls `report_error` for 4xx); this is the defense-in-depth
+/// net mirroring the `is_transient_*` filters, catching any future skills call
+/// site that reports a 4xx. Matched by the tags `report_error` writes:
+/// `domain=skills`, `failure=non_2xx`, and a 4xx `status`. A 5xx is a genuine
+/// remote failure and stays reportable. Drops TAURI-RUST-CGE (~1,446 events /
+/// 72 users on `[email protected]`).
+#[cfg(feature = "crash-reporting")]
+pub fn is_skills_install_client_error_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some("skills") {
+ return false;
+ }
+ if tags.get("failure").map(String::as_str) != Some("non_2xx") {
+ return false;
+ }
+ tags.get("status")
+ .and_then(|status| status.parse::<u16>().ok())
+ .is_some_and(|code| (400..500).contains(&code))
+}
+
+/// Transient updater failures from GitHub release probes/downloads.
+///
+/// Core-side reports carry structured tags (`domain=update`, often
+/// `operation=check_releases`, plus `failure/status`). Tauri's updater plugin
+/// can also emit message-only events such as
+/// `"failed to check for updates: error sending request for url (...latest.json)"`.
+/// Match both shapes, but never drop an arbitrary update-domain event unless
+/// it also has a transient status/transport marker.
+#[cfg(feature = "crash-reporting")]
+pub fn is_updater_transient_event(event: &sentry::protocol::Event<'_>) -> bool {
+ if event_has_updater_transient_message(event) {
+ return true;
+ }
+
+ if !event_has_updater_domain(event) {
+ return false;
+ }
+
+ match event.tags.get("failure").map(String::as_str) {
+ Some("non_2xx") => event
+ .tags
+ .get("status")
+ .and_then(|status| status.parse::<u16>().ok())
+ .is_some_and(is_updater_transient_http_status),
+ Some("transport") => event_has_transient_transport_phrase(event),
+ _ => false,
+ }
+}
+
+/// String tokens that mark a formatted error message as a transient HTTP
+/// failure. Used at upstream emit sites (`rpc.invoke_method`,
+/// `web_channel.run_chat_task`) where the error has already been stringified
+/// and the original `status` / `failure` tag context is gone.
+///
+/// Each token combines a status code with a non-numeric anchor (parenthesis
+/// or canonical reason phrase) so bare numeric coincidences ("process 502
+/// exited") do not match.
+const TRANSIENT_STATUS_MESSAGE_TOKENS: &[&str] = &[
+ "(408 ",
+ "(429 ",
+ "(502 ",
+ "(503 ",
+ "(504 ",
+ "(520 ",
+ "408 request timeout",
+ "429 too many requests",
+ "502 bad gateway",
+ "503 service unavailable",
+ "504 gateway timeout",
+ "520 <unknown status code>",
+];
+
+/// Returns true when a formatted error message describes a transient HTTP
+/// or transport-layer failure that has already been demoted further down the
+/// stack. Use at upstream re-emit sites (`rpc.invoke_method`,
+/// `web_channel.run_chat_task`) where `report_error` is called with the
+/// stringified downstream error and no `failure` / `status` tag context.
+pub fn is_transient_message_failure(msg: &str) -> bool {
+ let lower = msg.to_ascii_lowercase();
+ TRANSIENT_STATUS_MESSAGE_TOKENS
+ .iter()
+ .any(|token| lower.contains(token))
+ || contains_transient_transport_phrase(&lower)
+}
+
+/// Sentinel prefix stamped on a team-usage probe error that the
+/// failure-backoff in `crate::integrations::client::budget_gate` short-circuited — i.e. an
+/// already-reported repeat within the backoff window. The FIRST failure of a
+/// streak propagates its real error string and reports normally; only the
+/// suppressed repeats carry this prefix so the JSON-RPC boundary can demote
+/// them (no re-report) instead of re-flooding Sentry. See GH #4153.
+///
+/// Single source of truth: the producer (`team::ops::get_usage_with_cache`)
+/// builds its sentinel from this constant, and [`is_suppressed_usage_probe_backoff`]
+/// matches it — coupled by a unit test so the two cannot drift.
+pub const USAGE_PROBE_BACKOFF_PREFIX: &str = "USAGE_PROBE_BACKOFF:";
+
+/// Returns true when a message is the usage-probe failure-backoff sentinel
+/// (see [`USAGE_PROBE_BACKOFF_PREFIX`]). Anchored on the exact prefix so a real
+/// backend error string can never match.
+pub fn is_suppressed_usage_probe_backoff(msg: &str) -> bool {
+ msg.starts_with(USAGE_PROBE_BACKOFF_PREFIX)
+}
+
+/// Returns true when a Sentry event is a budget-exhausted 400 that should be
+/// dropped from `before_send`.
+///
+/// Match criteria (all required):
+/// - tag `failure == "non_2xx"`
+/// - tag `status == "400"`
+/// - the event message or any exception value contains one of the tight
+/// budget-exhaustion phrases
+///
+/// Note: `domain` is intentionally not gated here as defense-in-depth over
+/// the emit-site classifier — any non_2xx/400 event that carries the
+/// budget-exhausted phrasing is dropped regardless of which domain produced
+/// it, so a future re-emitter under a different tag still gets filtered.
+#[cfg(feature = "crash-reporting")]
+pub fn is_budget_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("failure").map(String::as_str) != Some("non_2xx") {
+ return false;
+ }
+ if tags.get("status").map(String::as_str) != Some("400") {
+ return false;
+ }
+ event_contains_budget_exhausted_message(event)
+}
+
+/// Whether a raw error / message string is a provider **insufficient-credits
+/// 402** — the BYO account (e.g. OpenRouter) genuinely lacks the balance to
+/// satisfy the request. Anchored on BOTH a 402-status shape AND a credit
+/// phrase, so a bare `402` (or a non-402 error whose body merely contains the
+/// digits `402`) is not swallowed and keeps reaching Sentry.
+///
+/// Single source of truth shared by the message-level cron halt
+/// (`crate::cron::scheduler`'s `is_insufficient_credits_failure`, which
+/// stops retrying a permanent 402 and skips its `report_error`) and the
+/// event-level `before_send` filter [`is_insufficient_credits_event`] — the
+/// same split as [`is_session_expired_message`] ↔ [`is_session_expired_event`].
+/// TAURI-RUST-514 / -C62.
+pub fn is_insufficient_credits_message(text: &str) -> bool {
+ let lower = text.to_ascii_lowercase();
+ // Anchor the 402 to a status shape — the emit sites format the message
+ // as "<provider> API error (402 Payment Required): <body>". Matching a
+ // bare "402" would false-positive on body digits (e.g. a 400 error
+ // whose body says "can only afford 402 tokens"), which is NOT this
+ // user-state and must keep reaching Sentry.
+ let is_402_status = lower.contains("(402") || lower.contains("402 payment required");
+ if !is_402_status {
+ return false;
+ }
+ // Check the credit/balance signal against the BODY only. The status prefix
+ // "(402 Payment Required)" itself contains the phrase "payment required",
+ // which `body_indicates_insufficient_credits` matches — so feeding it the
+ // whole string would classify ANY 402 (even one whose body is an unrelated
+ // condition) as insufficient-credits and suppress it (codex P2 on #3913).
+ // Slice off everything up to and including the formatted "): " status
+ // separator first; fall back to the whole text when the separator is
+ // absent (non-standard shape) so a credit phrase there still matches.
+ let body = lower
+ .split_once("): ")
+ .map_or(lower.as_str(), |(_, body)| body);
+ tinyinference_llm::failure::body_indicates_insufficient_credits(body)
+}
+
+/// Defense-in-depth `before_send` filter for **insufficient-credits 402**
+/// provider events (TAURI-RUST-C62): the user's own BYO provider account
+/// (e.g. OpenRouter) is out of balance — a billing state OpenHuman has no
+/// lever over once the request already caps `max_tokens`.
+///
+/// The primary emit-site demotion lives in the `Provider::chat()` native_chat
+/// cascade (`is_provider_insufficient_credits_402`), but the compatible
+/// provider reports the same failure from several other paths
+/// (`chat_with_system`, `chat_with_history`, the streaming gates, and the
+/// shared `api_error` helper) that don't run that cascade. This filter is the
+/// single outermost net that catches all of them, keyed on the formatted
+/// message rather than tags so it matches regardless of which path emitted it.
+///
+/// Match criteria (all required):
+/// - the event message or any exception value names a 402 / payment-required
+/// failure (`"402"` or `"payment required"`), AND
+/// - that same text carries an insufficient-credits phrase
+/// (`provider::body_indicates_insufficient_credits`).
+#[cfg(feature = "crash-reporting")]
+pub fn is_insufficient_credits_event(event: &sentry::protocol::Event<'_>) -> bool {
+ if event
+ .message
+ .as_deref()
+ .is_some_and(is_insufficient_credits_message)
+ {
+ return true;
+ }
+ event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(is_insufficient_credits_message)
+ })
+}
+
+/// Message-level matcher for a provider **monthly-quota / usage-limit
+/// exhausted** failure. Status-agnostic by design — unlike
+/// [`is_insufficient_credits_message`] it does NOT anchor on a 402 status,
+/// because the Kiro IDE proxy wraps its 402 inside a 500 envelope
+/// (TAURI-RUST-C9A). Delegates to the single-source quota-phrase set in
+/// [`tinyinference_llm::failure::body_indicates_quota_exhausted`], so
+/// the emit-site guard and this `before_send` net can't drift. Shared with the
+/// event-level filter [`is_quota_exhausted_event`].
+#[cfg(any(feature = "crash-reporting", test))]
+fn is_quota_exhausted_message(body: &str) -> bool {
+ tinyinference_llm::failure::body_indicates_quota_exhausted(body)
+}
+
+/// Defense-in-depth `before_send` filter for provider **monthly-quota
+/// exhausted** events (TAURI-RUST-C9A): the user's third-party plan has spent
+/// its allotment for the period — a billing/plan state OpenHuman has no lever
+/// over.
+///
+/// The primary emit-site demotion lives in the `Provider::chat()` native_chat
+/// cascade and the shared `api_error` helper (`is_provider_quota_exhausted`),
+/// but the compatible provider reports the same failure from several other
+/// paths that don't run those guards. This filter is the single outermost net
+/// that catches all of them, keyed on the formatted message rather than tags so
+/// it matches regardless of which path emitted it (and regardless of whether
+/// the upstream wrapped the 402 in a 500 envelope).
+#[cfg(feature = "crash-reporting")]
+pub fn is_quota_exhausted_event(event: &sentry::protocol::Event<'_>) -> bool {
+ if event
+ .message
+ .as_deref()
+ .is_some_and(is_quota_exhausted_message)
+ {
+ return true;
+ }
+ event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(is_quota_exhausted_message)
+ })
+}
+
+/// Whether a raw error / message string is an Ollama **Cloud** hosted-inference
+/// `500` (`Internal Server Error (ref: <uuid>)`). Matches either the raw emit
+/// shape (`ollama API error (500 …): {"error":"Internal Server Error (ref: …)"}`)
+/// or the actionable re-raise the emit sites swap in
+/// (`is_ollama_cloud_internal_500_message`). The raw arm requires BOTH the
+/// `ollama` provider name and the `internal server error (ref:` envelope, so a
+/// generic 500 from another provider, or a local Ollama daemon crash (which
+/// carries no `ref:` UUID), still reaches Sentry.
+pub fn is_ollama_cloud_internal_500_message_any(text: &str) -> bool {
+ if tinyinference_llm::failure::is_ollama_cloud_internal_500_message(text) {
+ return true;
+ }
+ let lower = text.to_ascii_lowercase();
+ lower.contains("ollama") && lower.contains("internal server error (ref:")
+}
+
+/// Defense-in-depth `before_send` filter for **Ollama Cloud hosted-inference
+/// 500s** (TAURI-RUST-5MV): ollama.com's `*:cloud` models intermittently
+/// return an opaque `Internal Server Error (ref: <uuid>)` with no client lever
+/// (non-deterministic, byte-identical request succeeds when healthy), retried +
+/// fallen-back by the reliable-provider layer.
+///
+/// The primary demotion lives at the `native_chat` / `streaming_chat` /
+/// `api_error` emit sites, and the agent re-report is demoted via
+/// `expected_error_kind` → `TransientUpstreamHttp`. This is the single outermost
+/// net for any other compatible-provider path (`chat_with_system`,
+/// `chat_with_history`, the non-native cascades) that reports the same body,
+/// keyed on the message rather than tags so it matches regardless of emitter.
+#[cfg(feature = "crash-reporting")]
+pub fn is_ollama_cloud_internal_500_event(event: &sentry::protocol::Event<'_>) -> bool {
+ if event
+ .message
+ .as_deref()
+ .is_some_and(is_ollama_cloud_internal_500_message_any)
+ {
+ return true;
+ }
+ event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(is_ollama_cloud_internal_500_message_any)
+ })
+}
+
+/// Defense-in-depth `before_send` filter for Windows `ERROR_FILE_SYSTEM_LIMITATION`
+/// (os error 665) — a host-filesystem condition (fragmentation, USN journal
+/// overflow, filter-driver resource cap) that is persistent, locale-independent
+/// in the `(os error N)` suffix, and unrecoverable by the app.
+///
+/// The primary suppression lives at the emit site via `expected_error_kind` →
+/// `ExpectedErrorKind::WindowsFileSystemLimitation` (called by
+/// `report_error_or_expected`). This filter catches any future call site that
+/// bypasses `report_error_or_expected` and emits the error via `report_error`
+/// or `tracing::error!` directly — keeping TAURI-RUST-QT0 (6,050 events /
+/// 1 user) permanently off Sentry.
+///
+/// Also catches `OS error 665` from the Tauri shell side
+/// (`crates/openhuman-app/`) which is compiled into a separate crate and therefore
+/// cannot route through the core's `expected_error_kind` classifier.
+#[cfg(feature = "crash-reporting")]
+pub fn is_windows_file_system_limitation_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let check = |text: &str| -> bool {
+ let lower = text.to_ascii_lowercase();
+ lower.contains("(os error 665)")
+ };
+ if event.message.as_deref().is_some_and(check) {
+ return true;
+ }
+ if event
+ .logentry
+ .as_ref()
+ .map(|log| log.message.as_str())
+ .is_some_and(check)
+ {
+ return true;
+ }
+ event
+ .exception
+ .values
+ .iter()
+ .any(|exception| exception.value.as_deref().is_some_and(check))
+}
+
+#[cfg(feature = "crash-reporting")]
+fn event_contains_budget_exhausted_message(event: &sentry::protocol::Event<'_>) -> bool {
+ if event
+ .message
+ .as_deref()
+ .is_some_and(tinyinference_providers::is_budget_exhausted_message)
+ {
+ return true;
+ }
+
+ event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(tinyinference_providers::is_budget_exhausted_message)
+ })
+}
+
+/// Defense-in-depth `before_send` filter for **user-config provider error
+/// patterns** — 4xx client errors, model-not-found, subscription/payment
+/// issues, and other misconfigurations that are not application bugs.
+///
+/// Matches on `event.message` or exception values against known user-config
+/// patterns. The primary suppression lives at the `report_error` / emit sites;
+/// this catches any future call site that bypasses those classifiers.
+///
+/// Returns true (should be dropped) when the event message matches any known
+/// user-config provider pattern. Target: ~22 Sentry issues / ~26k events.
+#[cfg(feature = "crash-reporting")]
+pub fn is_user_config_provider_event(event: &sentry::protocol::Event<'_>) -> bool {
+ // Collect all text sources to check against
+ let texts: Vec<&str> = {
+ let mut v = Vec::with_capacity(2 + event.exception.values.len());
+ if let Some(msg) = event.message.as_deref() {
+ v.push(msg);
+ }
+ if let Some(log) = event.logentry.as_ref().map(|l| l.message.as_str()) {
+ v.push(log);
+ }
+ for exc in &event.exception.values {
+ if let Some(val) = exc.value.as_deref() {
+ v.push(val);
+ }
+ }
+ v
+ };
+ if texts.is_empty() {
+ return false;
+ }
+
+ // Provider 4xx patterns — user config errors like:
+ // "HTTP 400", "HTTP 401", "HTTP 403", "HTTP 404"
+ // matching against llm_provider domain events
+ let tags = &event.tags;
+ let domain = tags.get("domain").map(String::as_str);
+ if domain == Some("llm_provider") {
+ if let Some(status) = tags.get("status") {
+ // 4xx user config errors. 401/403/404 are unambiguously
+ // auth/permission/config issues and safe to drop blanket.
+ // 400 (Bad Request) may also be a client-side serialization
+ // bug — only drop when the message content confirms it's
+ // a user config error (handled by message patterns below
+ // and the earlier `is_budget_event` / `is_insufficient_credits_event`
+ // filters in the before_send chain).
+ if matches!(status.as_str(), "401" | "403" | "404") {
+ return true;
+ }
+ }
+ // Message-level patterns
+ let lower = texts.join(" ").to_ascii_lowercase();
+ if lower.contains("401 payment required")
+ || lower.contains("subscription")
+ || lower.contains("max monthly spend")
+ || lower.contains("context length exceeded")
+ || lower.contains("context size exceeded")
+ {
+ return true;
+ }
+ }
+
+ // Embedding API 4xx / unauthorized — user's embedding provider config
+ if domain == Some("llm_provider") || domain == Some("local_ai") {
+ let lower = texts.join(" ").to_ascii_lowercase();
+ if (lower.contains("embed") || lower.contains("embedding"))
+ && (lower.contains("401")
+ || lower.contains("404")
+ || lower.contains("unauthorized")
+ || lower.contains("invalid model"))
+ {
+ return true;
+ }
+ }
+
+ false
+}
+
+#[cfg(feature = "crash-reporting")]
+include!("observability_connectivity.rs");
diff --git a/crates/openhuman-core/src/core/observability_event_classification.rs b/crates/openhuman-core/src/core/observability_event_classification.rs
new file mode 100644
index 0000000000..61afafde59
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_event_classification.rs
@@ -0,0 +1,288 @@
+pub fn is_transient_provider_http_failure(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some("llm_provider") {
+ return false;
+ }
+ if tags.get("failure").map(String::as_str) != Some("non_2xx") {
+ return false;
+ }
+ let Some(status_u16) = tags.get("status").and_then(|s| s.parse::<u16>().ok()) else {
+ return false;
+ };
+ tinyinference_llm::failure::TRANSIENT_PROVIDER_HTTP_STATUSES.contains(&status_u16)
+}
+
+/// Defense-in-depth `before_send` filter for managed-backend `errorCode`
+/// events (#870): drops any Sentry event whose message/exception text carries a
+/// backend `errorCode` that the backend owns (F2/F4) — *except* a backend-flagged
+/// malformed `BAD_REQUEST`, which the client caused and so still pages (F8).
+///
+/// Primary suppression lives at the emit sites (`api_error` /
+/// `compatible_*` streaming gates) and at the higher-layer re-report
+/// classifier (`expected_error_kind` → [`ExpectedErrorKind::BackendErrorCodeOwned`]).
+/// This catches any future call site that re-emits the same flattened error
+/// without routing through those funnels. Delegates the decision to the
+/// single-source [`crate::inference::provider::managed_error_skips_sentry`]
+/// (managed-envelope gated, so a BYO payload carrying an `errorCode`-shaped
+/// field is not wrongly dropped) so the layers can't drift.
+#[cfg(feature = "crash-reporting")]
+pub fn is_backend_error_code_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let direct = event.message.as_deref();
+ let from_logentry = event.logentry.as_ref().map(|log| log.message.as_str());
+ let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
+ [direct, from_logentry, from_exception]
+ .into_iter()
+ .flatten()
+ .any(|message| {
+ crate::inference::registry::classify_backend_error(None, message).is_some_and(
+ |classification| classification.backend_owned && !classification.reportable,
+ )
+ })
+}
+
+/// Defense-in-depth `before_send` filter for transient streaming **transport**
+/// failures (F7): drops `domain=llm_provider, failure=transport` events whose
+/// body is a transient transport phrase (timeout / reset / TLS-handshake EOF /
+/// "error sending request"). Flaky-network blips on the streaming send are
+/// recovered by retry/fallback and carry no actionable Sentry signal; a
+/// non-transient transport failure (DNS misconfig, unexpected protocol error)
+/// is not matched and still pages.
+///
+/// Primary gate lives at the two streaming emit sites in
+/// `compatible_provider_impl.rs` (`stream_chat` / `stream_chat_history`); this
+/// catches any future site that reports the same shape without gating.
+///
+/// Scoped to the **streaming** operations on purpose (CodeRabbit): only the
+/// `stream_chat` / `stream_chat_history` transport emits are meant to be
+/// suppressed. A non-streaming `domain=llm_provider, failure=transport` event
+/// carries a different `operation` tag and must keep paging, so the
+/// observability blind spot stays as narrow as F7 intends.
+#[cfg(feature = "crash-reporting")]
+pub fn is_transient_provider_transport_failure(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some("llm_provider") {
+ return false;
+ }
+ if tags.get("failure").map(String::as_str) != Some("transport") {
+ return false;
+ }
+ if !matches!(
+ tags.get("operation").map(String::as_str),
+ Some("stream_chat") | Some("stream_chat_history")
+ ) {
+ return false;
+ }
+ event_has_transient_transport_phrase(event)
+}
+
+/// Defense-in-depth filter for aggregate provider exhaustion events where the
+/// aggregate only restates transient attempt failures.
+///
+/// Keep ordinary `failure=all_exhausted` events: they are the useful "every
+/// fallback failed" signal. Drop only the narrow shape observed in #3542,
+/// where the aggregate body starts with the reliable-provider exhaustion
+/// prefix and contains transient HTTP/transport wording already classified by
+/// [`is_transient_message_failure`].
+#[cfg(feature = "crash-reporting")]
+pub fn is_all_transient_provider_exhaustion_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some("llm_provider") {
+ return false;
+ }
+ if tags.get("failure").map(String::as_str) != Some("all_exhausted") {
+ return false;
+ }
+
+ let direct = event.message.as_deref();
+ let from_logentry = event.logentry.as_ref().map(|log| log.message.as_str());
+ let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
+ [direct, from_logentry, from_exception]
+ .into_iter()
+ .flatten()
+ .any(all_provider_attempts_are_transient)
+}
+
+#[cfg(feature = "crash-reporting")]
+fn all_provider_attempts_are_transient(message: &str) -> bool {
+ let Some(attempts) = message.strip_prefix("All providers/models failed. Attempts:") else {
+ return false;
+ };
+ let mut saw_attempt = false;
+ for attempt in attempts.split(';').map(str::trim).filter(|s| !s.is_empty()) {
+ saw_attempt = true;
+ if !is_transient_message_failure(attempt) {
+ return false;
+ }
+ }
+ saw_attempt
+}
+
+/// Returns true when a Sentry event's message/exception text contains the
+/// canonical max-tool-iterations cap phrase (see
+/// `tinyagents_harness::agent_error::MAX_ITERATIONS_ERROR_PREFIX`).
+///
+/// Defense-in-depth filter for the Sentry `before_send` hook: the primary
+/// suppression lives at the call sites in `agent::session_host::
+/// runtime::run_single`, `channels::runtime::dispatch`, and
+/// `web_chat::run_chat_task`, all of which now skip
+/// `report_error` when this variant is detected. This filter catches any
+/// future call site that re-emits the message without going through those
+/// funnels — e.g. a new wrapper that calls `tracing::error!` directly with
+/// the typed error rendering — and keeps OPENHUMAN-TAURI-99 / -98
+/// permanently off Sentry without requiring touch-ups at each new site.
+///
+/// Match strategy: scans `event.message` first (the path used by
+/// `report_error_message` → `sentry::capture_message`) and falls back to
+/// the last exception's `value` (the shape `sentry-tracing` produces when
+/// stacktraces are attached). Both fields are checked for the canonical
+/// prefix so the filter stays robust to future Sentry plumbing changes.
+#[cfg(feature = "crash-reporting")]
+pub fn is_max_iterations_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let direct = event.message.as_deref();
+ let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
+ [direct, from_exception]
+ .into_iter()
+ .flatten()
+ .any(tinyagents_harness::agent_error::is_max_iterations_error)
+}
+
+/// Tag + body classifier for the `before_send` chain — drops Sentry events
+/// emitted at the OpenHuman backend / rpc layers for "401 Session
+/// expired" or the pre-flight "no session token stored" guards.
+///
+/// Pairs with [`is_session_expired_message`] (which classifies the
+/// message body at the emit site via `report_error_or_expected`). This
+/// fn runs in `before_send` so it catches any future call site that
+/// re-emits the same shape without routing through the classifier —
+/// keeps OPENHUMAN-TAURI-25 / -1Q / -27 / -1G permanently off Sentry
+/// (~185 events/day combined).
+///
+/// Scope: only the three domains that surface session-expired today
+/// (`llm_provider`, `backend_api`, `rpc`). Composio's OAuth-state 401
+/// is excluded — that's actionable and must reach Sentry.
+#[cfg(feature = "crash-reporting")]
+pub fn is_session_expired_event(event: &sentry::protocol::Event<'_>) -> bool {
+ let tags = &event.tags;
+ let Some(domain) = tags.get("domain").map(String::as_str) else {
+ return false;
+ };
+ if !matches!(domain, "llm_provider" | "backend_api" | "rpc") {
+ return false;
+ }
+
+ let status_is_401 = tags
+ .get("status")
+ .and_then(|s| s.parse::<u16>().ok())
+ .is_some_and(|code| code == 401);
+
+ let direct = event.message.as_deref();
+ let from_exception = event.exception.last().and_then(|e| e.value.as_deref());
+ let body_matches = [direct, from_exception]
+ .into_iter()
+ .flatten()
+ .any(is_session_expired_message);
+
+ if status_is_401 && body_matches {
+ return true;
+ }
+
+ // Pre-flight rpc guard has no status tag — accept on body alone,
+ // scoped to the rpc dispatcher (other domains don't emit the
+ // "no session token stored" sentinel).
+ if domain == "rpc" && body_matches {
+ return true;
+ }
+
+ false
+}
+
+pub fn is_transient_http_status(status: &str) -> bool {
+ TRANSIENT_HTTP_STATUSES.contains(&status)
+}
+
+pub fn is_transient_http_status_code(status: u16) -> bool {
+ let status = status.to_string();
+ is_transient_http_status(status.as_str())
+}
+
+pub fn contains_transient_transport_phrase(message: &str) -> bool {
+ let lower = message.to_ascii_lowercase();
+ TRANSIENT_TRANSPORT_PHRASES
+ .iter()
+ .any(|phrase| lower.contains(phrase))
+}
+
+pub fn is_updater_transient_http_status(status: u16) -> bool {
+ UPDATER_TRANSIENT_HTTP_STATUSES.contains(&status)
+}
+
+pub fn is_updater_transient_message(message: &str) -> bool {
+ let lower = message.to_ascii_lowercase();
+ UPDATER_TRANSIENT_MESSAGE_PHRASES
+ .iter()
+ .any(|phrase| lower.contains(phrase))
+}
+
+#[cfg(feature = "crash-reporting")]
+fn event_has_transient_transport_phrase(event: &sentry::protocol::Event<'_>) -> bool {
+ event
+ .message
+ .as_deref()
+ .is_some_and(contains_transient_transport_phrase)
+ || event
+ .logentry
+ .as_ref()
+ .is_some_and(|log| contains_transient_transport_phrase(&log.message))
+ || event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(contains_transient_transport_phrase)
+ })
+}
+
+#[cfg(feature = "crash-reporting")]
+fn event_has_updater_transient_message(event: &sentry::protocol::Event<'_>) -> bool {
+ event
+ .message
+ .as_deref()
+ .is_some_and(is_updater_transient_message)
+ || event
+ .logentry
+ .as_ref()
+ .is_some_and(|log| is_updater_transient_message(&log.message))
+ || event.exception.values.iter().any(|exception| {
+ exception
+ .value
+ .as_deref()
+ .is_some_and(is_updater_transient_message)
+ })
+}
+
+#[cfg(feature = "crash-reporting")]
+fn event_has_updater_domain(event: &sentry::protocol::Event<'_>) -> bool {
+ matches!(
+ event.tags.get("domain").map(String::as_str),
+ Some("update") | Some("update.check_releases") | Some("updater")
+ )
+}
+
+#[cfg(feature = "crash-reporting")]
+fn is_transient_domain_failure(event: &sentry::protocol::Event<'_>, domain: &str) -> bool {
+ let tags = &event.tags;
+ if tags.get("domain").map(String::as_str) != Some(domain) {
+ return false;
+ }
+
+ match tags.get("failure").map(String::as_str) {
+ Some("non_2xx") => tags
+ .get("status")
+ .is_some_and(|status| is_transient_http_status(status)),
+ Some("transport") => event_has_transient_transport_phrase(event),
+ _ => false,
+ }
+}
+
+/// Transient backend API failures (gateway hiccups, scheduled downtime).
+/// Match by event tags written by report_error at the authed_json call site.
+#[cfg(feature = "crash-reporting")]
diff --git a/crates/openhuman-core/src/core/observability_expected_error_classification.rs b/crates/openhuman-core/src/core/observability_expected_error_classification.rs
new file mode 100644
index 0000000000..d83c6e380d
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_expected_error_classification.rs
@@ -0,0 +1,532 @@
+pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
+ let lower = message.to_ascii_lowercase();
+ // Explicit host ownership/reportability metadata precedes text heuristics.
+ let backend_error = crate::inference::registry::classify_backend_error(None, message);
+ if backend_error
+ .as_ref()
+ .is_some_and(|classification| classification.backend_owned && !classification.reportable)
+ {
+ return Some(ExpectedErrorKind::BackendErrorCodeOwned);
+ }
+ // Explicitly reportable guard failures must bypass broader text demotions.
+ if backend_error.as_ref().is_some_and(|classification| {
+ classification.provider_owned
+ && classification.reportable
+ && matches!(
+ classification.kind,
+ Some(
+ crate::inference::registry::BackendErrorKind::PayloadTooLarge
+ | crate::inference::registry::BackendErrorKind::ContextLengthExceeded
+ )
+ )
+ }) {
+ return None;
+ }
+ // Check the Codex-CLI import envelope first: it is highly specific
+ // (literal `codex cli auth` / `.codex/auth.json`) and carries no overlap
+ // with the generic matchers below, so ordering is for clarity, not
+ // precedence. See `ExpectedErrorKind::CodexCliAuthUnavailable`. The
+ // keyring/persist failure path (`upsert_profile`) stringifies to generic
+ // keychain / "auth profile" / SQLite text that contains neither anchor,
+ // so a real defect in the import still falls through to capture.
+ if lower.contains("codex cli auth") || lower.contains(".codex/auth.json") {
+ return Some(ExpectedErrorKind::CodexCliAuthUnavailable);
+ }
+ // TAURI-RUST-5EH — `approval_decide` resolved a request_id whose row was
+ // already decided / lazily expired / superseded (benign race per the
+ // inline-approvals design spec). `rpc::approval_decide` emits this exact
+ // `"no pending approval found"` wording ONLY after `store::get_decision`
+ // confirms a persisted decision exists; the genuine never-registered case
+ // raises `"no pending approval ever registered"` (matched by neither this
+ // arm nor any below), so a real lost-registration defect still reaches
+ // Sentry. See `ExpectedErrorKind::ApprovalNoPendingRace`.
+ if lower.contains("no pending approval found") {
+ return Some(ExpectedErrorKind::ApprovalNoPendingRace);
+ }
+ // TAURI-RUST-CGP — a remote MCP server answered the connect handshake with
+ // HTTP 401 (`McpUnauthorizedError`). `connections::connect` already stores a
+ // `needs_auth` flag so the UI prompts for OAuth sign-in (#3733 / #3719), but
+ // the `mcp_clients_connect` RPC still re-raises the stringified error here.
+ // It is preventable user-state (the server needs sign-in) with no
+ // Sentry-actionable signal — demote it. Highly specific anchor; no overlap
+ // with the generic matchers below. See `is_mcp_server_needs_auth_message`.
+ if is_mcp_server_needs_auth_message(&lower) {
+ return Some(ExpectedErrorKind::McpServerNeedsAuth);
+ }
+ // #5805 — a wallet-derived key was needed and the user has no wallet. An
+ // optional feature nobody enabled is not a defect. Placed beside the other
+ // highly-specific anchors: the needle is a full sentence produced by one
+ // constant, so it cannot collide with the generic matchers below, and
+ // ordering here is for clarity rather than precedence.
+ if is_wallet_not_configured_message(&lower) {
+ return Some(ExpectedErrorKind::WalletNotConfigured);
+ }
+ // TAURI-RUST-117K et al. — see `ExpectedErrorKind::ModuleUnavailable`.
+ if is_module_unavailable_message(message) {
+ return Some(ExpectedErrorKind::ModuleUnavailable);
+ }
+ if lower.contains("local ai is disabled") {
+ return Some(ExpectedErrorKind::LocalAiDisabled);
+ }
+ // `"no api key is configured"` covers the composio direct-mode factory
+ // bail (`client.rs`: "composio direct mode selected but no api key is
+ // configured"). `composio_list_connections` now short-circuits that
+ // state to an empty list, so the dominant 5 s-poll leak (TAURI-RUST-R4)
+ // no longer reaches here — but other direct-mode ops the user invokes
+ // explicitly (execute / authorize) can still surface it, and it is the
+ // same user-config state with no Sentry-actionable signal.
+ //
+ // `"no api key supplied"` is Cohere's exact 401 body wording when the
+ // hosted `/v2/embed` endpoint is called with an empty bearer (the BYO-key
+ // embedding path with no key configured). The `CohereEmbedding::embed`
+ // guard now fast-fails before issuing that request, but this arm demotes
+ // any residual 401 of that shape — older clients, or a present-but-rejected
+ // key — so the flood (TAURI-RUST-52S) stays out of Sentry either way.
+ if is_api_key_unset_message(&lower) {
+ return Some(ExpectedErrorKind::ApiKeyMissing);
+ }
+ // Check `ChannelSupervisorRestart` BEFORE `is_loopback_unavailable` and
+ // `is_network_unreachable_message`: the supervisor wrapper contains
+ // substrings (`error sending request for url`, OS-localized WSAETIMEDOUT
+ // bodies, occasionally `connection refused`) that would otherwise classify
+ // as `NetworkUnreachable` (which only demotes to `warn!` — still a Sentry
+ // event) or `LoopbackUnavailable`. The supervisor's own restart loop
+ // handles the condition; per-restart messages carry no actionable Sentry
+ // signal (TAURI-RUST-15 / -BB). Sustained outages still surface via
+ // `health.bus` / `FAIL_ESCALATE_THRESHOLD`, which is a separate path.
+ if is_channel_supervisor_restart_message(&lower) {
+ return Some(ExpectedErrorKind::ChannelSupervisorRestart);
+ }
+ // Check `is_loopback_unavailable` BEFORE `is_network_unreachable_message`:
+ // a loopback `Connection refused` body shape would otherwise demote to the
+ // broader `NetworkUnreachable` bucket and lose the boot-window vs.
+ // user-environment distinction. Mirrors the `ProviderUserState`-before-
+ // `BackendUserError` precedence pattern from #1795 (PR comment).
+ if is_loopback_unavailable(&lower) {
+ return Some(ExpectedErrorKind::LoopbackUnavailable);
+ }
+ // Check `is_ollama_user_config_rejection` BEFORE the generic network /
+ // backend-error matchers: the GX "daemon unreachable at localhost" shape
+ // contains a loopback host but no `Connection refused (os error …)`
+ // marker, and the XS / MA / KM 400/404 shapes are pure user-config —
+ // wrong model name, model not pulled, daemon opted-in but not running.
+ // Route them to the dedicated arm so they share the `ProviderUserState`
+ // bucket with the composio / OAuth user-state errors instead of falling
+ // through to capture. See `is_ollama_user_config_rejection`.
+ if is_ollama_user_config_rejection(&lower) {
+ return Some(ExpectedErrorKind::ProviderUserState);
+ }
+ if is_network_unreachable_message(&lower) {
+ return Some(ExpectedErrorKind::NetworkUnreachable);
+ }
+ // Check `is_upstream_edge_block_message` BEFORE the transient/backend
+ // matchers: an HTML 403 gateway page from a provider embed call carries a
+ // `403`/`forbidden` token that no other matcher claims, but routing it to
+ // its own bucket keeps "edge/CDN block" distinct from genuine transient
+ // 5xx and from the actionable JSON 4xx that must still page (TAURI-RUST-8S3).
+ if is_upstream_edge_block_message(&lower) {
+ return Some(ExpectedErrorKind::UpstreamEdgeBlock);
+ }
+ if is_transient_upstream_http_message(&lower) {
+ return Some(ExpectedErrorKind::TransientUpstreamHttp);
+ }
+ if lower.contains("binary not found") {
+ return Some(ExpectedErrorKind::LocalAiBinaryMissing);
+ }
+ // Check `is_provider_user_state_message` BEFORE `is_backend_user_error_message`:
+ // composio's "Toolkit X is not enabled" lands as a 4xx that both would
+ // match, and the more specific `ProviderUserState` bucket is the right
+ // home — see the variant doc-comment for OPENHUMAN-TAURI-… coverage.
+ if is_provider_user_state_message(&lower) {
+ return Some(ExpectedErrorKind::ProviderUserState);
+ }
+ // TAURI-RUST-8FQ — the OpenAI ChatGPT/Codex OAuth access token expired with
+ // no usable refresh token. The provider HTTP layer
+ // (`provider::http_error::api_error` / `chat_via_responses`) already demotes its
+ // own per-attempt event, but the same `anyhow::bail!` string re-raises here
+ // at the RPC boundary (`jsonrpc` → `report_error_or_expected`); route it to
+ // the shared `ProviderUserState` bucket so the re-report is demoted too
+ // instead of leaking the event the emit-site already suppressed. User-state
+ // — recovery is reconnecting OpenAI; Sentry has no remediation path. The
+ // markers are distinct from the backend "invalid token" session-expiry
+ // wording matched below, so this does not shadow that arm.
+ if tinyinference_providers::is_openai_oauth_session_expired_message(message) {
+ return Some(ExpectedErrorKind::ProviderUserState);
+ }
+ // TAURI-RUST-5MV — ollama.com hosted-inference 500 (`Internal Server Error
+ // (ref: <uuid>)`) for `*:cloud` models. The provider HTTP layer
+ // (`native_chat` / `streaming_chat` / `api_error`) already demotes its own
+ // per-attempt event and re-raises the actionable
+ // "Ollama cloud is temporarily unavailable …" string; this catches the
+ // re-report at the agent / RPC boundary (`provider_chat` →
+ // `report_error_or_expected`, the `domain=agent` half of the flood). Routed
+ // to `TransientUpstreamHttp` — it is an external upstream 5xx the
+ // reliable-provider layer retries + falls back over, with no client lever.
+ // Delegates to the single-source provider matcher so the phrasing can't
+ // drift. Distinct anchor from the matchers above, so it shadows nothing.
+ if tinyinference_llm::failure::is_ollama_cloud_internal_500_message(message) {
+ return Some(ExpectedErrorKind::TransientUpstreamHttp);
+ }
+ if is_backend_user_error_message(&lower) || is_plan_required_message(message) {
+ return Some(ExpectedErrorKind::BackendUserError);
+ }
+ // Check `is_session_expired_message` BEFORE `is_embedding_backend_auth_failure`:
+ // the OpenHuman-backend embedding 401 "Invalid token" envelope
+ // (`Embedding API error (401 …): {"error":"Invalid token"}`) is a
+ // recoverable session expiry (TAURI-RUST-4K5, #2786), not a generic
+ // backend error. The broader `is_embedding_backend_auth_failure` matcher
+ // below would otherwise demote that exact wire shape to `BackendUserError`
+ // first and swallow the re-auth signal. `is_session_expired_message` is
+ // narrowly anchored (parenthesised `(401` + the `"error":"Invalid token"`
+ // envelope), so the bare-status `Embedding API error 401 …` shape and
+ // BYO-key 401s still fall through to the matchers below.
+ if is_session_expired_message(message) {
+ return Some(ExpectedErrorKind::SessionExpired);
+ }
+ // The backend rejected the stored TinyHumans API key (401 on an api-key
+ // credential, `BackendApiError::ApiKeyRejected`). User-state — the key was
+ // revoked or mistyped and only a new key recovers it — so it shares the
+ // credential-lapse bucket. Kept out of `is_session_expired_message` on
+ // purpose: that predicate feeds the `SessionExpired` publish, which must
+ // not clear a session because an API key failed.
+ if is_api_key_rejected_message(message) {
+ return Some(ExpectedErrorKind::SessionExpired);
+ }
+ if is_embedding_backend_auth_failure(&lower) {
+ return Some(ExpectedErrorKind::SessionExpired);
+ }
+ // TAURI-RUST-5JR — a custom embeddings endpoint with no embeddings route
+ // (the user pointed the Custom (OpenAI-compatible) provider at a chat-only
+ // base URL, e.g. DeepSeek, which 404s every `/embeddings` POST).
+ // Deterministic user-config state, re-emitted on every memory re-embed;
+ // the embeddings settings UI surfaces an actionable "pick an
+ // embeddings-capable provider" message. Demote to info. Scoped to 404/405
+ // only so a real 500 from a valid embeddings endpoint stays in Sentry.
+ if is_embedding_endpoint_absent(&lower) {
+ return Some(ExpectedErrorKind::ProviderConfigRejection);
+ }
+ // TAURI-RUST-9SK — the user entered a non-embedding (chat) model id as the
+ // embeddings model (e.g. an OpenRouter `…:free` chat model), so the
+ // embeddings endpoint 400s `Model <id> does not exist` on every memory
+ // re-embed (2205 events / 1 user). OpenRouter's bare `"does not exist"` +
+ // integer `"code":400` body matches none of the chat-side phrases in
+ // `is_provider_config_rejection_message` (which key on the OpenAI-native
+ // `"does not exist or you do not have access"` / `model_not_found`), so
+ // without this it reaches Sentry. Deterministic user-config state; the
+ // settings UI surfaces an actionable "pick an embeddings-capable model"
+ // remediation. Scoped to the 400 model-rejection body so a real 400
+ // (oversized input, server fault) stays visible — same polarity contract as
+ // `is_embedding_endpoint_absent`.
+ if is_embedding_model_rejected(&lower) {
+ return Some(ExpectedErrorKind::ProviderConfigRejection);
+ }
+ // Provider config-rejection (unknown model / abstract tier leaked to a
+ // custom provider / model-specific temperature). Body-shape based and
+ // intrinsically scoped to third-party providers — the OpenHuman
+ // backend never emits these phrases. See the predicate's polarity
+ // contract. Drops OPENHUMAN-TAURI-WJ / -QW / -HB / -NH re-reports
+ // (#2079 / #2076 / #2202).
+ if tinyinference_providers::is_provider_config_rejection_message(message) {
+ return Some(ExpectedErrorKind::ProviderConfigRejection);
+ }
+ if is_local_ai_capability_unavailable_message(&lower) {
+ return Some(ExpectedErrorKind::LocalAiCapabilityUnavailable);
+ }
+ if tinyinference_providers::is_budget_exhausted_message(message) {
+ return Some(ExpectedErrorKind::BudgetExhausted);
+ }
+ if is_backend_unavailable_message(message) {
+ return Some(ExpectedErrorKind::BackendUnavailable);
+ }
+ if is_prompt_injection_blocked_message(&lower) {
+ return Some(ExpectedErrorKind::PromptInjectionBlocked);
+ }
+ // Context-window-exceeded re-report from a higher layer (agent /
+ // web_channel). The provider api_error cascade suppresses its own
+ // emit; this catches the re-raise. Delegates to the single-source
+ // provider matcher so the phrasing can't drift. Runs last so a more
+ // specific matcher always wins.
+ if tinyinference_llm::failure::is_context_window_exceeded_message(message) {
+ return Some(ExpectedErrorKind::ContextWindowExceeded);
+ }
+ if is_disk_full_message(&lower) {
+ return Some(ExpectedErrorKind::DiskFull);
+ }
+ if is_windows_file_system_limitation_message(&lower) {
+ return Some(ExpectedErrorKind::WindowsFileSystemLimitation);
+ }
+ if is_config_load_timed_out_message(&lower) {
+ return Some(ExpectedErrorKind::ConfigLoadTimedOut);
+ }
+ // OS-level config-read denial on a file that exists (user-environment, zero
+ // local lever). Keyed on the config-read anchor + an access-denied/locked io
+ // signal; NotFound / unseen kinds fall through and keep paging. Requires the
+ // loader to surface the full io chain (#3962).
+ if is_config_read_io_failure_message(&lower) {
+ return Some(ExpectedErrorKind::ConfigReadIoFailure);
+ }
+ // Empty-provider-response re-report from the web-channel layer. Runs
+ // last so an earlier, more specific matcher always wins. See the
+ // variant doc-comment and [`is_empty_provider_response_message`] for
+ // the two-emit-site rationale (agent layer is handled by the typed
+ // `AgentError::skips_sentry()` in PR #2790; this covers the
+ // web_channel re-report where the type was flattened to a String).
+ if is_empty_provider_response_message(&lower) {
+ return Some(ExpectedErrorKind::EmptyProviderResponse);
+ }
+ // RPC-level filesystem path validation — explicit wire-shape anchors
+ // (root_path / hosted path) prevent accidental demotion of unrelated
+ // errors. See the variant doc-comment and
+ // [`is_filesystem_user_path_invalid_message`] polarity contract.
+ if is_filesystem_user_path_invalid_message(&lower) {
+ return Some(ExpectedErrorKind::FilesystemUserPathInvalid);
+ }
+ None
+}
+
+/// Detect filesystem-out-of-space errors that bubble up from any syscall
+/// (`open`, `write`, `mkdir`, `rename`). Three platform-stable renderings:
+///
+/// - **POSIX `ENOSPC`** (Linux / macOS / BSD): `std::io::Error` renders as
+/// `"No space left on device (os error 28)"`. The errno-name substring is
+/// what we anchor on — case-folded to `"no space left on device"`.
+/// - **Windows `ERROR_DISK_FULL` (112)**: `std::io::Error` renders as
+/// `"There is not enough space on the disk. (os error 112)"`. Anchor on
+/// `"not enough space on the disk"`.
+/// - **Windows `ERROR_HANDLE_DISK_FULL` (39)**: same wire text but errno 39.
+/// The text anchor already covers it.
+///
+/// A fourth shape comes from call sites that render the `io::ErrorKind` debug
+/// name instead of the `io::Error` Display — notably the auth-profile
+/// lock-create annotation, which emits
+/// `"... (kind=Some(StorageFull), os_code=Some(28))"` (Sentry TAURI-RUST-4SZ).
+/// That string carries no "no space left on device" text, so anchor
+/// additionally on the cross-platform `StorageFull` ErrorKind token (std maps
+/// ENOSPC / `ERROR_DISK_FULL` / `ERROR_HANDLE_DISK_FULL` all to
+/// `ErrorKind::StorageFull`).
+///
+/// A fifth shape comes from SQLite itself. When the engine detects the
+/// disk-full condition during its own page bookkeeping (journal/WAL extension)
+/// before the next syscall surfaces an errno, rusqlite renders the `SQLITE_FULL`
+/// result code as `"database or disk is full"` (Sentry TAURI-RUST-B6N, hit
+/// while committing a local SQLite transaction). `SQLITE_FULL` has only two causes:
+/// genuine ENOSPC/ERROR_DISK_FULL (always the case in practice — the same
+/// burst always produces an os-error-28/112 sibling event) or a
+/// `max_page_count` PRAGMA cap (we set none).
+///
+/// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is
+/// the five words `"database or disk is full"` — Our local SQLite write
+/// call-sites wrap it with `format!("<verb>: {e}")` (e.g. `"commit tx: ..."`),
+/// so the phrase lands as the **suffix** of the local emit. The **extended**
+/// shape carries the full error-code envelope, `"database or disk is full:
+/// Error code 13: Insertion failed because database is full"` (Sentry
+/// TAURI-RUST-4R8, a local job-queue claim); here
+/// the canonical phrase sits mid-string, so the suffix anchor can't catch it.
+/// We detect this shape by requiring **both** local fragments together — the
+/// `"database or disk is full"` phrase AND the libsqlite3-sys `code_to_str`
+/// token `"insertion failed because database is full"` — which only rusqlite's
+/// own `SQLITE_FULL` Display emits as a pair. Requiring both (rather than the
+/// `code_to_str` token alone) keeps the silencer from matching a remote
+/// provider body that merely quotes the token half — e.g. an OpenAI-compatible
+/// `OpenAiEmbedding::embed` failure framed as `"Embedding API error (… Error
+/// code 13: Insertion failed because database is full)"` whose *server-side*
+/// SQLite is full is operator-actionable and must still surface to Sentry
+/// (codex CR on #3911). Both arms anchor on local-emit fragments rather than a
+/// bare `contains("database or disk is full")` so the silencer does not match a
+/// non-2xx backend response body whose payload happens to mention the phrase
+/// (e.g. a managed-backend 5xx whose server-side SQLite is full). Non-2xx
+/// backend bodies are framed by
+/// `integrations::client::post` / `composio::client` as `"Backend returned
+/// <status> <reason> for <METHOD> <url>: <detail>"` — an operator-actionable
+/// server/storage failure that must still surface to Sentry. As
+/// defense-in-depth for the edge case where the backend body itself ends with
+/// the phrase, reject any message that also carries the `"backend returned "`
+/// envelope prefix (codex CR on #3672, mirrors the precedent set by
+/// [`is_backend_user_error_message`]).
+///
+/// This is defense-in-depth for the genuinely
+/// unpreventable **write** paths (a write can't succeed on a full disk); the
+/// read path no longer emits this error at all (it degrades to a lock-free
+/// read — see `AuthProfilesStore::load`).
+fn is_disk_full_message(lower: &str) -> bool {
+ if lower.contains("no space left on device")
+ || lower.contains("not enough space on the disk")
+ || lower.contains("storagefull")
+ {
+ return true;
+ }
+ // Two SQLITE_FULL renderings — see the fifth-shape section above. The
+ // **bare** shape lands the phrase as a suffix (after trimming trailing
+ // whitespace / punctuation that closures + JSON wrappers append). The
+ // **extended** shape (TAURI-RUST-4R8) puts the phrase mid-string followed
+ // by `: Error code 13: Insertion failed because database is full`; require
+ // BOTH local fragments so we match only rusqlite's own `SQLITE_FULL` Display
+ // and never a remote provider body that merely quotes the `code_to_str`
+ // half (codex CR on #3911). The negative `"backend returned "` guard
+ // rejects the remote 5xx envelope as a further line of defense.
+ let trimmed = lower.trim_end_matches(|c: char| {
+ c.is_ascii_whitespace() || matches!(c, '.' | ',' | ';' | ':' | '"' | '\'')
+ });
+ let bare_suffix = trimmed.ends_with("database or disk is full");
+ let extended_local = lower.contains("database or disk is full")
+ && lower.contains("insertion failed because database is full");
+ (bare_suffix || extended_local) && !lower.contains("backend returned ")
+}
+
+/// Detect Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665) —
+/// a host-filesystem condition (fragmentation, USN journal overflow,
+/// filter-driver resource cap) that is persistent, locale-independent
+/// in the `(os error N)` suffix, and unrecoverable by the app.
+///
+/// Anchor on the locale-stable `(os error 665)` suffix. The Italian
+/// locale rendering is `"Impossibile completare l'operazione a causa
+/// di un limite del file system (os error 665)"` (Sentry
+/// TAURI-RUST-QT0, 6,050 events / 1 user). The `(os error 665)` suffix
+/// is the same across every locale, so no locale-detection is needed.
+///
+/// Polarity contract — only match when `(os error 665)` appears in
+/// the message body. This is a standard `std::io::Error` Display
+/// rendering, so it will never appear in a remote backend body
+/// accidentally.
+fn is_windows_file_system_limitation_message(lower: &str) -> bool {
+ lower.contains("(os error 665)") && !lower.contains("backend returned ")
+}
+
+/// Detect the literal `"Config loading timed out"` string produced by
+/// [`crate::config::ops::load_config_with_timeout`] /
+/// [`crate::config::ops::reload_config_snapshot_with_timeout`]
+/// when `tokio::time::timeout` elapses around `Config::load_or_init` /
+/// `Config::load_from_config_path`.
+fn is_config_load_timed_out_message(lower: &str) -> bool {
+ lower.contains("config loading timed out")
+}
+
+/// Detect a config-file READ that failed because the operating system refused
+/// access to a file that **exists** — i.e. an unpreventable user-environment
+/// condition with zero local lever, not an OpenHuman defect.
+///
+/// `Config::load_or_init` (`impl_load.rs`) takes its read branch only after
+/// `config_path.exists()` returns true, then `read_to_string` is retried 5× and
+/// still fails. On a healthy install that never happens; in the wild a user's
+/// `config.toml` can be ACL-denied, held open by another process (antivirus /
+/// backup agent), or a OneDrive "files on demand" placeholder that won't
+/// hydrate. We cannot unlock or re-ACL a foreign-held file, so the per-poll
+/// re-report (TAURI-RUST-DME: `inference_downloads_progress` re-loads config on
+/// every poll → 36k events / 1 user) carries no Sentry-actionable signal.
+///
+/// Polarity contract — demote **only** when BOTH hold:
+/// 1. an OpenHuman config-read context anchor is present — either
+/// `"failed to read config file"` (`load_or_init` retry path,
+/// `impl_load.rs`, the DME surface) or `"reading config.toml from"`
+/// (`load_from_config_path` snapshot-reload path) — AND
+/// 2. an OS-level *access-denied / locked* signal is present.
+///
+/// `NotFound` (`os error 2` / "cannot find the file"), "is a directory", and any
+/// io kind not enumerated here are deliberately EXCLUDED: a file that vanished
+/// after the `exists()` check is a TOCTOU race / app defect and MUST keep
+/// paging. This matcher is only meaningful once the loader surfaces the full
+/// chain (`{:#}`, #3962) — the io fragment lives in the source, not the top
+/// `with_context` line.
+fn is_config_read_io_failure_message(lower: &str) -> bool {
+ let has_config_read_anchor =
+ lower.contains("failed to read config file") || lower.contains("reading config.toml from");
+ if !has_config_read_anchor {
+ return false;
+ }
+ // A directory (or otherwise non-regular file) at the config path is a
+ // bad-install / corruption signal that MUST keep paging. On Windows reading
+ // a directory surfaces the same `Access is denied. (os error 5)` shape as a
+ // genuine ACL denial, so the io-signal check below cannot tell them apart;
+ // the read site (`impl_load.rs`) now fails a directory fast with this
+ // distinct wording, and we belt-and-braces exclude it here too. (Codex P2.)
+ if lower.contains("is a directory") || lower.contains("not a file") {
+ return false;
+ }
+ // The file is owned by a uid other than the one reading it. That is an
+ // OpenHuman defect, not user-environment state: *we* pick the runtime uid
+ // (Dockerfile) and *we* write the config at 0600 (`impl_load.rs`), and the
+ // container entrypoint chowns only the workspace directory — so a
+ // stale-uid `config.toml` inside a reused volume denies every config RPC
+ // (including the sign-in `auth_store_session`) while `/health` stays green.
+ // There IS a local lever here, so it must keep paging. Genuine ACL /
+ // antivirus / OneDrive denials carry no marker and stay demoted.
+ //
+ // Referencing the loader's constant (rather than a copied literal) makes
+ // the producer/consumer coupling a compile-time one: the marker is already
+ // lowercase, so it matches `lower` verbatim.
+ if lower.contains(crate::config::schema::CONFIG_OWNER_MISMATCH_MARKER) {
+ return false;
+ }
+ lower.contains("access is denied")
+ || lower.contains("permission denied")
+ || lower.contains("being used by another process")
+ || lower.contains("cannot access the file")
+ || lower.contains("(os error 5)")
+ || lower.contains("(os error 32)")
+}
+
+// (The whatsapp_data SQLite busy/corrupt matchers were removed with the
+// store's move to the Tauri shell — the core no longer produces the
+// `[whatsapp_data] ingest failed:` envelope they anchored on.)
+
+fn is_embedding_backend_auth_failure(lower: &str) -> bool {
+ tinyinference_embeddings::probe::is_embedding_backend_auth_failure(lower)
+}
+
+/// Detect a custom embeddings endpoint that exposes **no embeddings API** —
+/// the `OpenAiEmbedding` client POSTed `/embeddings` and the host answered
+/// `404 Not Found` (route absent) or `405 Method Not Allowed`. Canonical wire
+/// shape from `tinyinference-embeddings/src/cloud.rs`:
+///
+/// ```text
+/// Embedding API error (404 Not Found): <body>
+/// Embedding API error (405 Method Not Allowed): <body>
+/// ```
+///
+/// Deterministic user-config state: the user pointed the Custom
+/// (OpenAI-compatible) embeddings provider at a base URL whose host has no
+/// embeddings endpoint (e.g. a chat-only provider like DeepSeek). Every memory
+/// re-embed re-emits it (TAURI-RUST-5JR, ~2685 events / 9 users) and the
+/// settings UI surfaces an actionable remediation — Sentry has no fix to make.
+///
+/// Polarity (important): scoped to **404/405 only**. A `500` from a valid
+/// embeddings endpoint is a real server fault and must keep reaching Sentry; a
+/// `400` (e.g. oversized input) is prevented at source by the chunk cap
+/// (#3598) and likewise stays visible. Reused by
+/// `inference::embedding_host::rpc::update_settings` as the save-time hard-block signal so the
+/// two never drift.
+pub(crate) fn is_embedding_endpoint_absent(lower: &str) -> bool {
+ tinyinference_embeddings::probe::is_embedding_endpoint_absent(lower)
+}
+
+/// Detect a custom/cloud embeddings endpoint that IS an embeddings API but
+/// **rejected the configured model id** — the user pasted a non-embedding
+/// (chat/reasoning) model into the embeddings model field. Canonical wire shape
+/// from `tinyinference-embeddings/src/cloud.rs` (TAURI-RUST-9SK, ~2205 events):
+///
+/// ```text
+/// Embedding API error (400 Bad Request): {"error":{"message":"Model nvidia/nemotron-3-super-120b-a12b does not exist","code":400}}
+/// ```
+///
+/// Deterministic user-config state, re-emitted on every memory re-embed; the
+/// embeddings settings UI surfaces an actionable "pick an embeddings-capable
+/// model" remediation (appended to the message at the emit site). The
+/// OpenRouter body — bare `"does not exist"` with an integer `"code":400` —
+/// matches none of the chat-side phrases in
+/// `tinyinference_providers::is_provider_config_rejection_message` (those key on the
+/// OpenAI-native `"does not exist or you do not have access"` /
+/// `model_not_found`), so this dedicated matcher is what demotes it.
+///
+/// Polarity (important): scoped to **400** + a model-rejection body. A bare
+/// 400 (oversized input — prevented at source by the chunk cap #3598) or a 500
+/// from a valid embeddings endpoint is a real fault and must keep reaching
+/// Sentry, so this never fires on them.
+fn is_embedding_model_rejected(lower: &str) -> bool {
+ tinyinference_embeddings::probe::is_embedding_model_rejected(lower)
+}
+
+/// Whether `msg` is the hosted backend refusing a feature the account's plan
+/// does not include (#8081): a `[…_PLAN_REQUIRED]` code in the head
+/// TinyMemory gives a hosted failure, `[CODE] memory API {route} …`. Matched
+/// as that delimited code, so a description that merely mentions the code
+/// is not a plan refusal. User state, never an expired session.
diff --git a/crates/openhuman-core/src/core/observability_network_provider_errors.rs b/crates/openhuman-core/src/core/observability_network_provider_errors.rs
new file mode 100644
index 0000000000..ac1b6676f9
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_network_provider_errors.rs
@@ -0,0 +1,553 @@
+fn is_network_unreachable_message(lower: &str) -> bool {
+ lower.contains("error sending request for url")
+ || lower.contains("dns error")
+ || lower.contains("failed to lookup address")
+ || lower.contains("nodename nor servname")
+ || lower.contains("connection refused")
+ || lower.contains("connection reset")
+ // OPENHUMAN-TAURI-EM (128 events): the channel supervisor wraps
+ // `discord_listen()`'s anyhow chain as `format!("Channel {} error:
+ // {e:#}; restarting", ...)`, which lands as
+ // `"Channel discord error: IO error: Operation timed out (os error
+ // 60); restarting"`. The discord gateway TCP/WebSocket connection
+ // timing out is transient network state, not a code bug — the
+ // supervisor already retries with exponential backoff. Same shape
+ // surfaces on every channel (slack/telegram/...) once the
+ // underlying socket hits ETIMEDOUT, so we match on the platform-
+ // agnostic phrase, symmetric with `"connection reset"` /
+ // `"connection refused"` above. Errno renderings are not pinned
+ // because `(os error 60)` (BSD/macOS), `(os error 110)` (Linux),
+ // `(os error 10060)` (Windows `WSAETIMEDOUT`), and bare prose
+ // `"operation timed out"` (hyper / tungstenite / std::io) all
+ // share the same lowercase substring.
+ || lower.contains("operation timed out")
+ || lower.contains("network is unreachable")
+ || lower.contains("no route to host")
+ || lower.contains("tls handshake")
+ || lower.contains("unexpected eof during handshake")
+ || lower.contains("certificate verify failed")
+ || lower.contains("http error: 200 ok")
+ || lower.contains("http version must be 1.1 or higher")
+}
+
+/// Detect the canonical supervisor-wrap shape emitted by
+/// `channels::runtime::supervision::spawn_supervised_listener` —
+/// `"Channel <name> error: <inner>; restarting"`. Language-agnostic
+/// (anchored on the Rust wrapper, not the inner error wording) so it
+/// covers OS-localized variants (TAURI-RUST-BB Chinese-Windows
+/// WSAETIMEDOUT body) that escape the English-only network anchors in
+/// [`is_network_unreachable_message`].
+///
+/// The supervisor restarts the listener with its own exponential backoff;
+/// sustained outages surface via separate `health.bus` events /
+/// `FAIL_ESCALATE_THRESHOLD`. Per-restart messages carry no actionable
+/// Sentry signal — Sentry has no remediation path beyond what the
+/// supervisor already does (TAURI-RUST-15 ~11.4 k events / -BB ~815
+/// events on self-hosted `tauri-rust`).
+///
+/// Anchors on three substrings together to avoid false positives:
+/// - leading `"channel "` (with trailing space disambiguates from
+/// unrelated mentions like `"channels"` or `"channel-runtime"`)
+/// - `" error:"` (the wrapper's literal separator)
+/// - `"; restarting"` (the wrapper's literal trailer)
+///
+/// A bare `"…; restarting"` log line without the `"Channel <name> error:"`
+/// preamble must NOT classify — that's a generic restart note from some
+/// other subsystem and Sentry signal there may still be actionable.
+fn is_channel_supervisor_restart_message(lower: &str) -> bool {
+ lower.starts_with("channel ") && lower.contains(" error:") && lower.contains("; restarting")
+}
+
+/// Detect transient upstream HTTP failures that have bubbled up out of the
+/// provider layer and into higher-level domains (`agent`, `web_channel`, …).
+///
+/// The reliable-provider stack already retries / falls back on
+/// [`TRANSIENT_PROVIDER_HTTP_STATUSES`] (408/429/502/503/504), and the
+/// `before_send` filter drops the per-attempt provider events that carry
+/// `domain=llm_provider`. But the same error is *also* returned via
+/// `Result::Err` and re-reported by callers that wrap the provider — e.g.
+/// `agent.run_single` (OPENHUMAN-TAURI-5Z), `web_channel.run_chat_task`,
+/// scheduler tick handlers — under a different `domain` tag, escaping the
+/// provider-scoped filter and producing one Sentry event per failed turn.
+///
+/// The canonical wire format from `provider::http_error::api_error` is:
+/// `"<provider> API error (<status>): <sanitized>"` — e.g.
+/// `"OpenHuman API error (504 Gateway Timeout): error code: 504"`. Pin the
+/// match to that exact `"api error (<status>"` prefix so an unrelated message
+/// that merely mentions "504" (a log line, a doc URL) is not silenced.
+///
+/// Also matches the second canonical wire shape: tungstenite's
+/// `WsError::Http(response)` Display, which renders as `"HTTP error: <status>"`
+/// (and which `socket::ws_loop::run_connection` wraps as
+/// `"WebSocket connect: HTTP error: 502 Bad Gateway"`). Per
+/// OPENHUMAN-TAURI-5P (~110 events) and -EZ (~51 events), backend
+/// staging/production load balancers emit HTTP 502/504 during the WebSocket
+/// upgrade handshake; tungstenite surfaces those as `WsError::Http` and the
+/// socket reconnect loop already handles them via exponential backoff. Each
+/// `FAIL_ESCALATE_THRESHOLD` escalation fires `report_error_or_expected` with
+/// the formatted reason, which would land in Sentry as `domain=socket`
+/// noise without this matcher (the existing `domain=integrations`
+/// before_send filter scopes too narrowly).
+///
+/// Three separator variants cover every observed shape: trailing space
+/// (`"HTTP error: 502 Bad Gateway"`), trailing newline (`"HTTP error: 502\n…"`
+/// from chained errors), and trailing colon (`"HTTP error: 502: …"`). Bare
+/// `"HTTP error: 502"` at end-of-string is not matched on purpose — the
+/// status integer alone could collide with unrelated log lines containing
+/// `"HTTP error: 5023"` (port number, runbook ID).
+fn is_transient_upstream_http_message(lower: &str) -> bool {
+ tinyinference_llm::failure::TRANSIENT_PROVIDER_HTTP_STATUSES
+ .iter()
+ .any(|code| {
+ lower.contains(&format!("api error ({code}"))
+ || lower.contains(&format!("api error {code} "))
+ || lower.contains(&format!("http error: {code} "))
+ || lower.contains(&format!("http error: {code}\n"))
+ || lower.contains(&format!("http error: {code}:"))
+ })
+}
+
+/// Detect a non-2xx **HTML 403/Forbidden gateway page** returned to a provider
+/// embedding call — the signature of an edge/CDN/WAF or regional block sitting
+/// in front of the provider API, where the request never reached the provider
+/// app and the body is a generic gateway error page rather than the provider's
+/// JSON error envelope.
+///
+/// Canonical wire shape (TAURI-RUST-8S3, `CohereEmbedding::embed` emit site):
+/// `"Cohere embed API error (403 Forbidden): <!doctype html>…<title>403</title>403 Forbidden"`.
+/// The same shape also covers the OpenAI/Voyage and custom OpenAI-compatible
+/// embed paths when their upstream is fronted by the same edge tier, so we do
+/// not pin to a single provider prefix.
+///
+/// Two conditions must BOTH hold:
+///
+/// 1. A **403 token** is present — either the `403` status inside the embed
+/// error prefix or a bare `403 forbidden` in the body.
+/// 2. An **HTML gateway marker** is present (`<!doctype html`, `<html`,
+/// `<title>403`) — proving the body is a gateway page, not the provider's
+/// structured error.
+///
+/// And the body must **not** look like a JSON envelope (no `{` … `"message"` /
+/// `"error"` JSON shape). This is the discrimination guard: Cohere's real JSON
+/// 403 (`{"message": "invalid api key"}`) and any genuine request-shape bug in
+/// our client that round-trips a JSON 4xx stay classified `None` and keep
+/// reaching Sentry. Only the bodiless edge HTML page is demoted.
+///
+/// `genuinely-unpreventable`: endpoint correct, key sent, retry can't clear an
+/// edge policy keyed on the user's network reputation / geo / IP — Sentry has
+/// no remediation path.
+fn is_upstream_edge_block_message(lower: &str) -> bool {
+ // Must originate from an embedding call so we don't silence an HTML 403
+ // surfaced by some unrelated path.
+ let is_embed = lower.contains("embed");
+ if !is_embed {
+ return false;
+ }
+ // A JSON envelope means the provider's app answered with a structured
+ // error — actionable, must page. Don't demote.
+ let looks_like_json = lower.contains('{')
+ && (lower.contains("\"message\"")
+ || lower.contains("\"error\"")
+ || lower.contains("\"detail\""));
+ if looks_like_json {
+ return false;
+ }
+ let has_403 = lower.contains("(403 ") || lower.contains("403 forbidden");
+ let has_html_marker =
+ lower.contains("<!doctype html") || lower.contains("<html") || lower.contains("<title>403");
+ has_403 && has_html_marker
+}
+
+/// Detect non-2xx HTTP failures returned from the backend integrations / composio
+/// clients that are by definition user-input or user-auth-state problems — not
+/// bugs Sentry can act on.
+///
+/// The canonical wire format from
+/// [`crate::integrations::client::IntegrationClient::post`] / `get`
+/// and [`crate::integrations::composio::client::ComposioClient`] is:
+/// `"Backend returned <status> <reason> for <METHOD> <url>: <detail>"` — e.g.
+/// `"Backend returned 400 Bad Request for POST https://api.example.com/integrations/composio/authorize: Composio authorization failed: 400 …"`
+/// (OPENHUMAN-TAURI-BC: user submitted SharePoint authorize without filling in
+/// the required Tenant Name field). The backend correctly returned a 4xx; the
+/// UI already surfaces the structured error to the user via toast — Sentry has
+/// no remediation path because the request was malformed *by the user's
+/// input*, not by our code.
+///
+/// We pin the match to the `"backend returned "` prefix so an unrelated
+/// message merely mentioning "400" (a log line, doc URL) is not silenced.
+///
+/// We classify only 4xx codes, with **two exclusions**:
+/// - `408 Request Timeout` and `429 Too Many Requests` are *transient* — they
+/// are surfaced via [`is_transient_upstream_http_message`] for the provider
+/// path and stay actionable for the backend path so a sustained 429 (rate
+/// limit cliff) still pages.
+///
+/// 5xx is intentionally **not** classified here — server-side failures from
+/// our backend are real bugs that should reach Sentry. The transient
+/// 502/503/504 deduplication is handled by the threshold logic in callers
+/// (see e.g. `crate::platform::socket::ws_loop::FAIL_ESCALATE_THRESHOLD`).
+fn is_backend_user_error_message(lower: &str) -> bool {
+ let Some(rest) = lower.split_once("backend returned ").map(|(_, r)| r) else {
+ return false;
+ };
+ let status_digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
+ let Ok(status) = status_digits.parse::<u16>() else {
+ return false;
+ };
+ // 4xx (except transient 408 / 429 which are handled separately).
+ matches!(status, 400..=499) && status != 408 && status != 429
+}
+
+/// Detect third-party provider validation failures that bubble up as
+/// user-state errors — composio trigger registry mismatch, toolkit not
+/// enabled, OAuth scopes missing, required fields left blank.
+///
+/// Unlike [`is_backend_user_error_message`], this classifier is **body-text
+/// shape-based** rather than HTTP-status-based, so it catches the cases
+/// where the composio backend wraps a Composio API 4xx as a 500 with the
+/// real validation message embedded in the body (OPENHUMAN-TAURI-3R / -3S
+/// / -97 — `"Backend returned 500 … Trigger type GITHUB_PUSH_EVENT not
+/// found"`, `"Backend returned 500 … Missing required fields: Your
+/// Subdomain"`). These would otherwise escape the 4xx-only matcher and
+/// fire as actionable Sentry events even though the underlying condition
+/// is user-state (the trigger slug isn't in composio's registry, the
+/// toolkit wasn't enabled by the user, the form field was left blank, …).
+///
+/// Also handles the gmail-sync 403 (OPENHUMAN-TAURI-33) where the
+/// composio sync loop surfaces the upstream Google OAuth scopes error as
+/// `"HTTP 403: Request had insufficient authentication scopes."`. The
+/// remediation is "user re-authorizes with the right scope" — nothing
+/// Sentry can act on.
+///
+/// All matches are substring-based against the lower-cased message so the
+/// classifier survives caller wrapping (rpc.invoke_method, agent.run_single,
+/// `[composio:gmail]` prefixes, anyhow chains, …).
+fn is_provider_user_state_message(lower: &str) -> bool {
+ // TAURI-RUST-HXF: a direct BYO provider (groq `on_demand` free tier)
+ // rejected a *single* request whose token count exceeds the account's
+ // tokens-per-minute cap — `413 Payload Too Large … Request too large …
+ // tokens per minute (TPM): Limit 8000, Requested 42084`. It is permanently
+ // non-viable on the current tier (not a burst that retry/backoff clears)
+ // and OpenHuman cannot raise a third-party account's TPM tier, so it is
+ // user-config state, not a product bug. NOTE: a *managed-backend*
+ // `PAYLOAD_TOO_LARGE` guard-leak is force-captured (returns `None`) earlier
+ // in `expected_error_kind`, before this matcher runs, so this arm only ever
+ // sees direct-provider TPM rejections. Shared matcher so the wording can't
+ // drift.
+ if tinyinference_llm::failure::is_provider_rate_cap_exceeded_message(lower) {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-3R / -3S: composio enable_trigger when the slug isn't
+ // in the trigger registry (e.g. user clicked a stale UI option).
+ // Backend returns 500 with `"Trigger type GITHUB_PUSH_EVENT not found"`.
+ // Also covers the alternate phrasing `"Cannot enable trigger … not found"`.
+ if (lower.contains("trigger type ") && lower.contains("not found"))
+ || (lower.contains("cannot enable trigger") && lower.contains("not found"))
+ {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-34: composio rejected a tool call because the user
+ // hasn't enabled the toolkit yet. Wire shape:
+ // `Backend returned 400 … Toolkit "get" is not enabled`.
+ if lower.contains("toolkit ") && lower.contains("is not enabled") {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-XX: custom_openai upstream rejected the request with
+ // its own 400. Wire shape produced by
+ // `inference/provider/compatible.rs::is_custom_openai_upstream_bad_request_http_400`:
+ //
+ // custom_openai API error (400 Bad Request): {"error":{
+ // "message":"Bad request to upstream provider",
+ // "type":"upstream_error","status":400}}
+ //
+ // Anchored to the `custom_openai api error (400` prefix so this can't
+ // silence unrelated errors that happen to mention both
+ // "bad request to upstream provider" and "upstream_error" elsewhere
+ // (e.g. a future provider whose envelope reuses one of those strings).
+ if lower.contains("custom_openai api error (400")
+ && lower.contains("bad request to upstream provider")
+ && lower.contains("upstream_error")
+ {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-97: composio authorize with a blank required field —
+ // SharePoint Subdomain, WhatsApp WABA ID, Tenant Name, etc.
+ // Backend returns 500 with `"Missing required fields: …"` body.
+ //
+ // **Intentionally broad** — unlike the trigger/toolkit arms, this is a
+ // single substring with no second anchor. Composio's wire shape varies
+ // per provider (`Missing required fields: Tenant Name`, `Missing
+ // required fields: Your Subdomain (example: 'your-subdomain' for…)`,
+ // `Missing required fields: WABA ID (WhatsApp Business Account ID…)`)
+ // and embedding every variant would be brittle. Accepted false-positive
+ // surface: a non-composio caller whose error happens to contain
+ // `"missing required fields"` (e.g. `"Internal error: missing required
+ // fields in config"`) will also demote to info. This is fine — every
+ // current emit site routed through `report_error_or_expected` is scoped
+ // to composio / integrations envelopes, so a stray collision would have
+ // to come from a brand-new call site that explicitly opts in.
+ // See `unrelated_missing_required_fields_classifies_as_accepted_false_positive`
+ // for the documented surface.
+ if lower.contains("missing required fields") {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-33: gmail sync hit an OAuth scope wall —
+ // `HTTP 403: Request had insufficient authentication scopes.`
+ // (or any sibling OAuth scope rejection from composio's toolkits).
+ if lower.contains("insufficient authentication scopes") {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-S7: provider policy rejection on Kimi's coding
+ // endpoint when requests are not sent from an approved coding-agent
+ // client. Canonical body contains `access_terminated_error` and:
+ // "currently only available for Coding Agents ...".
+ if lower.contains("access_terminated_error")
+ || lower.contains("currently only available for coding agents")
+ {
+ return true;
+ }
+
+ // TAURI-RUST-X9 (#1166): direct-mode composio call against the user's
+ // personal Composio v3 tenant rejected with a 401 because the stored
+ // API key is invalid / revoked / has the wrong prefix. The canonical
+ // wire shape rendered by
+ // `crates/openhuman-core/src/integrations/composio/tools/impl/network/composio.rs::response_error`
+ // and the various direct-mode op wrappers is:
+ //
+ // `[composio-direct] list_connections failed: Composio v3
+ // connected_accounts failed: HTTP 401: Invalid API key: ak_…`
+ //
+ // The "Invalid API key" body is rendered for every direct-mode
+ // endpoint (list_connections / list_tools / authorize / etc.), so we
+ // gate on the **`[composio-direct]` prefix** + either of the two
+ // anchors that prove the failure came from the v3 auth wall:
+ // - `HTTP 401` (the status the v3 wall returns)
+ // - `Invalid API key` (the body Composio puts in the JSON)
+ //
+ // Requiring the `[composio-direct]` prefix keeps this from
+ // accidentally swallowing unrelated bugs — backend-mode 401s from
+ // `integrations/composio/*` still carry the `Backend returned 401`
+ // shape (handled by the failure-tag flow with `status="401"`),
+ // not the `HTTP 401: Invalid API key` shape.
+ //
+ // Remediation is purely user-state: the user must rotate / re-enter
+ // their Composio key via Settings → Composio → Direct mode. Sentry
+ // has no actionable signal — the UI surfaces the "Invalid API key"
+ // toast and the polling layer already retries every 5 s.
+ //
+ // Drops Sentry TAURI-RUST-X9 (~15.7 k events / ~22 h, single user,
+ // release [email protected]+c25fc8e5fd3e).
+ if lower.contains("[composio-direct]")
+ && (lower.contains("http 401") || lower.contains("invalid api key"))
+ {
+ return true;
+ }
+
+ // TAURI-RUST-K27 — the set-key sibling of X9. `composio_set_api_key`'s
+ // validate-before-store probe (added in #4318) rejects an obviously invalid
+ // BYO key *before* persisting it and returns its own user-facing prose,
+ // `COMPOSIO_INVALID_API_KEY_USER_MESSAGE` ("Invalid Composio API key. Re-enter
+ // a valid key in Connections > Composio."). That string carries
+ // neither the `[composio-direct]` prefix nor an `HTTP 401` token, and the word
+ // "Composio" splits the `invalid … api key` sequence — so the X9 arm above
+ // never claims it and the RPC-boundary `report_error` leaked as a Sentry error
+ // (473 events / 53 users, starting ~5 h after #4318 merged). Same user-state as
+ // X9: an invalid/revoked BYO key with zero client-side lever to make it valid;
+ // the Settings UI already surfaces the actionable re-entry copy. Anchor on the
+ // distinctive `COMPOSIO_INVALID_API_KEY_ANCHOR` phrase — it can only be produced
+ // by our own const (a genuine set-path defect renders a different `store_…
+ // failed` / `save config failed` body that still pages). Keyed off the shared
+ // anchor const (not a copied literal) and coupled to the typed source by
+ // `demotes_composio_set_key_invalid_key_rejection` so a reword that drops the
+ // phrase fails CI instead of silently re-opening the leak.
+ if lower.contains(crate::integrations::composio::direct_auth::COMPOSIO_INVALID_API_KEY_ANCHOR) {
+ return true;
+ }
+
+ // TAURI-RUST-34H — composio backend endpoint (e.g.
+ // `/integrations/composio/connections`) wraps an upstream
+ // Cloudflare anti-bot challenge as `Backend returned 500 Internal
+ // Server Error … 403 <!DOCTYPE html>…<title>Just a moment...</title>…`.
+ // The CF interstitial is keyed by the user's network reputation /
+ // geo / cookie state — there is nothing in `openhuman_core` that
+ // can act on it. Backend ops or the user's network is the
+ // remediation path; Sentry has no signal.
+ //
+ // Double-anchor on the Cloudflare challenge title + the literal
+ // "cloudflare" token to avoid colliding with unrelated bodies that
+ // merely mention "Just a moment" in a different context.
+ //
+ // Drops ~8.9 k events / 14d (TAURI-RUST-34H, sibling -32G / -34J /
+ // -323 share the same cascade).
+ if lower.contains("just a moment...") && lower.contains("cloudflare") {
+ return true;
+ }
+
+ // OPENHUMAN-TAURI-YJ: `inference/provider/models/catalog_listing.rs::list_configured_models` probed a
+ // user-configured custom-provider's `/models` endpoint and the upstream
+ // server returned 404. Wire shape emitted at `ops.rs:118-122`:
+ //
+ // "provider returned 404: {\"error\":\"path \\\"/api/v1/models\\\" not found\"}"
+ //
+ // (the trailing body is whatever the upstream server wrote — `{"error":...}`,
+ // `{"detail":...}`, bare HTML, etc.; we only anchor on the `provider returned
+ // 404` prefix). The semantic is unambiguous: the user pointed a custom
+ // OpenAI-compatible provider at a base URL that does not host a `/models`
+ // listing endpoint (wrong base, model-only proxy, typo'd path). The model
+ // dropdown already surfaces the failure inline — Sentry has no remediation.
+ //
+ // **404 only**. Other 4xx from the same emit site stay actionable:
+ // - 401 / 403: BYO-key auth wall — actionable misconfiguration; the
+ // `does_not_classify_byo_key_provider_401_as_session_expired` contract
+ // (#2286) intentionally keeps these in Sentry.
+ // - 400: typically request-shape bugs in OUR client; must escalate.
+ // - 429 / 5xx: transient — handled by other matchers / retry policy.
+ //
+ // No `inference/provider/models/catalog_listing.rs::list_configured_models` other than this site emits
+ // the `provider returned NNN` prefix (verified via grep), so the prefix
+ // alone is a sufficient anchor.
+ //
+ // TAURI-RUST-8X3: anchor to the position where `provider returned 404` is
+ // the formatted *error prefix* — never to any occurrence in the response
+ // body. The primary fix classifies the *raw* error at the source
+ // (`inference/ops.rs::inference_list_models`) before any log prefix is
+ // applied, so the raw shape always starts with `provider returned 404:`.
+ // The one historically-observed prefixed re-report path is the
+ // `inference/ops.rs` `error!("[inference::ops] list_models:error: {err}")`
+ // log line — handled below as an explicit prefixed shape.
+ //
+ // A bare `contains` would mis-fire: a genuine 400/500 list-models failure
+ // formats as `provider returned 500: <body>`, and if `<body>` merely
+ // relays an upstream phrase like `upstream provider returned 404 ...`, the
+ // loose substring would demote that real 4xx/5xx defect out of Sentry —
+ // exactly the failures the discrimination guard
+ // (`does_not_classify_non_404_list_models_failures_as_user_state`) says
+ // must still escalate. So we require the anchor to be the prefix, not buried
+ // text. (Mirrors the parenthesised `(401` anchoring in
+ // `is_session_expired_message`.)
+ if lower.starts_with("provider returned 404")
+ || lower.contains("list_models:error: provider returned 404")
+ {
+ return true;
+ }
+
+ false
+}
+
+/// Detect "<capability> is disabled / unavailable for this RAM tier" errors
+/// emitted by the local-AI service when the user's hardware tier doesn't
+/// support a capability (OPENHUMAN-TAURI-3B: vision asset download invoked
+/// on a 0–4 GB tier). These are pure user-state conditions — the local-AI
+/// service surfaces them so the UI can prompt the user to switch tiers —
+/// and carry no remediable signal for Sentry.
+///
+/// The two canonical wire shapes today both contain `"for this ram tier"`:
+///
+/// - `"Vision is disabled for this RAM tier. Switch to the 4-8 GB tier or
+/// above to enable it."` — from `inference/local/service/assets.rs::ensure_capability_ready`
+/// - `"vision summaries are unavailable for this RAM tier. Use OCR-only
+/// summarization or switch to a higher local AI tier."` —
+/// from `inference/local/service/vision_embed.rs::summarize`
+///
+/// Anchor the classifier to that exact substring so an unrelated message
+/// that merely mentions "RAM tier" out of context is not silenced.
+fn is_local_ai_capability_unavailable_message(lower: &str) -> bool {
+ lower.contains("for this ram tier")
+}
+
+/// Detect prompts rejected by the in-process prompt-injection guard.
+///
+/// Both enforcement actions that produce a user-visible error — `Blocked`
+/// (score ≥ 0.70) and `ReviewBlocked` (score ≥ 0.55) — share a unique
+/// prefix that cannot appear in any other error path. Anchored to the exact
+/// strings emitted by `prompt_guard_user_message` in
+/// `crates/openhuman-core/src/inference/local/ops.rs`.
+fn is_prompt_injection_blocked_message(lower: &str) -> bool {
+ lower.contains("prompt flagged for security review")
+ || lower.contains("prompt blocked by security policy")
+}
+
+/// Detect an RPC-level filesystem path validation failure from user input.
+///
+/// Anchored on the two known wire shapes — both emitted at the RPC entry
+/// boundary when a user typed/picked a path that doesn't resolve to an
+/// existing directory:
+///
+/// - `"root_path is not a directory: <path>"` — historically emitted by the
+/// now-removed knowledge-vault `vault_create` path when the chosen folder
+/// didn't exist or pointed at a file (Sentry TAURI-RUST-4QH). Kept as a
+/// classifier fixture since the wire shape may recur from other callers.
+/// - `"hosted path is not a directory: <path>"` —
+/// `openhuman_rpc::http_host::path_utils` when an HTTP host config
+/// references a missing directory. Not yet observed in Sentry but
+/// shares the same user-input failure mode; preempts a future ID.
+///
+/// Both are deterministic Err returns at the validation gate of an RPC
+/// handler, BEFORE any side-effect happens. The UI already surfaces the
+/// typed error and Sentry has no remediation path.
+///
+/// **Polarity contract** — explicit wire-shape anchors prevent accidental
+/// demotion of future errors whose bodies happen to contain "path is not
+/// a directory:" in a different context:
+///
+/// - `skills::ops_install` emits `"{path} is not a directory — refusing
+/// to remove"` (em-dash separator, no "root_path" or "hosted path"
+/// prefix). That is an `rm -rf` safety guard catching an UNEXPECTED
+/// state, not user input — it must STAY actionable.
+/// - A generic `"input config path is not a directory: /etc/foo"` from a
+/// future provider/wallet/storage error would NOT match (no known
+/// prefix) and would reach Sentry as intended.
+///
+/// All matches are substring-based against the lower-cased message so
+/// the classifier survives caller wrapping (`rpc.invoke_method`,
+/// anyhow context chains, …).
+fn is_filesystem_user_path_invalid_message(lower: &str) -> bool {
+ lower.contains("root_path is not a directory:")
+ || lower.contains("hosted path is not a directory:")
+}
+
+/// Detect the agent harness's empty-provider-response bail.
+///
+/// Anchored on the literal user-facing string emitted at
+/// `agent::session_host::turn` —
+/// `"The model returned an empty response. Please try again."` — which is
+/// preserved verbatim as the provider/model returns a body with
+/// `text_chars=0 thinking_chars=0 tool_calls=0`.
+///
+/// This catches the **web-channel re-report** (Sentry TAURI-RUST-4Z1):
+/// `web_chat::run_chat_task` wraps the failure as
+/// `"run_chat_task failed client_id=… error=The model returned an empty
+/// response. Please try again."` and routes it through
+/// `report_error_or_expected` after the typed
+/// `AgentError::EmptyProviderResponse` was flattened to a `String` at the
+/// native-bus boundary (so the agent-layer `skips_sentry()` suppression
+/// from PR #2790 can't reach it).
+///
+/// Anchored on `"model returned an empty response"` (not the looser
+/// `"empty response"`) so the sibling phrases stay actionable:
+/// `"summarizer returned empty response, falling through"`
+/// (`payload_summarizer`) and `"provider returned an empty response;
+/// returning empty extraction"` (`subagent_host::extract_tool`) are
+/// internal fall-through paths with different wording and are NOT
+/// silenced.
+fn is_empty_provider_response_message(lower: &str) -> bool {
+ lower.contains("model returned an empty response")
+}
+
+/// Capture an error to Sentry with structured tags.
+///
+/// `domain` and `operation` are required and become tags `domain:<…>` and
+/// `operation:<…>`. `extra` is an optional list of extra tag pairs. The error
+/// itself is rendered via `Display` and emitted as a `tracing::error!` event,
+/// which the Sentry tracing layer turns into a Sentry event under the active
+/// scope.
+///
+/// Use stable, low-cardinality values for tag keys/values so Sentry can group
+/// and aggregate. High-cardinality data (full IDs, payloads) belongs in the
+/// error message body, not in tags.
diff --git a/crates/openhuman-core/src/core/observability_plan_and_session_errors.rs b/crates/openhuman-core/src/core/observability_plan_and_session_errors.rs
new file mode 100644
index 0000000000..32d82d07c4
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_plan_and_session_errors.rs
@@ -0,0 +1,201 @@
+pub fn is_plan_required_message(msg: &str) -> bool {
+ // The code runs from the nearest `[` before the head and is all
+ // `A-Z0-9_`, as TinyMemory cleans it.
+ msg.match_indices(PLAN_REQUIRED_HEAD).any(|(at, _)| {
+ msg[..at].rsplit_once('[').is_some_and(|(_, code)| {
+ code.chars()
+ .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_')
+ })
+ })
+}
+
+/// What follows a plan code in TinyMemory's hosted failure head.
+const PLAN_REQUIRED_HEAD: &str = "_PLAN_REQUIRED] memory API ";
+
+/// Detect **app-session-expired** boundary errors that bubble up from any
+/// backend-touching call site (agent, web channel, cron, integrations).
+///
+/// This is also the JSON-RPC dispatch-site classifier. Keep it stricter than
+/// a bare "401 + unauthorized" pair: OpenAI / Anthropic BYO-key failures,
+/// Composio scope failures, and channel-provider 401s are actionable scoped
+/// errors, not proof that the user's OpenHuman app session expired.
+///
+/// The canonical OpenHuman session-expired wire shapes:
+///
+/// - `"OpenHuman API error (401 Unauthorized): {…\"Session expired. Please
+/// log in again.\"…}"` — emitted by `provider::http_error::api_error` from the
+/// OpenHuman backend and re-raised through `agent::run_single` /
+/// `web_chat::run_chat_task` (OPENHUMAN-TAURI-26). The
+/// `"session expired"` substring anchors the match to the OpenHuman
+/// backend's session-renewal body, not the bare numeric status.
+/// - `"OpenHuman API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
+/// — same emit site, same wire shape as the `Session expired` body, but the
+/// OpenHuman backend swaps in `"Invalid token"` for the JWT-validity
+/// rejection branch (vs. the explicit session-renewal branch).
+/// OPENHUMAN-TAURI-4P0. The conjunctive anchor — `"OpenHuman API error
+/// (401"` **and** the envelope-shaped `"\"error\":\"Invalid token\""` —
+/// keeps the #2286 contract intact: bare `"Invalid token"`, OpenAI /
+/// Anthropic BYO-key 401s, Discord upstream-bot-token rejections, and
+/// provider scope errors still route to Sentry as actionable.
+/// - `"Embedding API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
+/// — TAURI-RUST-4K5 (~118 events, escalating on 0.56.0). Same OpenHuman
+/// backend session-expired envelope as 4P0, but the embedding client at
+/// `tinyinference-embeddings/src/cloud.rs` wraps it with the
+/// `"Embedding API error"` prefix instead of `"OpenHuman API error"`.
+/// Uses the same conjunctive-anchor pattern so BYO-key embedding 401s
+/// from third-party providers (OpenAI / Voyage / Cohere) still escalate
+/// — guarded by `does_not_classify_embedding_byo_key_401_as_session_expired`.
+/// - `"OpenHuman streaming API error (401 Unauthorized): {…\"error\":\"Invalid token\"…}"`
+/// — TAURI-RUST-1EE (~110 events, ongoing on 0.56.0). Same envelope as
+/// 4P0, wrapped by the streaming-chat path at
+/// `inference/provider/compatible.rs:949` with the
+/// `"OpenHuman streaming API error"` prefix. The `streaming` token means
+/// the 4P0 anchor doesn't match, so it needs its own prefix arm; BYO-key
+/// streaming 401s still escalate — guarded by
+/// `does_not_classify_streaming_byo_key_401_as_session_expired`.
+/// - `"SESSION_EXPIRED: backend session not active — sign in to resume LLM work"`
+/// — the `llm_gate::is_signed_out` sentinel from
+/// `providers::openhuman_backend::resolve_bearer`.
+/// - `"no backend session token; run auth_store_session first"` and
+/// `"session JWT required"` — local pre-flight guards that fire when the
+/// stored profile is empty (`#1465`-ish onboarding spam) or has been
+/// cleared by a previous 401 cycle. Both shapes are OpenHuman-specific.
+///
+/// At the JSON-RPC dispatch boundary the same strict match controls
+/// `DomainEvent::SessionExpired` publication, so downstream/provider 401s stay
+/// recoverable and do not clear the stored app session.
+pub fn is_session_expired_message(msg: &str) -> bool {
+ // #8081: tinymemory's hosted mapping words every 403, including
+ // `[MEMORY_PLAN_REQUIRED]` (memory not in the account's plan), as "the
+ // session expired or the API key was rejected (HTTP 403 …)". A 403 or a
+ // plan refusal never means the session expired, and signing out on one
+ // logged a valid user out on opening a Memory tab.
+ if is_plan_required_message(msg) || msg.contains("(HTTP 403") {
+ return false;
+ }
+ let lower = msg.to_ascii_lowercase();
+ lower.contains("session expired")
+ || lower.contains("no backend session token")
+ || lower.contains("session jwt required")
+ || msg.contains("SESSION_EXPIRED")
+ || (msg.contains("OpenHuman API error (401") && msg.contains("\"error\":\"Invalid token\""))
+ || (msg.contains("Embedding API error (401") && msg.contains("\"error\":\"Invalid token\""))
+ // OPENHUMAN-TAURI-4P0 — OpenHuman backend's "Invalid token" 401
+ // envelope. Both anchors must be present: the OpenHuman-scoped
+ // `"OpenHuman API error (401"` prefix (so a third-party provider's
+ // `"OpenAI API error (401 Unauthorized): invalid_api_key"` cannot
+ // match), AND the envelope-shaped `"\"error\":\"Invalid token\""`
+ // (so bare prose mentions of "invalid token" — Discord OAuth
+ // failures, generic upstream errors covered by #2286 — stay
+ // actionable in Sentry).
+ || (msg.contains("OpenHuman API error (401")
+ && msg.contains("\"error\":\"Invalid token\""))
+ // TAURI-RUST-4K5 — same OpenHuman backend "Invalid token" envelope
+ // wrapped by `tinyinference-embeddings/src/cloud.rs` with the
+ // `"Embedding API error"` prefix instead of `"OpenHuman API error"`.
+ // Same conjunctive-anchor pattern as 4P0: the embedding-scoped
+ // prefix gates the match so a third-party BYO-key embedding 401
+ // (e.g. OpenAI/Voyage/Cohere rejecting the user's own API key)
+ // stays actionable — guarded by
+ // `does_not_classify_embedding_byo_key_401_as_session_expired`.
+ || (msg.contains("Embedding API error (401")
+ && msg.contains("\"error\":\"Invalid token\""))
+ // TAURI-RUST-1EE — same OpenHuman backend "Invalid token" envelope
+ // wrapped by the streaming-chat path at
+ // `inference/provider/compatible.rs:949` with the
+ // `"OpenHuman streaming API error"` prefix. The `streaming` token
+ // between `OpenHuman` and `API error` means the 4P0 anchor
+ // (`"OpenHuman API error (401"`) does not match it, so the
+ // streaming path needs its own prefix arm. Same conjunctive-anchor
+ // pattern keeps third-party BYO-key streaming 401s
+ // (`"OpenAI streaming API error (401): invalid_api_key"`)
+ // escalating — guarded by
+ // `does_not_classify_streaming_byo_key_401_as_session_expired`.
+ || (msg.contains("OpenHuman streaming API error (401")
+ && msg.contains("\"error\":\"Invalid token\""))
+ // TAURI-RUST-N — same OpenHuman backend "Invalid token" envelope
+ // wrapped by the tinyagents `ProviderError::Display` format
+ // (`TinyAgentsError::Provider(Box<ProviderError>)`). The historical
+ // `OpenHumanBackendModel` delegated to `OpenAiModel`
+ // which formats errors as `"{provider} returned HTTP {status}: {body}"`,
+ // producing `"OpenHuman returned HTTP 401: ..."` — different from the
+ // `"OpenHuman API error (401"` prefix the classic `api_error` path emits.
+ // Same conjunctive-anchor pattern: scoped to `"OpenHuman returned HTTP
+ // {status}"` (so a third-party BYO-key 401 from a provider whose label
+ // contains "OpenHuman" cannot match) AND the envelope-shaped
+ // `"\"error\":\"Invalid token\""` (so bare prose mentions of "invalid
+ // token" stay actionable).
+ || (msg.contains("OpenHuman returned HTTP 401")
+ && msg.contains("\"error\":\"Invalid token\""))
+}
+
+/// Detect a remote MCP server's connect-time 401 — the user must sign in to
+/// that server (OAuth), not a code defect. Anchored on the canonical
+/// TinyMCP unauthorized error text, which renders as
+/// `"mcp unauthorized for \`<endpoint>\` (HTTP 401…)"`.
+///
+/// Conjunctive match — both anchors must hit (input already lower-cased):
+///
+/// 1. `"mcp unauthorized for "` — the stable module error prefix. Scopes the match to
+/// the MCP transport's own 401 so an unrelated "unauthorized" / "401" from
+/// another domain cannot borrow this demotion.
+/// 2. `"(http 401"` — the parenthesised status the module message always emits
+/// (with or without the trailing `resource metadata:` discovery hint).
+///
+/// `connections::connect` already classifies this and stores a `needs_auth`
+/// flag so the UI prompts for sign-in; this predicate keeps the parallel
+/// `mcp_clients_connect` RPC re-report out of Sentry. See
+/// [`ExpectedErrorKind::McpServerNeedsAuth`].
+fn is_mcp_server_needs_auth_message(lower: &str) -> bool {
+ lower.contains("mcp unauthorized for ") && lower.contains("(http 401")
+}
+
+/// Detect the wallet's "not set up yet" sentinel, anywhere in the message.
+///
+/// Substring rather than equality **on purpose**: the same condition arrives
+/// bare from a direct RPC and context-wrapped from any caller that does
+/// `format!("{context}: {e}")`. Both are the same user-state and both must be
+/// demoted, so the predicate must not care how many layers wrapped it.
+///
+/// The needle is the shared producer constant, not a copy, so a wording change
+/// moves both sides at once. `wallet_not_configured_sentinel_is_ascii_lowercase`
+/// pins the remaining assumption — that the constant is already lowercase, so
+/// it can be compared against the pre-lowercased haystack without allocating.
+/// See [`ExpectedErrorKind::WalletNotConfigured`].
+fn is_wallet_not_configured_message(lower: &str) -> bool {
+ lower.contains(crate::web3::wallet::WALLET_NOT_CONFIGURED_MESSAGE)
+}
+
+/// Detect the "a configured provider has no API key" user-config state.
+///
+/// Single source of truth for the `ApiKeyMissing` wording so the
+/// `expected_error_kind` demotion arm and the cron scheduler's halt-on-first
+/// classifier (`is_api_key_unset_failure`, TAURI-RUST-HCK) can never drift
+/// apart. The phrasing is emitted deterministically, before any HTTP, by the
+/// credential guards:
+/// - `inference/provider/compatible_request.rs::credential_for_request`
+/// ("<provider> API key not set. Configure via the web UI …")
+/// - the embeddings credential guards (cohere/openai) + the composio
+/// direct-mode factory bail ("no api key is configured" / "no api key
+/// supplied").
+///
+/// Distinct from a *rejected* key (a provider 401 "Invalid API key"): that is a
+/// present-but-wrong key and stays actionable in Sentry — this matcher is for
+/// the **absent** key only, which has no Sentry remediation path.
+pub fn is_api_key_unset_message(text: &str) -> bool {
+ let lower = text.to_ascii_lowercase();
+ lower.contains("api key not set")
+ || lower.contains("missing api key")
+ || lower.contains("no api key is configured")
+ || lower.contains("no api key supplied")
+}
+
+/// Does this error text describe a **local** LLM provider (loopback host, e.g.
+/// LM Studio / Ollama / llama.cpp on `127.0.0.1:<port>` or `localhost:<port>`)
+/// refusing the connection because its server isn't running?
+///
+/// Single-source wrapper over [`is_loopback_unavailable`] so callers outside
+/// this module (the cron scheduler's retry-halt guard) key off the exact same
+/// matcher the [`expected_error_kind`] classifier uses — the phrasing cannot
+/// drift between the source demotion and the cron suppression. Lowercases the
+/// input to match the classifier's internal contract (TAURI-RUST-12K).
diff --git a/crates/openhuman-core/src/core/observability_provider_local_errors.rs b/crates/openhuman-core/src/core/observability_provider_local_errors.rs
new file mode 100644
index 0000000000..d07f2e3b75
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_provider_local_errors.rs
@@ -0,0 +1,237 @@
+pub fn is_local_provider_unreachable_message(text: &str) -> bool {
+ is_loopback_unavailable(&text.to_ascii_lowercase())
+}
+
+/// Detect the in-process-core boot-window shape: a sibling component
+/// (frontend RPC relay, integrations / composio HTTP clients) tried to
+/// reach the embedded core's `127.0.0.1:<port>` listener before it finished
+/// binding, so the kernel returned `Connection refused`. The condition
+/// self-resolves once startup completes — Sentry has no remediation path.
+///
+/// Conjunctive match — both anchors must hit:
+///
+/// 1. **Loopback host with port**: substring `127.0.0.1:` or `localhost:` so
+/// a doc URL or unrelated mention without a port (`localhost`,
+/// `127.0.0.1\n`) does not match. Pinned to the colon+port pattern
+/// because every observed shape from reqwest / hyper / our own
+/// `IntegrationClient` wraps the host as `<host>:<port>` in the URL the
+/// error chain renders.
+/// 2. **Connection refused with platform errno**: `connection refused (os
+/// error 61)` (macOS / BSD), `connection refused (os error 111)`
+/// (Linux), or `connection refused (os error 10061)` (Windows
+/// `WSAECONNREFUSED`). Pinning to `(os error N)` keeps the matcher from
+/// swallowing higher-level wrappers that merely mention "connection
+/// refused" in prose.
+///
+/// 3. **Locale-independent fallback**: on non-English OS locales the kernel
+/// renders the `ECONNREFUSED` / `WSAECONNREFUSED` text translated (e.g. a
+/// zh-CN Windows host emits `由于目标计算机积极拒绝,无法连接。 (os error
+/// 10061)`), so the English `connection refused` prefix in (2) is absent
+/// and a genuine loopback-refused body would leak past this matcher into
+/// the broad [`is_network_unreachable_message`] bucket (TAURI-RUST-12K).
+/// Recover it by pairing the reqwest/hyper-stable `tcp connect error`
+/// marker with the connect-refused errno alone. Scoped to the three
+/// connect-refused errnos only — the distinct timeout errnos (`60` / `110`
+/// / `10060`, `WSAETIMEDOUT`) are NOT matched, so a loopback *timeout*
+/// still classifies as its own shape rather than being mislabelled
+/// "refused".
+///
+/// Drops OPENHUMAN-TAURI-R5 (~2.5k events, `integrations.get` emit site)
+/// and OPENHUMAN-TAURI-R6 (~2.5k events, the `rpc.invoke_method` re-wrap of
+/// the same trace). Both share `trace_id=6ebf5b62748d5144e541e2cddeabbbd0`
+/// and the canonical body shape:
+///
+/// ```text
+/// error sending request for url (http://127.0.0.1:18474/integrations/composio/connections)
+/// → client error (Connect) → tcp connect error → Connection refused (os error 61)
+/// ```
+///
+/// Without this matcher the body falls through to
+/// [`is_network_unreachable_message`] and demotes as `NetworkUnreachable`,
+/// which conflates an internal lifecycle race with user-environment problems
+/// (VPN drop, captive portal, ISP block) and makes the "what's spiking?"
+/// question un-answerable. See [`ExpectedErrorKind::LoopbackUnavailable`].
+fn is_loopback_unavailable(lower: &str) -> bool {
+ let has_loopback_host = lower.contains("127.0.0.1:") || lower.contains("localhost:");
+ if !has_loopback_host {
+ return false;
+ }
+ // (2) English errno-prefixed form.
+ if lower.contains("connection refused (os error 61)")
+ || lower.contains("connection refused (os error 111)")
+ || lower.contains("connection refused (os error 10061)")
+ {
+ return true;
+ }
+ // (3) Locale-independent fallback: the OS-refused text may be translated,
+ // leaving only the errno. Require the transport-layer `tcp connect error`
+ // marker so a non-connect loopback failure cannot match, and pin to the
+ // connect-refused errnos (not the timeout errnos 60 / 110 / 10060).
+ lower.contains("tcp connect error")
+ && (lower.contains("(os error 61)")
+ || lower.contains("(os error 111)")
+ || lower.contains("(os error 10061)"))
+}
+
+/// Detect Ollama embed call sites that surface a user-config rejection from
+/// the local Ollama daemon — pure user-state errors the UI already surfaces
+/// (toast / settings page warning) where Sentry has no remediation path.
+///
+/// Several canonical wire shapes are covered, all emitted by the TinyAgents
+/// Ollama embedder and the host embed service fallback path:
+///
+/// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the
+/// embedder at a chat / vision model id with a temperature suffix (e.g.
+/// `qwen3-vl:[email protected]`) which Ollama parses as malformed. Wire shape:
+/// `ollama embed failed with status 400 Bad Request: {"error":"invalid model name"}`.
+/// - **OPENHUMAN-TAURI-MA / -KM** (deferred follow-up from PR #2216), and
+/// **TAURI-RUST-K** (~1990 events) / **TAURI-RUST-8K** (~411 events) on
+/// self-hosted Sentry: user configured a model id that the local Ollama
+/// daemon hasn't pulled yet. Wire shape:
+/// `ollama embed failed with status 404 Not Found: {"error":"model \"<id>\" not found, try pulling it first"}`.
+/// (Self-hosted Sentry events still flow from older client releases that
+/// predate this matcher; they drop off naturally as users upgrade.)
+/// - **TAURI-RUST-3X / -8WA** (~982 events on 0.57.52): 501 "embeddings not
+/// supported". Two bodies — the model is chat/vision-only
+/// (`{"error":"this model does not support embeddings"}`) or the Ollama
+/// daemon was started without embed support
+/// (`{"error":"This server does not support embeddings. Start it with `--embeddings`"}`).
+/// - **TAURI-RUST-3E** (~249 events): 401 auth-required Ollama endpoint with
+/// no credentials configured. Wire shape:
+/// `ollama embed failed with status 401 Unauthorized: {"error": "unauthorized"}`.
+/// - **OPENHUMAN-TAURI-GX**: user opted into Ollama embeddings but the
+/// daemon isn't running on `localhost:11434`, so the embed service falls
+/// back to cloud embeddings for the session. Wire shape:
+/// `ollama embeddings opted-in but daemon unreachable at http://localhost:11434; falling back to cloud embeddings for this session`.
+///
+/// All are user-config: the user picked the wrong model id, forgot to pull
+/// it, ran a daemon without embed support, omitted credentials, or forgot to
+/// start the daemon. The remediation is "fix the model id in Settings" /
+/// "run `ollama pull <id>`" / "start ollama with `--embeddings`" / "add a
+/// key" / "start ollama" — none of which Sentry can do for them.
+///
+/// Each arm is anchored on the `"ollama embed"` prefix
+/// (`"ollama embed failed"` for the failed-request shapes,
+/// `"ollama embeddings opted-in"` for the daemon-unreachable fallback) so
+/// unrelated errors elsewhere in the codebase that happen to contain
+/// `"invalid model name"`, `"not found"`, or `"does not support embeddings"`
+/// substrings are not silenced.
+///
+/// Routes to [`ExpectedErrorKind::ProviderUserState`] — the same bucket that
+/// holds the composio / gmail / OAuth user-state errors. We deliberately do
+/// **not** introduce a dedicated Ollama enum variant: the demotion semantics
+/// (drop to `info` log, skip Sentry capture) are identical and adding a new
+/// variant for every provider would balloon the enum without changing
+/// behavior.
+fn is_ollama_user_config_rejection(lower: &str) -> bool {
+ // XS — 400-status user-config (invalid model name, including the
+ // temperature-suffix shape `qwen3-vl:[email protected]` Ollama parses as malformed).
+ if lower.contains("ollama embed failed") && lower.contains("invalid model name") {
+ return true;
+ }
+
+ // MA / KM — 404-status pull-required. The wire shape is JSON-escaped
+ // (`\"<model-id>\" not found`); after lower-casing we still see the
+ // backslash-quoted form. Anchor on `model \"` + `\" not found` so an
+ // unrelated 404 that merely contains `"model"` and `"not found"` is not
+ // swallowed. The `\\"` byte pair in Rust source matches the literal
+ // `\"` sequence in the wire shape.
+ if lower.contains("ollama embed failed")
+ && lower.contains("model \\\"")
+ && lower.contains("\\\" not found")
+ {
+ return true;
+ }
+
+ // 3X / 8WA — 501-status "embeddings not supported". Ollama emits two
+ // bodies for this: the model is chat/vision-only
+ // (`{"error":"this model does not support embeddings"}`) or the daemon
+ // itself was started without embedding support
+ // (`{"error":"This server does not support embeddings. Start it with `--embeddings`"}`,
+ // TAURI-RUST-8WA, ~982 events on 0.57.52). Both are user-side Ollama
+ // config the app can't fix — it can neither swap the user's model nor
+ // restart their daemon with `--embeddings`. Anchor on the shared
+ // `does not support embeddings` phrase (still gated by the
+ // `ollama embed failed` prefix) so a future qualifier wording still demotes.
+ if lower.contains("ollama embed failed") && lower.contains("does not support embeddings") {
+ return true;
+ }
+
+ // TAURI-RUST-3E (~249 events) — 401-status auth failure from Ollama
+ // (user pointed the embedder at an authenticated Ollama endpoint
+ // without configuring credentials, e.g. self-hosted Ollama behind an
+ // auth proxy or Ollama Cloud without API key). Body shape:
+ // `{"error": "unauthorized"}`. Anchor on `ollama embed failed`
+ // + `status 401` so unrelated 401s from other call sites (provider
+ // chat, backend API) aren't silenced.
+ if lower.contains("ollama embed failed") && lower.contains("status 401") {
+ return true;
+ }
+
+ // GX — daemon-unreachable opt-in state. The wire shape is emitted by
+ // the embed service when the user has opted into Ollama in settings
+ // but the daemon isn't responding, so the service falls back to cloud
+ // embeddings for the session. Anchor on the full prefix to keep the
+ // matcher from colliding with unrelated `"daemon unreachable"`
+ // messages from other domains (e.g. backend connection-health logs).
+ if lower.contains("ollama embeddings opted-in but daemon unreachable at") {
+ return true;
+ }
+
+ false
+}
+
+/// Detect transport-level connection failures that fire before any HTTP status
+/// is observed — DNS resolution failures, TCP connect refused/reset, TLS
+/// handshake failures, or ISP/firewall blocks. The canonical shape is
+/// reqwest's `"error sending request for url (…)"`, which surfaces from any
+/// HTTP call site (provider chat, embeddings, backend RPC) when the request
+/// can't reach the server at all.
+///
+/// These are user-environment problems — VPN drop, captive portal, ISP-level
+/// block (OPENHUMAN-TAURI-32: user in RU couldn't reach the managed backend),
+/// firewall — that no amount of retry / fallback on our side can resolve.
+/// Sentry has no signal to act on (no status, no trace, no payload), so each
+/// occurrence is pure noise. Classify them as expected so the report site
+/// logs a breadcrumb rather than spawning an error event.
+///
+/// Loopback `127.0.0.1:<port>` `Connection refused` shapes are routed
+/// through [`is_loopback_unavailable`] *before* this matcher so the
+/// boot-window race against the embedded core keeps its own bucket — see
+/// the precedence comment in [`expected_error_kind`].
+///
+/// Three additional substrings cover wire-shape variants observed in
+/// Wave 4 that the original `"dns error"` / status-code matchers miss:
+///
+/// - `"failed to lookup address"` / `"nodename nor servname"` —
+/// `getaddrinfo()` failure renderings on macOS / BSD libc and POSIX
+/// resolvers (`OPENHUMAN-TAURI-44` ~50 events,
+/// `[socket] Connection failed: WebSocket connect: IO error: failed to
+/// lookup address information: nodename nor servname provided, or not
+/// known`).
+/// - `"http error: 200 ok"` — tungstenite's `WsError::Http(200)` render
+/// when a corporate proxy / captive portal intercepts the WebSocket
+/// handshake and returns a plain HTML 200 page (`OPENHUMAN-TAURI-4P`
+/// ~66 events). Tungstenite-only — reqwest renders HTTP 200 as
+/// `"HTTP status server error (200)"`, so this can't collide with the
+/// regular HTTP call path.
+/// - `"unexpected eof during handshake"` — `native-tls`'s render when the
+/// peer (or an intercepting firewall / antivirus / corporate TLS proxy)
+/// closes the TCP connection mid-TLS-handshake, surfacing as
+/// `"TLS error: native-tls error: unexpected EOF during handshake"`
+/// wrapped by `socket::ws_loop::run_connection` into
+/// `"WebSocket connect: …"` (`TAURI-RUST-4ZD`, first seen on
+/// `[email protected]`, Windows). The existing `"tls handshake"` anchor
+/// misses it because the words aren't contiguous (`"tls error"` …
+/// `"during handshake"`). Same user-environment shape as the other
+/// handshake-stage entries — the socket supervisor already retries with
+/// exponential backoff and Sentry has no actionable signal.
+/// - `"http version must be 1.1 or higher"` — tungstenite's
+/// `ProtocolError::WrongHttpVersion` render. Fires when a server (or
+/// intermediary proxy / HTTP/2-only edge) responds to the WebSocket
+/// upgrade with HTTP/2+, which the WS spec forbids — the handshake
+/// requires HTTP/1.1 (`CORE-RUST-DP`, ~2 events / 24h, first seen on
+/// `[email protected]`). Same shape as the existing handshake-stage
+/// entries: a user-environment / infra misconfiguration that the
+/// client cannot fix; Sentry has no actionable signal beyond what the
+/// socket supervisor's exponential backoff already provides.
diff --git a/crates/openhuman-core/src/core/observability_reporting.rs b/crates/openhuman-core/src/core/observability_reporting.rs
new file mode 100644
index 0000000000..87ce999b3d
--- /dev/null
+++ b/crates/openhuman-core/src/core/observability_reporting.rs
@@ -0,0 +1,689 @@
+pub fn report_error<E: Display + ?Sized>(
+ err: &E,
+ domain: &str,
+ operation: &str,
+ extra: &[Tag<'_>],
+) {
+ // Use the alternate format specifier so `anyhow::Error` renders its full
+ // context chain (outer context + every wrapped cause, joined by ": ").
+ // Plain `Display` impls fall back to the standard representation. Without
+ // this, anyhow's default `to_string()` only emits the outermost context
+ // and the underlying cause (e.g. a `toml::de::Error` with line/column) is
+ // dropped — making the captured Sentry event undiagnosable. See
+ // OPENHUMAN-TAURI-B2 for an instance where this masked the real failure.
+ let message = format!("{err:#}");
+ report_error_message(&message, domain, operation, extra);
+}
+
+/// Report an error unless it is an expected user-state/config condition.
+///
+/// Expected conditions are logged at `info` or `warn` so the Sentry tracing
+/// layer records at most a breadcrumb, not an error event.
+pub fn report_error_or_expected<E: Display + ?Sized>(
+ err: &E,
+ domain: &str,
+ operation: &str,
+ extra: &[Tag<'_>],
+) {
+ let message = format!("{err:#}");
+ if let Some(kind) = expected_error_kind(&message) {
+ report_expected_message(kind, &message, domain, operation);
+ return;
+ }
+ report_error_message(&message, domain, operation, extra);
+}
+
+fn report_expected_message(kind: ExpectedErrorKind, message: &str, domain: &str, operation: &str) {
+ match kind {
+ ExpectedErrorKind::LocalAiDisabled => {
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected local-ai disabled error: {message}"
+ );
+ }
+ ExpectedErrorKind::ApiKeyMissing => {
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected API-key configuration error: {message}"
+ );
+ }
+ ExpectedErrorKind::NetworkUnreachable => {
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected network-unreachable error: {message}"
+ );
+ }
+ ExpectedErrorKind::TransientUpstreamHttp => {
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped transient upstream HTTP error: {message}"
+ );
+ }
+ ExpectedErrorKind::LocalAiBinaryMissing => {
+ // User-state condition: the piper binary (local TTS) isn't
+ // installed on this host; OpenHuman never installs it. The error message itself is
+ // the user-facing instruction ("Set PIPER_BIN or install
+ // piper.") — Sentry has nothing to act on, since we can't
+ // install the binary for them. OPENHUMAN-TAURI-9N is the
+ // canonical instance: `local_ai_tts` fails immediately
+ // (elapsed_ms=1) on a Windows host without piper installed.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected local-ai binary-missing error: {message}"
+ );
+ }
+ ExpectedErrorKind::BackendUserError => {
+ // 4xx from the integrations / composio backend client —
+ // user-input or auth-state failure that the backend already
+ // surfaced to the user via the structured error toast.
+ // OPENHUMAN-TAURI-BC: SharePoint authorize 400 because the
+ // user didn't fill in the required Tenant Name field.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected backend user-error response: {message}"
+ );
+ }
+ ExpectedErrorKind::ProviderUserState => {
+ // Third-party provider (composio, gmail OAuth, …) rejected the
+ // request for a user-state reason: trigger slug missing from
+ // composio's registry (OPENHUMAN-TAURI-3R / -3S), toolkit not
+ // enabled (OPENHUMAN-TAURI-34), OAuth scopes missing
+ // (OPENHUMAN-TAURI-33), or a required form field was left blank
+ // (OPENHUMAN-TAURI-97). The UI already surfaces the actionable
+ // error to the user — Sentry has no remediation path.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "provider_user_state",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected provider-user-state error: {message}"
+ );
+ }
+ ExpectedErrorKind::McpServerNeedsAuth => {
+ // A remote MCP server rejected the connect handshake with HTTP 401:
+ // it needs OAuth sign-in. `mcp::registry::connections::connect`
+ // already stores a `needs_auth` flag and the UI prompts the user to
+ // authenticate (#3733 / #3719) — but `mcp_clients_connect` re-raises
+ // the stringified error to the RPC dispatcher, where it was being
+ // captured as a full Sentry error (TAURI-RUST-CGP: ~1.2k events / 79
+ // users). Preventable user-state with no Sentry-actionable signal;
+ // demote to info so the breadcrumb survives but no error event fires.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "mcp_server_needs_auth",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected MCP needs-auth (401) error: {message}"
+ );
+ }
+ ExpectedErrorKind::WalletNotConfigured => {
+ // The user has not set up a wallet. That is the default state of an
+ // optional feature, the UI already prompts for setup, and no
+ // core-side change makes the call succeed — so demote to info: the
+ // breadcrumb survives for correlation, no error event fires.
+ // See `ExpectedErrorKind::WalletNotConfigured` (#5805).
+ //
+ // The raw message is deliberately NOT logged, and that follows from
+ // how this kind is matched. The classifier accepts the sentinel
+ // anywhere in the string so the demotion survives context wrapping —
+ // which means everything *around* the sentinel is arbitrary
+ // caller-supplied text. Today's wrappers are tame (`self_identity
+ // key_status: …`), but nothing constrains a future one, and a
+ // wrapper is exactly where an id, a path or a pasted value ends up.
+ // The permissive matcher is the right trade for correctness; paying
+ // for it with a careful log is the other half of that trade.
+ //
+ // Nothing is lost: the only part of the body this arm can vouch for
+ // is the sentinel itself, and it is a constant. `domain` and
+ // `operation` carry the correlation, which is what a breadcrumb is
+ // for. Same reasoning as the param-validation skip in
+ // `openhuman-rpc/src/server/http/rpc_handler.rs`, which redacts because its messages embed
+ // caller-supplied param names.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "wallet_not_configured",
+ "[observability] {domain}.{operation} skipped expected wallet-not-configured \
+ error (message withheld: the wrapper around the sentinel is caller-supplied)"
+ );
+ }
+ ExpectedErrorKind::ProviderConfigRejection => {
+ // User-config state: a custom cloud provider rejected the
+ // request because of the user's model / parameter setup — an
+ // OpenHuman abstract tier alias leaked to a provider that only
+ // speaks its native ids (#2079), an unknown / stale model pin
+ // (#2202), or a model-specific temperature constraint (#2076,
+ // Moonshot Kimi K2). The provider HTTP layer already demoted
+ // its own per-attempt event; this is the re-report raised
+ // again by agent.run_single / web_channel.run_chat_task. The
+ // UI surfaces an actionable "fix your model/provider settings"
+ // error — Sentry has no remediation path
+ // (OPENHUMAN-TAURI-WJ / -QW / -HB / -NH).
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "provider_config_rejection",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected provider config-rejection error: {message}"
+ );
+ }
+ ExpectedErrorKind::LocalAiCapabilityUnavailable => {
+ // User-state condition: the local-AI service refused a
+ // capability (vision summarization, vision asset download)
+ // because the user's RAM tier doesn't support it. The
+ // error message itself is the user-facing remediation
+ // ("Switch to the 4-8 GB tier or above to enable it.") —
+ // Sentry has nothing to act on. OPENHUMAN-TAURI-3B: 28
+ // hits in 4 days from `local_ai_download_asset` on a
+ // 0–4 GB tier requesting vision.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected local-ai capability-unavailable error: {message}"
+ );
+ }
+ ExpectedErrorKind::BudgetExhausted => {
+ // User-state condition: the backend reports the user is out of
+ // budget / credits / balance (HTTP 400 from the OpenHuman backend,
+ // surfaced by `providers::is_budget_exhausted_message`). The UI
+ // already surfaces this as an actionable toast — Sentry would
+ // turn each affected turn into noise (OPENHUMAN-TAURI-3M / -12 /
+ // -13). Demote to info so it still appears in breadcrumbs but
+ // never spawns a Sentry error event.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "budget",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected budget-exhausted error: {message}"
+ );
+ }
+ ExpectedErrorKind::ModuleUnavailable => log_module_unavailable(domain, operation, message),
+ ExpectedErrorKind::BackendUnavailable => {
+ // Build-state condition: no hosted backend is registered, so
+ // the hosted backend is unreachable by construction. Nothing to
+ // fix in Sentry — the host chose a backend-less core.
+ tracing::debug!(
+ domain = domain,
+ operation = operation,
+ kind = "backend_unavailable",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected backend-unavailable error: {message}"
+ );
+ }
+ ExpectedErrorKind::SessionExpired => {
+ // Auth-boundary condition: the user's JWT expired (or was never
+ // present). The JSON-RPC dispatch layer already handles the
+ // teardown — `Err` propagation publishes `DomainEvent::SessionExpired`
+ // which clears the stored token and flips the scheduler-gate
+ // signed-out override so background workers stand down — and the
+ // UI re-auths the user. The per-attempt error event from the
+ // upstream call site (agent.run_single, web_channel.run_chat_task)
+ // adds noise without signal: every mid-conversation 401 would
+ // emit one event before the cascade dampener kicks in
+ // (OPENHUMAN-TAURI-26, and the same upstream gap that
+ // OPENHUMAN-TAURI-1T's #1516 cascade fix dampened but did not
+ // close). Demote to info so the breadcrumb survives for trace
+ // correlation but Sentry sees no error event.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected session-expired error: {message}"
+ );
+ }
+ ExpectedErrorKind::LoopbackUnavailable => {
+ // In-process-core boot-window condition: a sibling component
+ // tried to reach `127.0.0.1:<port>` before the embedded core's
+ // HTTP listener finished binding (OPENHUMAN-TAURI-R5 / -R6).
+ // Self-resolves once startup completes. Demote at `debug!` —
+ // lower than the `warn!` we use for NetworkUnreachable because
+ // this isn't a user-environment problem; it's an internal
+ // lifecycle race that always recovers. We deliberately drop the
+ // raw `message` from the structured fields and format string and
+ // log only `domain` / `operation` / `kind` — the body adds no
+ // remediation signal (the URL is always loopback, the error is
+ // always "Connection refused") and keeping the breadcrumb sparse
+ // mirrors the per-#1719 review feedback (metadata over raw text
+ // for noise demotions).
+ tracing::debug!(
+ domain = domain,
+ operation = operation,
+ kind = "loopback_unavailable",
+ "[observability] {domain}.{operation} skipped expected loopback-unavailable error"
+ );
+ }
+ ExpectedErrorKind::PromptInjectionBlocked => {
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "prompt_injection_blocked",
+ "[observability] {domain}.{operation} skipped expected prompt-injection-blocked error"
+ );
+ }
+ ExpectedErrorKind::ContextWindowExceeded => {
+ // Request too long for the model's context window. The provider
+ // api_error cascade already demotes its own emit; this is the
+ // higher-layer re-report. Deterministic user-state — the UI
+ // shows the retry message and the user trims / starts a new
+ // chat. Demote to `warn!` (breadcrumb only) — same tier as the
+ // other usage-state conditions.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "context_window_exceeded",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected context-window-exceeded error: {message}"
+ );
+ }
+ ExpectedErrorKind::DiskFull => {
+ // Host filesystem out of space. The user must free space on
+ // their machine — Sentry can't help. Demote at `warn!` so a
+ // sustained spike still shows up in operator dashboards
+ // without turning every affected user-session into a Sentry
+ // error event. Drops TAURI-RUST-H4.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "disk_full",
+ "[observability] {domain}.{operation} skipped expected disk-full error"
+ );
+ }
+ ExpectedErrorKind::WindowsFileSystemLimitation => {
+ // Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665) —
+ // caused by fragmentation, USN journal overflow, or a
+ // filesystem filter driver bottleneck. The user must restart
+ // their machine, run a defrag, or free disk space — Sentry
+ // has no remediation path. Demote at `warn!` so a sustained
+ // spike shows up in dashboards without flooding Sentry.
+ // Drops TAURI-RUST-QT0 (6,050 events / 1 user).
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "windows_file_system_limitation",
+ "[observability] {domain}.{operation} skipped expected Windows file-system-limitation error (os error 665)"
+ );
+ }
+ ExpectedErrorKind::FilesystemUserPathInvalid => {
+ // User-input validation failure surfaced at the RPC
+ // boundary — e.g. `openhuman.vault_create` called with a
+ // `root_path` that doesn't exist. The typed error is
+ // already shown to the user; Sentry has no remediation
+ // path. Demote to `info!` — same tier as
+ // `PromptInjectionBlocked`, which is the closest severity
+ // class ("user input we already surfaced a typed error for";
+ // not operator-actionable like `DiskFull` / `NetworkUnreachable`).
+ //
+ // **Do not include the raw `message` here.** The message
+ // body embeds the user's local filesystem layout (username,
+ // project name, document directory, …) and
+ // `sentry_tracing_layer` in `core::logging` maps
+ // `Level::INFO` to `EventFilter::Breadcrumb` — so any
+ // formatted body would be attached as a breadcrumb to
+ // every subsequent Sentry event from this hub, leaking
+ // user paths into unrelated reports. Log only `domain` /
+ // `operation` / `kind` (no PII), matching the
+ // `LoopbackUnavailable` arm above ("metadata over raw text
+ // for noise demotions", per the #1719 review feedback).
+ // Full-path diagnostics for local debugging stay available
+ // via `RUST_LOG=…=debug` since `Level::DEBUG` / `TRACE`
+ // are mapped to `EventFilter::Ignore`.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "filesystem_user_path_invalid",
+ "[observability] {domain}.{operation} skipped expected filesystem path validation error"
+ );
+ }
+ ExpectedErrorKind::EmptyProviderResponse => {
+ // Model/user-config condition — the provider returned a
+ // completely empty body and the agent harness bailed with the
+ // user-facing retry message. The agent layer already suppresses
+ // this via the typed `AgentError::skips_sentry()` (PR #2790);
+ // this arm covers the `web_channel.run_chat_task` re-report
+ // where the type was flattened to a String. Demote to `warn!`
+ // (breadcrumb only) — same tier as `MaxIterationsExceeded`,
+ // the other deterministic agent-state outcome surfaced to the
+ // user via the `chat_error` event.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "empty_provider_response",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected empty-provider-response error: {message}"
+ );
+ }
+ ExpectedErrorKind::ChannelSupervisorRestart => {
+ // Channel supervisor caught a transient error from a channel
+ // listener (`spawn_supervised_listener`) and restarted it. The
+ // wrapper is language-agnostic — anchored on the Rust supervisor
+ // shape, not the inner error wording — so this catches both the
+ // English Discord-gateway body (TAURI-RUST-15 ~11.4 k events) and
+ // OS-localized variants (TAURI-RUST-BB Chinese WSAETIMEDOUT,
+ // ~815 events) that the English-only `NetworkUnreachable`
+ // matchers miss. Self-resolving via the supervisor's exponential
+ // backoff — Sentry has no remediation path. Sustained outages
+ // still surface through `health.bus` / `FAIL_ESCALATE_THRESHOLD`
+ // (separate code path, not affected by this demotion). Demote to
+ // `info!` so the breadcrumb survives for trace correlation but
+ // Sentry sees no error or warn event.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "channel_supervisor_restart",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected channel-supervisor restart: {message}"
+ );
+ }
+ ExpectedErrorKind::ConfigLoadTimedOut => {
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "config_load_timed_out",
+ error = %message,
+ "[observability] {domain}.{operation} skipped expected config-load timeout: {message}"
+ );
+ }
+ ExpectedErrorKind::ConfigReadIoFailure => {
+ // OS refused to read an existing config.toml (ACL-denied, locked by
+ // another process, OneDrive placeholder). User-environment state —
+ // we cannot make the file readable, and the same poll re-reports it
+ // every cycle (TAURI-RUST-DME). Demote at `warn!` so it stays in the
+ // local log for support without paging on every poll.
+ // Metadata-only: the raw message embeds the absolute config path
+ // (username / home dir). Keep this arm PII-free like the other
+ // path-sensitive demotions — domain/operation/kind are enough to
+ // see the condition without leaking the path into local logs.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "config_read_io_failure",
+ "[observability] {domain}.{operation} skipped expected config-read io failure (OS access denied/locked)"
+ );
+ }
+ ExpectedErrorKind::CodexCliAuthUnavailable => {
+ // User-state condition: the Codex CLI login at `~/.codex/auth.json`
+ // is missing / unparseable / has no tokens. The import RPC already
+ // returned the actionable "Run `codex login` first" string and the
+ // UI surfaces it inline — Sentry has nothing to act on. Demote at
+ // `info!` (mirrors `LocalAiBinaryMissing`). Do NOT include the raw
+ // `message`: it embeds the absolute `~/.codex/auth.json` path
+ // (home dir / username). Log only domain/operation/kind — no PII.
+ tracing::info!(
+ domain = domain,
+ operation = operation,
+ kind = "codex_cli_auth_unavailable",
+ "[observability] {domain}.{operation} skipped expected codex-cli auth-unavailable error"
+ );
+ }
+ ExpectedErrorKind::BackendErrorCodeOwned => {
+ // The classifier's canonical token aids triage without re-parsing
+ // provider-specific strings in this generic module.
+ let code = crate::inference::registry::classify_backend_error(None, message)
+ .and_then(|classification| classification.token)
+ .unwrap_or_default();
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "backend_error_code",
+ error_code = %code,
+ "[observability] {domain}.{operation} skipped backend-owned errorCode={code} error: {message}"
+ );
+ }
+ ExpectedErrorKind::UpstreamEdgeBlock => {
+ // Provider embed call hit an edge/CDN/WAF or regional 403 block —
+ // the body is a generic HTML gateway page, not the provider's JSON
+ // error. Endpoint correct, key sent, retry can't clear an edge
+ // policy; the embedding caller degrades gracefully and the UI
+ // surfaces it. Demote at `warn!` so the breadcrumb retains the
+ // shape for triage without spawning an event (TAURI-RUST-8S3).
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "upstream_edge_block",
+ error = %message,
+ "[observability] {domain}.{operation} skipped upstream edge/CDN 403 block: {message}"
+ );
+ }
+ ExpectedErrorKind::ApprovalNoPendingRace => {
+ // Benign approval race (TAURI-RUST-5EH): the request_id was already
+ // decided / lazily expired / superseded — `store::get_decision`
+ // confirmed a persisted decision exists, so this is a double-tap,
+ // two-operator, or expiry-while-live race classified benign by the
+ // inline-approvals design spec, not a lost registration. Demote at
+ // `warn!` so a sustained spike still shows in operator dashboards
+ // without paging. The genuine never-registered case takes the
+ // capture path (distinct wording) and stays a Sentry signal.
+ tracing::warn!(
+ domain = domain,
+ operation = operation,
+ kind = "approval_no_pending_race",
+ "[observability] {domain}.{operation} skipped benign already-decided/expired approval race: {message}"
+ );
+ }
+ }
+}
+
+/// Distinct `tracing::Metadata::target()` we set on the diagnostic
+/// `tracing::error!` emitted from [`report_error_message`].
+///
+/// Sentry capture for this helper happens via an explicit
+/// `sentry::capture_message` call below — not via the `sentry-tracing`
+/// layer scooping up the `tracing::error!` event. The production
+/// `sentry_tracing_layer()` in `core::logging` filters events with this
+/// target to `EventFilter::Ignore` so we never double-report (one direct
+/// `capture_message`, one tracing-bridge capture of the same condition).
+///
+/// Why direct capture instead of relying on the bridge: the bridge worked
+/// in steady-state but flaked under parallel test scheduling
+/// (`thread_not_found_rpc_error_does_not_report_to_sentry` repeatedly hit
+/// `events.len() == 0` in CI even with a thread-default subscriber wired
+/// up — likely a Linux-only thread-local ordering quirk in
+/// `sentry-tracing`'s `Hub::current()` lookup at event-emit time). Direct
+/// `sentry::capture_message` synchronously routes through the active hub
+/// and is deterministic, which keeps both production reporting and tests
+/// honest.
+pub const REPORT_ERROR_TRACING_TARGET: &str = "openhuman::observability::report_error";
+
+/// Cooldown period for rate-limiting repeated filesystem-limit errors.
+/// Within this window, the same (domain, operation) pair that carries
+/// `(os error 665)` is suppressed — Sentry never receives a duplicate
+/// Sentry event, and only a `debug!` log line is emitted.
+#[cfg(feature = "crash-reporting")]
+const FS_ERROR_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(300); // 5 min
+
+/// In-process state tracking the last-reported time of `ERROR_FILE_SYSTEM_LIMITATION`
+/// (os error 665) events, keyed by `(domain, operation)`. This prevents
+/// unthrottled retry loops from flooding Sentry (TAURI-RUST-QT0: 6,050 events /
+/// 1 user).
+#[cfg(feature = "crash-reporting")]
+static LAST_FS_LIMIT_REPORT: std::sync::LazyLock<
+ std::sync::Mutex<std::collections::HashMap<(String, String), std::time::Instant>>,
+> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new()));
+
+/// Returns `true` when `(domain, operation)` was already reported with
+/// an error-665 message within [`FS_ERROR_COOLDOWN`]. Cleans up stale
+/// entries older than the cooldown on every call.
+#[cfg(feature = "crash-reporting")]
+fn was_recently_reported(domain: &str, operation: &str) -> bool {
+ let mut guard = LAST_FS_LIMIT_REPORT
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ let now = std::time::Instant::now();
+ // Prune stale entries older than cooldown (single-pass to keep map bounded).
+ guard.retain(|_, last| now.duration_since(*last) < FS_ERROR_COOLDOWN);
+ let key = (domain.to_string(), operation.to_string());
+ if let Some(last) = guard.get(&key) {
+ if now.duration_since(*last) < FS_ERROR_COOLDOWN {
+ return true;
+ }
+ }
+ guard.insert(key, now);
+ false
+}
+
+pub(crate) fn report_error_message(
+ message: &str,
+ domain: &str,
+ operation: &str,
+ extra: &[Tag<'_>],
+) {
+ // Redact secret-looking spans (bearer tokens, API keys, `sk-` keys) before
+ // `message` reaches any log sink or Sentry event. The parallel `tracing`
+ // log line below is emitted in every build — including slim builds with no
+ // `crash-reporting` `before_send` hook — so scrub once, up front.
+ let scrubbed = crate::core::log_redaction::scrub_secrets(message);
+ let message = scrubbed.as_str();
+ // Rate-limit Windows `ERROR_FILE_SYSTEM_LIMITATION` (os error 665)
+ // events: skip the Sentry capture if the same (domain, operation) pair
+ // was already reported within the last 5 minutes. This prevents
+ // unthrottled retry loops from flooding Sentry (TAURI-RUST-QT0:
+ // 6,050 events / 1 user). The `before_send` chain also filters these,
+ // but this early-exit avoids the Sentry scope overhead entirely.
+ #[cfg(feature = "crash-reporting")]
+ if message.contains("os error 665") && was_recently_reported(domain, operation) {
+ tracing::debug!(
+ target: REPORT_ERROR_TRACING_TARGET,
+ domain = domain,
+ operation = operation,
+ "[observability] {domain}.{operation} rate-limited os error 665 (reported within cooldown)"
+ );
+ return;
+ }
+ // Sentry-touching behaviour is gated behind `crash-reporting`. The
+ // diagnostic `tracing::error!` stays compiled in both builds (see the
+ // `#[cfg(not(...))]` companion below) so stderr / file appenders keep the
+ // record even in a sentry-free build.
+ #[cfg(feature = "crash-reporting")]
+ sentry::with_scope(
+ |scope| {
+ scope.set_tag("domain", domain);
+ scope.set_tag("operation", operation);
+ for (k, v) in extra {
+ scope.set_tag(k, v);
+ }
+ },
+ || {
+ // Direct, synchronous Sentry capture — see
+ // `REPORT_ERROR_TRACING_TARGET` for why we don't rely on the
+ // `sentry-tracing` layer for this call site.
+ sentry::capture_message(message, sentry::Level::Error);
+ // Diagnostic log line for stderr / file appenders. Tagged with
+ // the marker target so the production sentry-tracing layer
+ // skips it (no double Sentry event).
+ tracing::error!(
+ target: REPORT_ERROR_TRACING_TARGET,
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} failed: {message}"
+ );
+ },
+ );
+ #[cfg(not(feature = "crash-reporting"))]
+ {
+ // Sentry compiled out: `extra` tags have no scope to attach to, so
+ // discard them explicitly to avoid an unused-variable warning while
+ // still emitting the diagnostic log line.
+ let _ = extra;
+ tracing::error!(
+ target: REPORT_ERROR_TRACING_TARGET,
+ domain = domain,
+ operation = operation,
+ error = %message,
+ "[observability] {domain}.{operation} failed: {message}"
+ );
+ }
+}
+
+/// Capture a message to Sentry at **warning** severity with structured tags.
+///
+/// Mirror of [`report_error_message`] but at `sentry::Level::Warning`: the
+/// event is still recorded in Sentry (so it stays available for triage and
+/// dashboards) while warning-severity events do not trip the error-rate
+/// alert/paging rules that `Level::Error` events do (see the
+/// `sentry_tracing_layer` mapping in `core::logging`, where `ERROR` becomes a
+/// captured `Event` and `WARN`/`INFO` only a `Breadcrumb`). Use this for
+/// transport-boundary conditions worth seeing in aggregate that are never an
+/// actionable core defect — e.g. unrecognised RPC method names (#3567).
+///
+/// Like [`report_error_message`], capture is an explicit, synchronous
+/// `sentry::capture_message` rather than the `sentry-tracing` bridge; the
+/// accompanying diagnostic line is tagged with [`REPORT_ERROR_TRACING_TARGET`]
+/// so the production layer ignores it and we never double-report.
+// Its sole caller is the `http-server`-gated RPC handler (unrecognised-method
+// reporting, #3567), so it has no caller in a slim build (#5048). Kept compiled
+// for the crash-reporting carve-out; the allow keeps the disabled build quiet.
+#[cfg_attr(not(feature = "http-server"), allow(dead_code))]
+pub fn report_warning_message(message: &str, domain: &str, operation: &str, extra: &[Tag<'_>]) {
+ // Redact secret-looking spans before `message` reaches any log sink or
+ // Sentry event — see the note in `report_error_message`.
+ let scrubbed = crate::core::log_redaction::scrub_secrets(message);
+ let message = scrubbed.as_str();
+ // Sentry-touching behaviour is gated behind `crash-reporting`; the
+ // diagnostic `tracing::warn!` stays compiled in both builds (see the
+ // `#[cfg(not(...))]` companion below).
+ #[cfg(feature = "crash-reporting")]
+ sentry::with_scope(
+ |scope| {
+ scope.set_tag("domain", domain);
+ scope.set_tag("operation", operation);
+ for (k, v) in extra {
+ scope.set_tag(k, v);
+ }
+ },
+ || {
+ sentry::capture_message(message, sentry::Level::Warning);
+ tracing::warn!(
+ target: REPORT_ERROR_TRACING_TARGET,
+ domain = domain,
+ operation = operation,
+ message = %message,
+ "[observability] {domain}.{operation} warning: {message}"
+ );
+ },
+ );
+ #[cfg(not(feature = "crash-reporting"))]
+ {
+ let _ = extra;
+ tracing::warn!(
+ target: REPORT_ERROR_TRACING_TARGET,
+ domain = domain,
+ operation = operation,
+ message = %message,
+ "[observability] {domain}.{operation} warning: {message}"
+ );
+ }
+}
+
+/// Returns true when a Sentry event is a per-attempt provider HTTP failure
+/// that the reliable-provider layer already handles via retry + fallback.
+///
+/// The primary suppression lives at the call site
+/// (`crate::inference::provider::http_error::should_report_provider_http_failure`),
+/// which short-circuits transient codes before `report_error` ever fires.
+/// This helper is intended for use inside the `sentry::ClientOptions`
+/// `before_send` hook as defense-in-depth — it catches any future call
+/// site that emits a `tracing::error!` with the same shape but bypasses
+/// the classifier.
+///
+/// Match criteria (all required):
+/// - tag `domain == "llm_provider"` — pins the filter to provider-originated
+/// events so an unrelated subsystem emitting `failure=non_2xx`/`status=503`
+/// for its own reasons doesn't get silently dropped
+/// - tag `failure == "non_2xx"` (the marker set by `ops::api_error`)
+/// - tag `status` parses to one of [`TRANSIENT_PROVIDER_HTTP_STATUSES`]
+#[cfg(feature = "crash-reporting")]